diff --git a/src/JobHost/Program.cs b/src/JobHost/Program.cs index 20421b4..3f38401 100644 --- a/src/JobHost/Program.cs +++ b/src/JobHost/Program.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.Tracing; using System.Linq; using System.Text; @@ -13,6 +14,12 @@ class Program { static void Main(string[] args) { + if (args.Length > 0 && String.Equals(args[0], "dbg", StringComparison.OrdinalIgnoreCase)) + { + args = args.Skip(1).ToArray(); + Debugger.Launch(); + } + Arguments parsed; try { diff --git a/src/NuGet.Services.Work.Cloud/ServiceConfiguration.Local.cscfg b/src/NuGet.Services.Work.Cloud/ServiceConfiguration.Local.cscfg index 8980178..051b193 100644 --- a/src/NuGet.Services.Work.Cloud/ServiceConfiguration.Local.cscfg +++ b/src/NuGet.Services.Work.Cloud/ServiceConfiguration.Local.cscfg @@ -12,7 +12,7 @@ - + diff --git a/src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs b/src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs new file mode 100644 index 0000000..9a80efd --- /dev/null +++ b/src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs @@ -0,0 +1,41 @@ +using System; +using System.Collections.Generic; +using System.Data.SqlClient; +using System.Diagnostics.Tracing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using NuGet.Indexing; +using NuGet.Services.Configuration; + +namespace NuGet.Services.Work.Jobs.Bases +{ + public abstract class SearchIndexJobHandlerBase : JobHandler + where T : EventSource + { + public SqlConnectionStringBuilder PackageDatabase { get; set; } + public CloudStorageAccount StorageAccount { get; set; } + public string StorageContainerName { get; set; } + public string LocalIndexFolder { get; set; } + + protected ConfigurationHub Config { get; set; } + + public SearchIndexJobHandlerBase(ConfigurationHub config) + { + Config = config; + } + + protected override InvocationResult BindContext(InvocationContext context) + { + var result = base.BindContext(context); + + // Load default values + PackageDatabase = PackageDatabase ?? Config.Sql.Legacy; + StorageAccount = StorageAccount ?? Config.Storage.Primary; + StorageContainerName = StorageContainerName ?? "ng-search"; + + return result; + } + } +} diff --git a/src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs b/src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs index 6159741..8044282 100644 --- a/src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs +++ b/src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs @@ -9,6 +9,7 @@ using Dapper; using Microsoft.WindowsAzure.Storage; using Microsoft.WindowsAzure.Storage.Blob; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NuGet.Services.Client; using NuGet.Services.Configuration; @@ -56,8 +57,10 @@ protected internal override async Task Execute() } // Gather overall rankings + JObject report = new JObject(); Log.GatheringOverallRankings(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog); var overallData = await GatherOverallRankings(); + report.Add("Rank", overallData); Log.GatheredOverallRankings(overallData.Count); // Get project types @@ -66,27 +69,18 @@ protected internal override async Task Execute() Log.GotAvailableProjectTypes(projectTypes.Count); // Gather data by project type - IDictionary> byProjectType = new Dictionary>(); int count = 0; Log.GatheringProjectTypeRankings(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog); foreach (var projectType in projectTypes) { Log.GatheringProjectTypeRanking(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog, projectType); var data = await GatherProjectTypeRanking(projectType); + report.Add(projectType, data); Log.GatheredProjectTypeRanking(data.Count, projectType); count += data.Count; - - byProjectType.Add(projectType, data); } Log.GatheredProjectTypeRankings(count); - // Generate the report - var report = new SearchRankingReport() - { - Overall = overallData, - ByProjectType = byProjectType - }; - // Write the JSON blob if (!String.IsNullOrEmpty(OutputDirectory)) { @@ -99,7 +93,7 @@ protected internal override async Task Execute() } } - private async Task WriteToFile(SearchRankingReport report) + private async Task WriteToFile(JObject report) { string fullPath = Path.Combine(OutputDirectory, ReportName); string parentDir = Path.GetDirectoryName(fullPath); @@ -116,20 +110,20 @@ private async Task WriteToFile(SearchRankingReport report) } using (var writer = new StreamWriter(File.OpenWrite(fullPath))) { - await writer.WriteAsync(JsonFormat.Serialize(report)); + await writer.WriteAsync(report.ToString(Formatting.Indented)); } } Log.WroteReportBlob(fullPath); } - private async Task WriteToBlob(SearchRankingReport report) + private async Task WriteToBlob(JObject report) { var blob = DestinationContainer.GetBlockBlobReference(ReportName); Log.WritingReportBlob(blob.Uri.AbsoluteUri); if (!WhatIf) { blob.Properties.ContentType = MimeTypes.Json; - await blob.UploadTextAsync(JsonFormat.Serialize(report)); + await blob.UploadTextAsync(report.ToString(Formatting.Indented)); } Log.WroteReportBlob(blob.Uri.AbsoluteUri); } @@ -145,7 +139,7 @@ private void LoadDefaults() } } - private async Task> GatherOverallRankings() + private async Task GatherOverallRankings() { using (var connection = await WarehouseConnection.ConnectTo()) { @@ -153,7 +147,9 @@ private async Task> GatherOverallRankings() var script = await ResourceHelpers.ReadResourceFile("NuGet.Services.Work.Jobs.Scripts.SearchRanking_Overall.sql"); // Execute it and return the results - return (await connection.QueryAsync(script)).ToList(); + return new JArray( + (await connection.QueryAsync(script)) + .Select(e => e.PackageId)); } } @@ -166,7 +162,7 @@ private async Task> GetProjectTypes() } } - private async Task> GatherProjectTypeRanking(string projectType) + private async Task GatherProjectTypeRanking(string projectType) { using (var connection = await WarehouseConnection.ConnectTo()) { @@ -174,7 +170,9 @@ private async Task> GatherProjectTypeRanking(string pr var script = await ResourceHelpers.ReadResourceFile("NuGet.Services.Work.Jobs.Scripts.SearchRanking_ByProjectType.sql"); // Execute it and return the results - return (await connection.QueryAsync(script, new { ProjectGuid = projectType })).ToList(); + return new JArray( + (await connection.QueryAsync(script, new { ProjectGuid = projectType })) + .Select(e => e.PackageId)); } } } diff --git a/src/NuGet.Services.Work/Jobs/Models/SearchRankingReport.cs b/src/NuGet.Services.Work/Jobs/Models/SearchRankingReport.cs deleted file mode 100644 index fac82a9..0000000 --- a/src/NuGet.Services.Work/Jobs/Models/SearchRankingReport.cs +++ /dev/null @@ -1,20 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; - -namespace NuGet.Services.Work.Jobs.Models -{ - public class SearchRankingReport - { - public IList Overall { get; set; } - public IDictionary> ByProjectType { get; set; } - - public SearchRankingReport() - { - Overall = new List(); - ByProjectType = new Dictionary>(); - } - } -} diff --git a/src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs b/src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs new file mode 100644 index 0000000..bca4e29 --- /dev/null +++ b/src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections.Generic; +using System.Data.SqlClient; +using System.Diagnostics.Tracing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Microsoft.WindowsAzure.Storage; +using Microsoft.WindowsAzure.Storage.Blob; +using NuGet.Indexing; +using NuGet.Services.Configuration; +using NuGet.Services.Work.Jobs.Bases; +using NuGet.Services.Work.Monitoring; + +namespace NuGet.Services.Work.Jobs +{ + public class RebuildSearchIndexJob : SearchIndexJobHandlerBase + { + public RebuildSearchIndexJob(ConfigurationHub config) : base(config) { } + + protected internal override async Task Execute() + { + // This job can take a long time and is run manually. Make the job timeout long + await Extend(TimeSpan.FromHours(12)); + + // Run the task + Log.BeginningIndex( + PackageDatabase.DataSource + "/" + PackageDatabase.InitialCatalog, + String.IsNullOrEmpty(LocalIndexFolder) ? + (StorageAccount.Credentials.AccountName + "/" + StorageContainerName) : + LocalIndexFolder); + FullBuildTask task = new FullBuildTask() + { + SqlConnectionString = PackageDatabase.ConnectionString, + StorageAccount = StorageAccount, + Container = String.IsNullOrEmpty(LocalIndexFolder) ? + StorageContainerName : + null, + Folder = LocalIndexFolder, + Log = new EventSourceWriter(Log.IndexingTrace) + }; + task.Execute(); + Log.FinishedIndex(); + } + } + + [EventSource(Name="Outercurve-NuGet-Jobs-RebuildSearchIndex")] + public class RebuildSearchIndexEventSource : EventSource + { + public static readonly RebuildSearchIndexEventSource Log = new RebuildSearchIndexEventSource(); + private RebuildSearchIndexEventSource() { } + + [Event( + eventId: 1, + Level = EventLevel.Informational, + Message = "Indexing Trace: {0}")] + public void IndexingTrace(string message) { WriteEvent(1, message); } + + [Event( + eventId: 2, + Level = EventLevel.Informational, + Message = "Beginning Index Rebuild from {0} to {1}", + Task = Tasks.Indexing, + Opcode = EventOpcode.Start)] + public void BeginningIndex(string source, string destination) { WriteEvent(2, source, destination); } + + [Event( + eventId: 3, + Level = EventLevel.Informational, + Message = "Finished Index Rebuild", + Task = Tasks.Indexing, + Opcode = EventOpcode.Stop)] + public void FinishedIndex() { WriteEvent(3); } + + public static class Tasks + { + public const EventTask Indexing = (EventTask)0x1; + } + } +} diff --git a/src/NuGet.Services.Work/Jobs/UpdateSearchIndexJob.cs b/src/NuGet.Services.Work/Jobs/UpdateSearchIndexJob.cs new file mode 100644 index 0000000..ea26ab7 --- /dev/null +++ b/src/NuGet.Services.Work/Jobs/UpdateSearchIndexJob.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics.Tracing; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using NuGet.Indexing; +using NuGet.Services.Configuration; +using NuGet.Services.Work.Jobs.Bases; +using NuGet.Services.Work.Monitoring; + +namespace NuGet.Services.Work.Jobs +{ + public class UpdateSearchIndexJob : SearchIndexJobHandlerBase + { + public UpdateSearchIndexJob(ConfigurationHub config) : base(config) { } + + protected internal override Task Execute() + { + // Run the task + UpdateIndexTask task = new UpdateIndexTask() + { + SqlConnectionString = PackageDatabase.ConnectionString, + StorageAccount = StorageAccount, + Container = String.IsNullOrEmpty(LocalIndexFolder) ? + (StorageContainerName ?? "ng-search") : + null, + Folder = LocalIndexFolder, + Log = new EventSourceWriter(Log.IndexingTrace), + }; + task.Execute(); + + return Task.FromResult(0); + } + } + + [EventSource(Name="Outercurve-NuGet-Jobs-UpdateSearchIndex")] + public class UpdateSearchIndexEventSource : EventSource + { + public static readonly UpdateSearchIndexEventSource Log = new UpdateSearchIndexEventSource(); + private UpdateSearchIndexEventSource() { } + + [Event( + eventId: 1, + Level = EventLevel.Informational, + Message = "Indexing Trace: {0}")] + public void IndexingTrace(string message) { WriteEvent(1, message); } + } +} diff --git a/src/NuGet.Services.Work/Monitoring/EventSourceWriter.cs b/src/NuGet.Services.Work/Monitoring/EventSourceWriter.cs new file mode 100644 index 0000000..43651ac --- /dev/null +++ b/src/NuGet.Services.Work/Monitoring/EventSourceWriter.cs @@ -0,0 +1,49 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace NuGet.Services.Work.Monitoring +{ + /// + /// Writes lines to a specified EventSource method + /// + public class EventSourceWriter : TextWriter + { + private Action _receiver; + private StringBuilder _buffer = new StringBuilder(); + + public override Encoding Encoding + { + get { return Encoding.Default; } + } + + public EventSourceWriter(Action receiver) + { + _receiver = receiver; + } + + public override void Write(char value) + { + _buffer.Append(value); + CheckFlushLine(); + } + + public override void Flush() + { + _receiver(_buffer.ToString().Trim()); + _buffer.Clear(); + } + + private void CheckFlushLine() + { + if (_buffer.ToString().EndsWith(NewLine)) + { + // Flush the buffer + Flush(); + } + } + } +} diff --git a/src/NuGet.Services.Work/NuGet.Services.Work.csproj b/src/NuGet.Services.Work/NuGet.Services.Work.csproj index 89ef179..15c7e37 100644 --- a/src/NuGet.Services.Work/NuGet.Services.Work.csproj +++ b/src/NuGet.Services.Work/NuGet.Services.Work.csproj @@ -44,6 +44,48 @@ ..\..\packages\Dapper.1.13\lib\net45\Dapper.dll + + ..\..\packages\EntityFramework.5.0.0\lib\net45\EntityFramework.dll + + + ..\..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll + + + ..\..\packages\Lucene.Net.3.0.3\lib\NET40\Lucene.Net.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Analyzers.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Core.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.FastVectorHighlighter.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Highlighter.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Memory.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Queries.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Regex.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.SimpleFacetedSearch.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Snowball.dll + + + ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.SpellChecker.dll + + + ..\..\packages\Lucene.Net.Store.Azure.2.0.4937.26631\lib\net40\Lucene.Net.Store.Azure.dll + ..\..\packages\Microsoft.Data.Edm.5.6.0\lib\net40\Microsoft.Data.Edm.dll @@ -95,11 +137,20 @@ False ..\..\packages\Nuget.Core.2.8.0\lib\net40-Client\NuGet.Core.dll - - ..\..\packages\NuGet.Services.Platform.3.0.1-rel\lib\net45\NuGet.Services.Platform.dll + + False + ..\..\packages\NuGet.Indexing.3.0.2-alpha-9\lib\net45\NuGet.Indexing.dll + + + False + ..\..\packages\NuGet.Services.Platform.3.0.1-rel-3\lib\net45\NuGet.Services.Platform.dll - - ..\..\packages\NuGet.Services.Platform.Client.3.0.1-rel\lib\portable-net45+wp80+win\NuGet.Services.Platform.Client.dll + + False + ..\..\packages\NuGet.Services.Platform.Client.3.0.1-rel-3\lib\portable-net45+wp80+win\NuGet.Services.Platform.Client.dll + + + ..\..\packages\NuGetGallery.Core.2.0.0-alpha-118\lib\net45\NuGetGallery.Core.dll ..\..\packages\Owin.1.0\lib\net40\Owin.dll @@ -107,6 +158,7 @@ + @@ -130,9 +182,6 @@ ..\..\packages\Rx-Linq.2.2.2\lib\net45\System.Reactive.Linq.dll - - ..\..\packages\Rx-PlatformServices.2.2.2\lib\net45\System.Reactive.PlatformServices.dll - @@ -190,6 +239,7 @@ + @@ -200,9 +250,9 @@ - + @@ -213,7 +263,9 @@ + + diff --git a/src/NuGet.Services.Work/WorkService.cs b/src/NuGet.Services.Work/WorkService.cs index 6c8fca2..ab1df46 100644 --- a/src/NuGet.Services.Work/WorkService.cs +++ b/src/NuGet.Services.Work/WorkService.cs @@ -210,22 +210,24 @@ public IObservable RunJob(string job, string payload) QueuedAt = DateTime.UtcNow, NextVisibleAt = DateTime.UtcNow + TimeSpan.FromMinutes(5) }); - var buffer = new ReplaySubject(); - var capture = new InvocationLogCapture(invocation); - capture.Subscribe(buffer.OnNext, buffer.OnError); - runner.Dispatch(invocation, capture, CancellationToken.None).ContinueWith(t => + return Observable.Create(observer => { - if (t.IsFaulted) + var capture = new InvocationLogCapture(invocation); + capture.Subscribe(e => observer.OnNext(e), ex => observer.OnError(ex)); + runner.Dispatch(invocation, capture, CancellationToken.None).ContinueWith(t => { - buffer.OnError(t.Exception); - } - else - { - buffer.OnCompleted(); - } - return t; + if (t.IsFaulted) + { + observer.OnError(t.Exception); + } + else + { + observer.OnCompleted(); + } + return t; + }); + return () => { }; // No action on unsubscribe }); - return buffer; } protected override void ConfigureAttributeRouting(DefaultInlineConstraintResolver resolver) diff --git a/src/NuGet.Services.Work/app.config b/src/NuGet.Services.Work/app.config index b00b86a..fe616c5 100644 --- a/src/NuGet.Services.Work/app.config +++ b/src/NuGet.Services.Work/app.config @@ -1,65 +1,76 @@  - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/NuGet.Services.Work/packages.config b/src/NuGet.Services.Work/packages.config index 69b3123..c83f187 100644 --- a/src/NuGet.Services.Work/packages.config +++ b/src/NuGet.Services.Work/packages.config @@ -5,6 +5,10 @@ + + + + @@ -23,14 +27,17 @@ - - + + + + + - \ No newline at end of file +