diff --git a/src/JobHost/Program.cs b/src/JobHost/Program.cs
index 20421b4..3f38401 100644
--- a/src/JobHost/Program.cs
+++ b/src/JobHost/Program.cs
@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
+using System.Diagnostics;
using System.Diagnostics.Tracing;
using System.Linq;
using System.Text;
@@ -13,6 +14,12 @@ class Program
{
static void Main(string[] args)
{
+ if (args.Length > 0 && String.Equals(args[0], "dbg", StringComparison.OrdinalIgnoreCase))
+ {
+ args = args.Skip(1).ToArray();
+ Debugger.Launch();
+ }
+
Arguments parsed;
try
{
diff --git a/src/NuGet.Services.Work.Cloud/ServiceConfiguration.Local.cscfg b/src/NuGet.Services.Work.Cloud/ServiceConfiguration.Local.cscfg
index 8980178..051b193 100644
--- a/src/NuGet.Services.Work.Cloud/ServiceConfiguration.Local.cscfg
+++ b/src/NuGet.Services.Work.Cloud/ServiceConfiguration.Local.cscfg
@@ -12,7 +12,7 @@
-
+
diff --git a/src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs b/src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs
new file mode 100644
index 0000000..9a80efd
--- /dev/null
+++ b/src/NuGet.Services.Work/Jobs/Bases/SearchIndexJobHandlerBase.cs
@@ -0,0 +1,41 @@
+using System;
+using System.Collections.Generic;
+using System.Data.SqlClient;
+using System.Diagnostics.Tracing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.WindowsAzure.Storage;
+using NuGet.Indexing;
+using NuGet.Services.Configuration;
+
+namespace NuGet.Services.Work.Jobs.Bases
+{
+ public abstract class SearchIndexJobHandlerBase : JobHandler
+ where T : EventSource
+ {
+ public SqlConnectionStringBuilder PackageDatabase { get; set; }
+ public CloudStorageAccount StorageAccount { get; set; }
+ public string StorageContainerName { get; set; }
+ public string LocalIndexFolder { get; set; }
+
+ protected ConfigurationHub Config { get; set; }
+
+ public SearchIndexJobHandlerBase(ConfigurationHub config)
+ {
+ Config = config;
+ }
+
+ protected override InvocationResult BindContext(InvocationContext context)
+ {
+ var result = base.BindContext(context);
+
+ // Load default values
+ PackageDatabase = PackageDatabase ?? Config.Sql.Legacy;
+ StorageAccount = StorageAccount ?? Config.Storage.Primary;
+ StorageContainerName = StorageContainerName ?? "ng-search";
+
+ return result;
+ }
+ }
+}
diff --git a/src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs b/src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs
index 6159741..8044282 100644
--- a/src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs
+++ b/src/NuGet.Services.Work/Jobs/GenerateSearchRankingsJob.cs
@@ -9,6 +9,7 @@
using Dapper;
using Microsoft.WindowsAzure.Storage;
using Microsoft.WindowsAzure.Storage.Blob;
+using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NuGet.Services.Client;
using NuGet.Services.Configuration;
@@ -56,8 +57,10 @@ protected internal override async Task Execute()
}
// Gather overall rankings
+ JObject report = new JObject();
Log.GatheringOverallRankings(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog);
var overallData = await GatherOverallRankings();
+ report.Add("Rank", overallData);
Log.GatheredOverallRankings(overallData.Count);
// Get project types
@@ -66,27 +69,18 @@ protected internal override async Task Execute()
Log.GotAvailableProjectTypes(projectTypes.Count);
// Gather data by project type
- IDictionary> byProjectType = new Dictionary>();
int count = 0;
Log.GatheringProjectTypeRankings(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog);
foreach (var projectType in projectTypes)
{
Log.GatheringProjectTypeRanking(WarehouseConnection.DataSource, WarehouseConnection.InitialCatalog, projectType);
var data = await GatherProjectTypeRanking(projectType);
+ report.Add(projectType, data);
Log.GatheredProjectTypeRanking(data.Count, projectType);
count += data.Count;
-
- byProjectType.Add(projectType, data);
}
Log.GatheredProjectTypeRankings(count);
- // Generate the report
- var report = new SearchRankingReport()
- {
- Overall = overallData,
- ByProjectType = byProjectType
- };
-
// Write the JSON blob
if (!String.IsNullOrEmpty(OutputDirectory))
{
@@ -99,7 +93,7 @@ protected internal override async Task Execute()
}
}
- private async Task WriteToFile(SearchRankingReport report)
+ private async Task WriteToFile(JObject report)
{
string fullPath = Path.Combine(OutputDirectory, ReportName);
string parentDir = Path.GetDirectoryName(fullPath);
@@ -116,20 +110,20 @@ private async Task WriteToFile(SearchRankingReport report)
}
using (var writer = new StreamWriter(File.OpenWrite(fullPath)))
{
- await writer.WriteAsync(JsonFormat.Serialize(report));
+ await writer.WriteAsync(report.ToString(Formatting.Indented));
}
}
Log.WroteReportBlob(fullPath);
}
- private async Task WriteToBlob(SearchRankingReport report)
+ private async Task WriteToBlob(JObject report)
{
var blob = DestinationContainer.GetBlockBlobReference(ReportName);
Log.WritingReportBlob(blob.Uri.AbsoluteUri);
if (!WhatIf)
{
blob.Properties.ContentType = MimeTypes.Json;
- await blob.UploadTextAsync(JsonFormat.Serialize(report));
+ await blob.UploadTextAsync(report.ToString(Formatting.Indented));
}
Log.WroteReportBlob(blob.Uri.AbsoluteUri);
}
@@ -145,7 +139,7 @@ private void LoadDefaults()
}
}
- private async Task> GatherOverallRankings()
+ private async Task GatherOverallRankings()
{
using (var connection = await WarehouseConnection.ConnectTo())
{
@@ -153,7 +147,9 @@ private async Task> GatherOverallRankings()
var script = await ResourceHelpers.ReadResourceFile("NuGet.Services.Work.Jobs.Scripts.SearchRanking_Overall.sql");
// Execute it and return the results
- return (await connection.QueryAsync(script)).ToList();
+ return new JArray(
+ (await connection.QueryAsync(script))
+ .Select(e => e.PackageId));
}
}
@@ -166,7 +162,7 @@ private async Task> GetProjectTypes()
}
}
- private async Task> GatherProjectTypeRanking(string projectType)
+ private async Task GatherProjectTypeRanking(string projectType)
{
using (var connection = await WarehouseConnection.ConnectTo())
{
@@ -174,7 +170,9 @@ private async Task> GatherProjectTypeRanking(string pr
var script = await ResourceHelpers.ReadResourceFile("NuGet.Services.Work.Jobs.Scripts.SearchRanking_ByProjectType.sql");
// Execute it and return the results
- return (await connection.QueryAsync(script, new { ProjectGuid = projectType })).ToList();
+ return new JArray(
+ (await connection.QueryAsync(script, new { ProjectGuid = projectType }))
+ .Select(e => e.PackageId));
}
}
}
diff --git a/src/NuGet.Services.Work/Jobs/Models/SearchRankingReport.cs b/src/NuGet.Services.Work/Jobs/Models/SearchRankingReport.cs
deleted file mode 100644
index fac82a9..0000000
--- a/src/NuGet.Services.Work/Jobs/Models/SearchRankingReport.cs
+++ /dev/null
@@ -1,20 +0,0 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
-
-namespace NuGet.Services.Work.Jobs.Models
-{
- public class SearchRankingReport
- {
- public IList Overall { get; set; }
- public IDictionary> ByProjectType { get; set; }
-
- public SearchRankingReport()
- {
- Overall = new List();
- ByProjectType = new Dictionary>();
- }
- }
-}
diff --git a/src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs b/src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs
new file mode 100644
index 0000000..bca4e29
--- /dev/null
+++ b/src/NuGet.Services.Work/Jobs/RebuildSearchIndexJob.cs
@@ -0,0 +1,80 @@
+using System;
+using System.Collections.Generic;
+using System.Data.SqlClient;
+using System.Diagnostics.Tracing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Blob;
+using NuGet.Indexing;
+using NuGet.Services.Configuration;
+using NuGet.Services.Work.Jobs.Bases;
+using NuGet.Services.Work.Monitoring;
+
+namespace NuGet.Services.Work.Jobs
+{
+ public class RebuildSearchIndexJob : SearchIndexJobHandlerBase
+ {
+ public RebuildSearchIndexJob(ConfigurationHub config) : base(config) { }
+
+ protected internal override async Task Execute()
+ {
+ // This job can take a long time and is run manually. Make the job timeout long
+ await Extend(TimeSpan.FromHours(12));
+
+ // Run the task
+ Log.BeginningIndex(
+ PackageDatabase.DataSource + "/" + PackageDatabase.InitialCatalog,
+ String.IsNullOrEmpty(LocalIndexFolder) ?
+ (StorageAccount.Credentials.AccountName + "/" + StorageContainerName) :
+ LocalIndexFolder);
+ FullBuildTask task = new FullBuildTask()
+ {
+ SqlConnectionString = PackageDatabase.ConnectionString,
+ StorageAccount = StorageAccount,
+ Container = String.IsNullOrEmpty(LocalIndexFolder) ?
+ StorageContainerName :
+ null,
+ Folder = LocalIndexFolder,
+ Log = new EventSourceWriter(Log.IndexingTrace)
+ };
+ task.Execute();
+ Log.FinishedIndex();
+ }
+ }
+
+ [EventSource(Name="Outercurve-NuGet-Jobs-RebuildSearchIndex")]
+ public class RebuildSearchIndexEventSource : EventSource
+ {
+ public static readonly RebuildSearchIndexEventSource Log = new RebuildSearchIndexEventSource();
+ private RebuildSearchIndexEventSource() { }
+
+ [Event(
+ eventId: 1,
+ Level = EventLevel.Informational,
+ Message = "Indexing Trace: {0}")]
+ public void IndexingTrace(string message) { WriteEvent(1, message); }
+
+ [Event(
+ eventId: 2,
+ Level = EventLevel.Informational,
+ Message = "Beginning Index Rebuild from {0} to {1}",
+ Task = Tasks.Indexing,
+ Opcode = EventOpcode.Start)]
+ public void BeginningIndex(string source, string destination) { WriteEvent(2, source, destination); }
+
+ [Event(
+ eventId: 3,
+ Level = EventLevel.Informational,
+ Message = "Finished Index Rebuild",
+ Task = Tasks.Indexing,
+ Opcode = EventOpcode.Stop)]
+ public void FinishedIndex() { WriteEvent(3); }
+
+ public static class Tasks
+ {
+ public const EventTask Indexing = (EventTask)0x1;
+ }
+ }
+}
diff --git a/src/NuGet.Services.Work/Jobs/UpdateSearchIndexJob.cs b/src/NuGet.Services.Work/Jobs/UpdateSearchIndexJob.cs
new file mode 100644
index 0000000..ea26ab7
--- /dev/null
+++ b/src/NuGet.Services.Work/Jobs/UpdateSearchIndexJob.cs
@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Tracing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using NuGet.Indexing;
+using NuGet.Services.Configuration;
+using NuGet.Services.Work.Jobs.Bases;
+using NuGet.Services.Work.Monitoring;
+
+namespace NuGet.Services.Work.Jobs
+{
+ public class UpdateSearchIndexJob : SearchIndexJobHandlerBase
+ {
+ public UpdateSearchIndexJob(ConfigurationHub config) : base(config) { }
+
+ protected internal override Task Execute()
+ {
+ // Run the task
+ UpdateIndexTask task = new UpdateIndexTask()
+ {
+ SqlConnectionString = PackageDatabase.ConnectionString,
+ StorageAccount = StorageAccount,
+ Container = String.IsNullOrEmpty(LocalIndexFolder) ?
+ (StorageContainerName ?? "ng-search") :
+ null,
+ Folder = LocalIndexFolder,
+ Log = new EventSourceWriter(Log.IndexingTrace),
+ };
+ task.Execute();
+
+ return Task.FromResult(0);
+ }
+ }
+
+ [EventSource(Name="Outercurve-NuGet-Jobs-UpdateSearchIndex")]
+ public class UpdateSearchIndexEventSource : EventSource
+ {
+ public static readonly UpdateSearchIndexEventSource Log = new UpdateSearchIndexEventSource();
+ private UpdateSearchIndexEventSource() { }
+
+ [Event(
+ eventId: 1,
+ Level = EventLevel.Informational,
+ Message = "Indexing Trace: {0}")]
+ public void IndexingTrace(string message) { WriteEvent(1, message); }
+ }
+}
diff --git a/src/NuGet.Services.Work/Monitoring/EventSourceWriter.cs b/src/NuGet.Services.Work/Monitoring/EventSourceWriter.cs
new file mode 100644
index 0000000..43651ac
--- /dev/null
+++ b/src/NuGet.Services.Work/Monitoring/EventSourceWriter.cs
@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace NuGet.Services.Work.Monitoring
+{
+ ///
+ /// Writes lines to a specified EventSource method
+ ///
+ public class EventSourceWriter : TextWriter
+ {
+ private Action _receiver;
+ private StringBuilder _buffer = new StringBuilder();
+
+ public override Encoding Encoding
+ {
+ get { return Encoding.Default; }
+ }
+
+ public EventSourceWriter(Action receiver)
+ {
+ _receiver = receiver;
+ }
+
+ public override void Write(char value)
+ {
+ _buffer.Append(value);
+ CheckFlushLine();
+ }
+
+ public override void Flush()
+ {
+ _receiver(_buffer.ToString().Trim());
+ _buffer.Clear();
+ }
+
+ private void CheckFlushLine()
+ {
+ if (_buffer.ToString().EndsWith(NewLine))
+ {
+ // Flush the buffer
+ Flush();
+ }
+ }
+ }
+}
diff --git a/src/NuGet.Services.Work/NuGet.Services.Work.csproj b/src/NuGet.Services.Work/NuGet.Services.Work.csproj
index 89ef179..15c7e37 100644
--- a/src/NuGet.Services.Work/NuGet.Services.Work.csproj
+++ b/src/NuGet.Services.Work/NuGet.Services.Work.csproj
@@ -44,6 +44,48 @@
..\..\packages\Dapper.1.13\lib\net45\Dapper.dll
+
+ ..\..\packages\EntityFramework.5.0.0\lib\net45\EntityFramework.dll
+
+
+ ..\..\packages\SharpZipLib.0.86.0\lib\20\ICSharpCode.SharpZipLib.dll
+
+
+ ..\..\packages\Lucene.Net.3.0.3\lib\NET40\Lucene.Net.dll
+
+
+ ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Analyzers.dll
+
+
+ ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Core.dll
+
+
+ ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.FastVectorHighlighter.dll
+
+
+ ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Highlighter.dll
+
+
+ ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Memory.dll
+
+
+ ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Queries.dll
+
+
+ ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Regex.dll
+
+
+ ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.SimpleFacetedSearch.dll
+
+
+ ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.Snowball.dll
+
+
+ ..\..\packages\Lucene.Net.Contrib.3.0.3\lib\net40\Lucene.Net.Contrib.SpellChecker.dll
+
+
+ ..\..\packages\Lucene.Net.Store.Azure.2.0.4937.26631\lib\net40\Lucene.Net.Store.Azure.dll
+
..\..\packages\Microsoft.Data.Edm.5.6.0\lib\net40\Microsoft.Data.Edm.dll
@@ -95,11 +137,20 @@
False
..\..\packages\Nuget.Core.2.8.0\lib\net40-Client\NuGet.Core.dll
-
- ..\..\packages\NuGet.Services.Platform.3.0.1-rel\lib\net45\NuGet.Services.Platform.dll
+
+ False
+ ..\..\packages\NuGet.Indexing.3.0.2-alpha-9\lib\net45\NuGet.Indexing.dll
+
+
+ False
+ ..\..\packages\NuGet.Services.Platform.3.0.1-rel-3\lib\net45\NuGet.Services.Platform.dll
-
- ..\..\packages\NuGet.Services.Platform.Client.3.0.1-rel\lib\portable-net45+wp80+win\NuGet.Services.Platform.Client.dll
+
+ False
+ ..\..\packages\NuGet.Services.Platform.Client.3.0.1-rel-3\lib\portable-net45+wp80+win\NuGet.Services.Platform.Client.dll
+
+
+ ..\..\packages\NuGetGallery.Core.2.0.0-alpha-118\lib\net45\NuGetGallery.Core.dll
..\..\packages\Owin.1.0\lib\net40\Owin.dll
@@ -107,6 +158,7 @@
+
@@ -130,9 +182,6 @@
..\..\packages\Rx-Linq.2.2.2\lib\net45\System.Reactive.Linq.dll
-
- ..\..\packages\Rx-PlatformServices.2.2.2\lib\net45\System.Reactive.PlatformServices.dll
-
@@ -190,6 +239,7 @@
+
@@ -200,9 +250,9 @@
-
+
@@ -213,7 +263,9 @@
+
+
diff --git a/src/NuGet.Services.Work/WorkService.cs b/src/NuGet.Services.Work/WorkService.cs
index 6c8fca2..ab1df46 100644
--- a/src/NuGet.Services.Work/WorkService.cs
+++ b/src/NuGet.Services.Work/WorkService.cs
@@ -210,22 +210,24 @@ public IObservable RunJob(string job, string payload)
QueuedAt = DateTime.UtcNow,
NextVisibleAt = DateTime.UtcNow + TimeSpan.FromMinutes(5)
});
- var buffer = new ReplaySubject();
- var capture = new InvocationLogCapture(invocation);
- capture.Subscribe(buffer.OnNext, buffer.OnError);
- runner.Dispatch(invocation, capture, CancellationToken.None).ContinueWith(t =>
+ return Observable.Create(observer =>
{
- if (t.IsFaulted)
+ var capture = new InvocationLogCapture(invocation);
+ capture.Subscribe(e => observer.OnNext(e), ex => observer.OnError(ex));
+ runner.Dispatch(invocation, capture, CancellationToken.None).ContinueWith(t =>
{
- buffer.OnError(t.Exception);
- }
- else
- {
- buffer.OnCompleted();
- }
- return t;
+ if (t.IsFaulted)
+ {
+ observer.OnError(t.Exception);
+ }
+ else
+ {
+ observer.OnCompleted();
+ }
+ return t;
+ });
+ return () => { }; // No action on unsubscribe
});
- return buffer;
}
protected override void ConfigureAttributeRouting(DefaultInlineConstraintResolver resolver)
diff --git a/src/NuGet.Services.Work/app.config b/src/NuGet.Services.Work/app.config
index b00b86a..fe616c5 100644
--- a/src/NuGet.Services.Work/app.config
+++ b/src/NuGet.Services.Work/app.config
@@ -1,65 +1,76 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
\ No newline at end of file
diff --git a/src/NuGet.Services.Work/packages.config b/src/NuGet.Services.Work/packages.config
index 69b3123..c83f187 100644
--- a/src/NuGet.Services.Work/packages.config
+++ b/src/NuGet.Services.Work/packages.config
@@ -5,6 +5,10 @@
+
+
+
+
@@ -23,14 +27,17 @@
-
-
+
+
+
+
+
-
\ No newline at end of file
+