Merge pull request #5 from NuGet/anurse/1922-relevancejob

Fix NuGet/NuGetGallery#1922 by creating a job to generate search relevance rankings
NuGet · Mar 4, 2014 · 987dafe · 987dafe
2 parents eb61427 + 8595f74
commit 987dafe
Show file tree

Hide file tree

Showing 20 changed files with 764 additions and 15 deletions.
diff --git a/NuGet.Services.Work.sln b/NuGet.Services.Work.sln
@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
 # Visual Studio 2013
-VisualStudioVersion = 12.0.21005.1
+VisualStudioVersion = 12.0.30110.0
 MinimumVisualStudioVersion = 10.0.40219.1
 Project("{00D1A9C2-B5F0-4AF3-8072-F6C62B433612}") = "NuGet.Services.Work.Database", "src\NuGet.Services.Work.Database\NuGet.Services.Work.Database.sqlproj", "{A8B0845E-CF36-4ADB-B641-CA8542202878}"
 EndProject
@@ -18,6 +18,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		nuget.config = nuget.config
 	EndProjectSection
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "JobHost", "src\JobHost\JobHost.csproj", "{6A98537C-EE21-46E6-AB13-3E04637A72CF}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -94,6 +96,14 @@ Global
 		{19324B97-1C22-49E2-8C1A-E4EAB682A974}.Release|x64.Build.0 = Release|Any CPU
 		{19324B97-1C22-49E2-8C1A-E4EAB682A974}.Release|x86.ActiveCfg = Release|Any CPU
 		{19324B97-1C22-49E2-8C1A-E4EAB682A974}.Release|x86.Build.0 = Release|Any CPU
+		{6A98537C-EE21-46E6-AB13-3E04637A72CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{6A98537C-EE21-46E6-AB13-3E04637A72CF}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{6A98537C-EE21-46E6-AB13-3E04637A72CF}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{6A98537C-EE21-46E6-AB13-3E04637A72CF}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{6A98537C-EE21-46E6-AB13-3E04637A72CF}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{6A98537C-EE21-46E6-AB13-3E04637A72CF}.Release|Any CPU.Build.0 = Release|Any CPU
+		{6A98537C-EE21-46E6-AB13-3E04637A72CF}.Release|x64.ActiveCfg = Release|Any CPU
+		{6A98537C-EE21-46E6-AB13-3E04637A72CF}.Release|x86.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

diff --git a/jobhost.cmd b/jobhost.cmd
@@ -0,0 +1 @@
+@%~dp0src\JobHost\bin\Debug\JobHost.exe %*
diff --git a/src/JobHost/App.config b/src/JobHost/App.config
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+    <startup> 
+        <supportedRuntime version="v4.0" sku=".NETFramework,Version=v4.5" />
+    </startup>
+  <runtime>
+    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+      <dependentAssembly>
+        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-6.0.0.0" newVersion="6.0.0.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="System.Web.Http" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-5.1.0.0" newVersion="5.1.0.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="System.Net.Http.Formatting" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-5.1.0.0" newVersion="5.1.0.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Microsoft.WindowsAzure.Storage" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-3.0.3.0" newVersion="3.0.3.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="Microsoft.Owin" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-2.1.0.0" newVersion="2.1.0.0" />
+      </dependentAssembly>
+      <dependentAssembly>
+        <assemblyIdentity name="System.Net.Http.Primitives" publicKeyToken="b03f5f7f11d50a3a" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.2.18.0" newVersion="4.2.18.0" />
+      </dependentAssembly>
+    </assemblyBinding>
+  </runtime>
+</configuration>
diff --git a/src/JobHost/Arguments.cs b/src/JobHost/Arguments.cs
@@ -0,0 +1,34 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using PowerArgs;
+
+namespace NuGet.Services.Work.JobHost
+{
+    public class Arguments
+    {
+        [ArgRequired()]
+        [ArgPosition(0)]
+        [ArgShortcut("j")]
+        [ArgDescription("The job to invoke")]
+        public string Job { get; set; }
+
+        [ArgShortcut("p")]
+        [ArgDescription("The JSON dictionary payload to provide to the job")]
+        public string Payload { get; set; }
+
+        [ArgShortcut("ep")]
+        [ArgDescription("A base64-encoded UTF8 payload string to use. Designed for command-line piping")]
+        public string EncodedPayload { get; set; }
+
+        [ArgShortcut("c")]
+        [ArgDescription("The JSON dictionary configuration to provide to the job")]
+        public string Configuration { get; set; }
+
+        [ArgShortcut("ec")]
+        [ArgDescription("A base64-encoded UTF8 configuration string to use. Designed for command-line piping")]
+        public string EncodedConfiguration { get; set; }
+    }
+}
diff --git a/src/JobHost/JobHost.csproj b/src/JobHost/JobHost.csproj
@@ -0,0 +1,165 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <Import Project="$([MSBuild]::GetDirectoryNameOfFileAbove($(MSBuildThisFileDirectory), 'Repository.props'))\Repository.props" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{6A98537C-EE21-46E6-AB13-3E04637A72CF}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>NuGet.Services.Work.JobHost</RootNamespace>
+    <AssemblyName>JobHost</AssemblyName>
+    <TargetFrameworkVersion>v4.5</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="Autofac">
+      <HintPath>..\..\packages\Autofac.3.3.0\lib\net40\Autofac.dll</HintPath>
+    </Reference>
+    <Reference Include="Autofac.Integration.WebApi">
+      <HintPath>..\..\packages\Autofac.WebApi5.3.0.0-rc1\lib\net45\Autofac.Integration.WebApi.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.Data.Edm, Version=5.6.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\Microsoft.Data.Edm.5.6.0\lib\net40\Microsoft.Data.Edm.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.Data.OData, Version=5.6.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\Microsoft.Data.OData.5.6.0\lib\net40\Microsoft.Data.OData.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.Data.Services.Client, Version=5.6.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\Microsoft.Data.Services.Client.5.6.0\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.Owin">
+      <HintPath>..\..\packages\Microsoft.Owin.2.1.0\lib\net45\Microsoft.Owin.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.Owin.Hosting">
+      <HintPath>..\..\packages\Microsoft.Owin.Hosting.2.1.0\lib\net45\Microsoft.Owin.Hosting.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.Owin.Security">
+      <HintPath>..\..\packages\Microsoft.Owin.Security.2.1.0\lib\net45\Microsoft.Owin.Security.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.Practices.EnterpriseLibrary.SemanticLogging">
+      <HintPath>..\..\packages\EnterpriseLibrary.SemanticLogging.1.0.1304.0\lib\NET45\Microsoft.Practices.EnterpriseLibrary.SemanticLogging.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.Practices.EnterpriseLibrary.SemanticLogging.WindowsAzure">
+      <HintPath>..\..\packages\EnterpriseLibrary.SemanticLogging.WindowsAzure.1.0.1304.1\lib\NET45\Microsoft.Practices.EnterpriseLibrary.SemanticLogging.WindowsAzure.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.Web.XmlTransform">
+      <HintPath>..\..\packages\Microsoft.Web.Xdt.1.0.0\lib\net40\Microsoft.Web.XmlTransform.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.WindowsAzure.Configuration, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\Microsoft.WindowsAzure.ConfigurationManager.2.0.3\lib\net40\Microsoft.WindowsAzure.Configuration.dll</HintPath>
+    </Reference>
+    <Reference Include="Microsoft.WindowsAzure.Storage, Version=3.0.3.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\packages\WindowsAzure.Storage.3.0.3.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
+    </Reference>
+    <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\packages\Newtonsoft.Json.6.0.1\lib\net45\Newtonsoft.Json.dll</HintPath>
+    </Reference>
+    <Reference Include="NuGet.Core, Version=2.8.50126.400, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\packages\NuGet.Core.2.8.0\lib\net40-Client\NuGet.Core.dll</HintPath>
+    </Reference>
+    <Reference Include="NuGet.Services.Platform">
+      <HintPath>..\..\packages\NuGet.Services.Platform.3.0.1-rel\lib\net45\NuGet.Services.Platform.dll</HintPath>
+    </Reference>
+    <Reference Include="Owin">
+      <HintPath>..\..\packages\Owin.1.0\lib\net40\Owin.dll</HintPath>
+    </Reference>
+    <Reference Include="PowerArgs">
+      <HintPath>..\..\packages\PowerArgs.2.0.4.0-preview\lib\net40\PowerArgs.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Data.Services.Client" />
+    <Reference Include="System.Net.Http" />
+    <Reference Include="System.Net.Http.Extensions">
+      <HintPath>..\..\packages\Microsoft.Net.Http.2.2.18\lib\net45\System.Net.Http.Extensions.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Net.Http.Formatting, Version=5.1.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\packages\Microsoft.AspNet.WebApi.Client.5.1.1\lib\net45\System.Net.Http.Formatting.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Net.Http.Primitives">
+      <HintPath>..\..\packages\Microsoft.Net.Http.2.2.18\lib\net45\System.Net.Http.Primitives.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Net.Http.WebRequest" />
+    <Reference Include="System.Reactive.Core">
+      <HintPath>..\..\packages\Rx-Core.2.2.2\lib\net45\System.Reactive.Core.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Reactive.Interfaces">
+      <HintPath>..\..\packages\Rx-Interfaces.2.2.2\lib\net45\System.Reactive.Interfaces.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Reactive.Linq">
+      <HintPath>..\..\packages\Rx-Linq.2.2.2\lib\net45\System.Reactive.Linq.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Reactive.PlatformServices">
+      <HintPath>..\..\packages\Rx-PlatformServices.2.2.2\lib\net45\System.Reactive.PlatformServices.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Spatial, Version=5.6.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <HintPath>..\..\packages\System.Spatial.5.6.0\lib\net40\System.Spatial.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Web.Http, Version=5.1.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\..\packages\Microsoft.AspNet.WebApi.Core.5.1.1\lib\net45\System.Web.Http.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Web.Http.Owin">
+      <HintPath>..\..\packages\Microsoft.AspNet.WebApi.Owin.5.1.1\lib\net45\System.Web.Http.Owin.dll</HintPath>
+    </Reference>
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="$(CommonSourceDirectory)\CommonAssemblyInfo.cs">
+      <Link>Properties\CommonAssemblyInfo.cs</Link>
+    </Compile>
+    <Compile Include="Arguments.cs" />
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <None Include="App.config" />
+    <None Include="packages.config" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\NuGet.Services.Work.Client\NuGet.Services.Work.Client.csproj">
+      <Project>{19324b97-1c22-49e2-8c1a-e4eab682a974}</Project>
+      <Name>NuGet.Services.Work.Client</Name>
+    </ProjectReference>
+    <ProjectReference Include="..\NuGet.Services.Work\NuGet.Services.Work.csproj">
+      <Project>{7f92c174-5310-4090-a6a3-3cc7c67c19d6}</Project>
+      <Name>NuGet.Services.Work</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(BuildToolsDirectory)\NuGet.targets" />
+  <Import Project="..\..\packages\Microsoft.Bcl.Build.1.0.13\tools\Microsoft.Bcl.Build.targets" Condition="Exists('..\..\packages\Microsoft.Bcl.Build.1.0.13\tools\Microsoft.Bcl.Build.targets')" />
+  <Target Name="EnsureBclBuildImported" BeforeTargets="BeforeBuild" Condition="'$(BclBuildImported)' == ''">
+    <Error Condition="!Exists('..\..\packages\Microsoft.Bcl.Build.1.0.13\tools\Microsoft.Bcl.Build.targets')" Text="This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=317567." HelpKeyword="BCLBUILD2001" />
+    <Error Condition="Exists('..\..\packages\Microsoft.Bcl.Build.1.0.13\tools\Microsoft.Bcl.Build.targets')" Text="The build restored NuGet packages. Build the project again to include these packages in the build. For more information, see http://go.microsoft.com/fwlink/?LinkID=317568." HelpKeyword="BCLBUILD2002" />
+  </Target>
+</Project>
diff --git a/src/JobHost/Program.cs b/src/JobHost/Program.cs
@@ -0,0 +1,109 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Tracing;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Microsoft.Practices.EnterpriseLibrary.SemanticLogging;
+using PowerArgs;
+
+namespace NuGet.Services.Work.JobHost
+{
+    class Program
+    {
+        static void Main(string[] args)
+        {
+            Arguments parsed;
+            try
+            {
+                parsed = Args.Parse<Arguments>(args);
+                AsyncMain(parsed).Wait();
+            }
+            catch (ArgException ex)
+            {
+                Console.WriteLine(ex.Message);
+                WriteUsage();
+            }
+        }
+
+        private static void WriteUsage()
+        {
+            Console.WriteLine(ArgUsage.GetUsage<Arguments>());
+
+            Console.WriteLine();
+            Console.WriteLine("Available jobs: ");
+            var jobs = LocalWorkService.GetAllAvailableJobs();
+            var maxName = jobs.Max(d => d.Name.Length);
+            foreach (var job in jobs)
+            {
+                Console.WriteLine("* {0} {1}", job.Name.PadRight(maxName), job.Description);
+            }
+        }
+
+        private static async Task AsyncMain(Arguments args)
+        {
+            if (!String.IsNullOrEmpty(args.EncodedPayload))
+            {
+                args.Payload = Encoding.UTF8.GetString(Convert.FromBase64String(args.EncodedPayload));
+            }
+
+            if (!String.IsNullOrEmpty(args.EncodedConfiguration))
+            {
+                args.Configuration = Encoding.UTF8.GetString(Convert.FromBase64String(args.EncodedConfiguration));
+            }
+
+            var configuration = InvocationPayloadSerializer.Deserialize(args.Configuration);
+
+            var service = await LocalWorkService.Create(configuration);
+
+            var tcs = new TaskCompletionSource<object>();
+
+            string message = String.Format("Invoking job: {0}.", args.Job);
+            Console.WriteLine(message);
+            Console.WriteLine(new String('-', message.Length));
+
+            try
+            {
+                var observable = service.RunJob(args.Job, args.Payload);
+                observable
+                    .Subscribe(
+                        evt => RenderEvent(evt),
+                        ex => tcs.SetException(ex),
+                        () => tcs.SetResult(null));
+                await tcs.Task;
+            }
+            catch (AggregateException aex)
+            {
+                Console.Error.WriteLine(aex.InnerException.ToString());
+            }
+            catch (Exception ex)
+            {
+                Console.Error.WriteLine(ex.ToString());
+            }
+
+            message = String.Format("Completed invocation of job {0}.", args.Job);
+            Console.WriteLine(new String('-', message.Length));
+            Console.WriteLine(message);
+        }
+
+        private static readonly Dictionary<EventLevel, string> _levelMap = new Dictionary<EventLevel,string>() {
+            { EventLevel.Critical, "fatal" },
+            { EventLevel.Error, "error" },
+            { EventLevel.Informational, "info" },
+            { EventLevel.Verbose, "trace" },
+            { EventLevel.Warning, "warn" }
+        };
+        private static readonly int _maxLen = _levelMap.Values.Max(s => s.Length);
+
+        private static void RenderEvent(EventEntry evt)
+        {
+            string message = evt.FormattedMessage;
+            string levelStr;
+            if(!_levelMap.TryGetValue(evt.Schema.Level, out levelStr)) {
+                levelStr = "????";
+            }
+            levelStr = levelStr.PadRight(_maxLen);
+            Console.WriteLine("[{0}] {1}", levelStr, message);
+        }
+    }
+}
diff --git a/src/JobHost/Properties/AssemblyInfo.cs b/src/JobHost/Properties/AssemblyInfo.cs
@@ -0,0 +1,5 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+
+[assembly: AssemblyTitle("JobHost")]
+[assembly: AssemblyDescription("Runs a single worker job, in process")]