Merge pull request #1969 from NuGet/anurse/1964-hijackodata

Fix #1964 by hijacking the OData request

src/NuGetGallery/DataServices/SearchHijacker.cs

@@ -0,0 +1,266 @@
+using System;
+using System.Collections.Generic;
+using System.Data.Objects;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Reflection;
+using System.Text;
+using NuGet.Services.Search.Models;
+
+namespace NuGetGallery.DataServices
+{
+    public class SearchHijacker : IQueryable<Package>
+    {
+        private SearchHijackerProvider _provider;
+        public IQueryable<Package> Inner { get; private set; }
+
+        public SearchHijacker(IQueryable<Package> inner, ISearchService service, string feed, string siteRoot, bool includeLicenseReport)
+        {
+            Inner = inner;
+            _provider = new SearchHijackerProvider(inner.Provider, service as IRawSearchService, feed, siteRoot, includeLicenseReport);
+        }
+
+        public IEnumerator<Package> GetEnumerator()
+        {
+            return Inner.GetEnumerator();
+        }
+
+        System.Collections.IEnumerator System.Collections.IEnumerable.GetEnumerator()
+        {
+            return Inner.GetEnumerator();
+        }
+
+        public Type ElementType
+        {
+            get { return Inner.ElementType; }
+        }
+
+        public Expression Expression
+        {
+            get { return Inner.Expression; }
+        }
+
+        public IQueryProvider Provider
+        {
+            get { return _provider; }
+        }
+
+        private class SearchHijackerProvider : IQueryProvider
+        {
+            private static MemberInfo _normalizedVersionMember = typeof(V2FeedPackage).GetProperty("NormalizedVersion");
+            private static MemberInfo _versionMember = typeof(V2FeedPackage).GetProperty("Version");
+            private static MemberInfo _idMember = typeof(V2FeedPackage).GetProperty("Id");
+
+            public IQueryProvider Inner { get; private set; }
+            public IRawSearchService SearchService { get; private set; }
+            public string Feed { get; private set; }
+            public string SiteRoot { get; private set; }
+            public bool IncludeLicenseReport { get; private set; }
+
+            public SearchHijackerProvider(IQueryProvider inner, IRawSearchService service, string feed, string siteRoot, bool includeLicenseReport)
+            {
+                Inner = inner;
+                SearchService = service;
+                Feed = feed;
+                SiteRoot = siteRoot;
+                IncludeLicenseReport = includeLicenseReport;
+            }
+
+            public IQueryable<TElement> CreateQuery<TElement>(Expression expression)
+            {
+                return Inner.CreateQuery<TElement>(expression);
+            }
+
+            public IQueryable CreateQuery(Expression expression)
+            {
+                IQueryable result;
+                if (!TryHijack(expression, out result))
+                {
+                    return Inner.CreateQuery(expression);
+                }
+                return result;
+            }
+
+            public TResult Execute<TResult>(Expression expression)
+            {
+                return Inner.Execute<TResult>(expression);
+            }
+
+            public object Execute(Expression expression)
+            {
+                return Inner.Execute(expression);
+            }
+
+            private bool TryHijack(Expression expression, out IQueryable result)
+            {
+                result = null;
+
+                if (SearchService == null)
+                {
+                    return false;
+                }
+
+                // Unravel the comparisons into a list we can reason about
+                IList<Tuple<Target, string>> comparisons = new List<Tuple<Target, string>>();
+                Expression remnant = expression;
+                MethodCallExpression where;
+                while (IsQueryableWhere(where = remnant as MethodCallExpression))
+                {
+                    var comparison = ExtractComparison(where);
+                    if (comparison == null)
+                    {
+                        break;
+                    }
+                    else
+                    {
+                        // We recognize this comparison, record it and keep iterating on the nested expression
+                        comparisons.Add(comparison);
+                        remnant = where.Arguments[0];
+                    }
+                }
+
+                // What's left?
+                if (IsSelectV2FeedPackage(remnant as MethodCallExpression))
+                {
+                    // We can hijack!
+                    result = Hijack(comparisons);
+                    return true;
+                }
+
+                return false;
+            }
+
+            private static bool IsSelectV2FeedPackage(MethodCallExpression expr)
+            {
+                // We expect:
+                //  Queryable.Select(<nested expression>, p => new V2FeedPackage() ...)
+                var isSelect = expr != null &&
+                    expr.Method.DeclaringType == typeof(Queryable) &&
+                    String.Equals(expr.Method.Name, "Select", StringComparison.Ordinal);
+                if (isSelect)
+                {
+                    var arg = Unquote(expr.Arguments[1]);
+                    if (arg.NodeType == ExpressionType.Lambda) // p => new V2FeedPackage ...
+                    {
+                        var lambda = arg as LambdaExpression;
+                        if (lambda.ReturnType == typeof(V2FeedPackage))
+                        {
+                            return true;
+                        }
+                    }
+                }
+                return false;
+            }
+
+            private IQueryable Hijack(IList<Tuple<Target, string>> comparisons)
+            {
+                // Perform the search using the search service and just return the result.
+                return SearchService.RawSearch(new SearchFilter()
+                {
+                    SearchTerm = BuildQuery(comparisons),
+                    IncludePrerelease = true,
+                    IncludeAllVersions = true,
+                    Take = 40,
+                    CuratedFeed = new CuratedFeed() { Name = Feed },
+                    SortOrder = SortOrder.Relevance
+                }).Result.Data.ToV2FeedPackageQuery(SiteRoot, IncludeLicenseReport);
+            }
+
+            private static string BuildQuery(IList<Tuple<Target, string>> comparisons)
+            {
+                StringBuilder query = new StringBuilder();
+                bool first = true;
+                foreach (var comparison in comparisons)
+                {
+                    if (first)
+                    {
+                        first = false;
+                    }
+                    else
+                    {
+                        query.Append(" AND ");
+                    }
+                    query.Append(comparison.Item1.ToString());
+                    query.Append(":\"");
+                    query.Append(comparison.Item2);
+                    query.Append("\"");
+                }
+                return query.ToString();
+            }
+
+            private static Tuple<Target, string> ExtractComparison(MethodCallExpression outerWhere)
+            {
+                // We expect to see an expression that looks like this:
+                //  Queryable.Where(<nested expression>, p => <constant> == p.<property>);
+                var arg = Unquote(outerWhere.Arguments[1]);
+                if (arg.NodeType != ExpressionType.Lambda)
+                {
+                    return null;
+                }
+                var lambda = arg as LambdaExpression; // p => <constant> == p.<property>
+                if (lambda.Body.NodeType != ExpressionType.Equal) 
+                {
+                    return null;
+                }
+                var binExpr = lambda.Body as BinaryExpression; // <constant> == p.<property>
+
+                // Get the two sides, we don't care which side is left and which is right.
+                ConstantExpression constSide = (binExpr.Left as ConstantExpression) ?? (binExpr.Right as ConstantExpression);
+                if (constSide == null || constSide.Type != typeof(string))
+                {
+                    return null;
+                }
+                MemberExpression memberSide = (binExpr.Right as MemberExpression) ?? (binExpr.Left as MemberExpression);
+                if (memberSide == null)
+                {
+                    return null;
+                }
+
+                // Check if it's a known member comparison
+                if (memberSide.Member == _normalizedVersionMember)
+                {
+                    return Tuple.Create(Target.Version, (string)constSide.Value);
+                }
+                else if (memberSide.Member == _versionMember)
+                {
+                    return Tuple.Create(Target.Version, SemanticVersionExtensions.Normalize((string)constSide.Value));
+                }
+                else if (memberSide.Member == _idMember)
+                {
+                    return Tuple.Create(Target.Id, (string)constSide.Value);
+                }
+                return null;
+            }
+
+            private static Expression Unquote(Expression expression)
+            {
+                if (expression.NodeType == ExpressionType.Quote)
+                {
+                    return ((UnaryExpression)expression).Operand;
+                }
+                return expression;
+            }
+
+            private static bool IsQueryableWhere(MethodCallExpression expr)
+            {
+                return expr != null &&
+                    expr.Method.DeclaringType == typeof(Queryable) &&
+                    String.Equals(expr.Method.Name, "Where", StringComparison.Ordinal);
+            }
+
+            private enum Target
+            {
+                Version,
+                Id
+            }
+        }
+    }
+
+    public static class SearchHijackerExtensions
+    {
+        public static IQueryable<Package> UseSearchService(this IQueryable<Package> self, ISearchService searchService, string feed, string siteRoot, bool includeLicenseReport)
+        {
+            return new SearchHijacker(self, searchService, feed, siteRoot, includeLicenseReport);
+        }
+    }
+}

src/NuGetGallery/DataServices/V2Feed.svc.cs

@@ -9,7 +9,7 @@
 using System.Web.Routing;
 using NuGet;
 using NuGetGallery.Configuration;
-using NuGetGallery.Helpers;
+using NuGetGallery.DataServices;
 using QueryInterceptor;
 
 namespace NuGetGallery
@@ -33,7 +33,9 @@ protected override V2FeedContext CreateDataSource()
         {
             return new V2FeedContext
                 {
-                    Packages = PackageRepository.GetAll()
+                    Packages = PackageRepository
+                        .GetAll()
+                        .UseSearchService(SearchService, null, Configuration.GetSiteRoot(UseHttps()), Configuration.Features.FriendlyLicenses)
                         .WithoutVersionSort()
                         .ToV2FeedPackageQuery(Configuration.GetSiteRoot(UseHttps()), Configuration.Features.FriendlyLicenses)
                         .InterceptWith(new NormalizeVersionInterceptor())

src/NuGetGallery/Infrastructure/Lucene/ExternalSearchService.cs

@@ -16,13 +16,13 @@
 
 namespace NuGetGallery.Infrastructure.Lucene
 {
-    public class ExternalSearchService : ISearchService, IIndexingService
+    public class ExternalSearchService : ISearchService, IIndexingService, IRawSearchService
     {
         private SearchClient _client;
         private JObject _diagCache;
 
         public Uri ServiceUri { get; private set; }
-
+        
         protected IDiagnosticsSource Trace { get; private set; }
 
         public string IndexPath
@@ -61,10 +61,24 @@ public ExternalSearchService(IAppConfiguration config, IDiagnosticsService diagn
             _client = new SearchClient(ServiceUri, credentials, new TracingHttpHandler(Trace));
         }
 
-        public async Task<SearchResults> Search(SearchFilter filter)
+        public Task<SearchResults> RawSearch(SearchFilter filter)
+        {
+            return SearchCore(filter, raw: true);
+        }
+
+        public Task<SearchResults> Search(SearchFilter filter)
+        {
+            return SearchCore(filter, raw: false);
+        }
+
+        private async Task<SearchResults> SearchCore(SearchFilter filter, bool raw)
         {
             // Convert the query
-            string query = BuildLuceneQuery(filter.SearchTerm);
+            string query = filter.SearchTerm;
+            if (!raw && !String.IsNullOrEmpty(filter.SearchTerm))
+            {
+                query = BuildLuceneQuery(filter.SearchTerm);
+            }
 
             // Query!
             var result = await _client.Search(
@@ -78,7 +92,7 @@ public async Task<SearchResults> Search(SearchFilter filter)
                 isLuceneQuery: true,
                 countOnly: filter.CountOnly,
                 explain: false,
-                getAllVersions: false);
+                getAllVersions: filter.IncludeAllVersions);
 
             result.HttpResponse.EnsureSuccessStatusCode();
             var content = await result.ReadContent();

src/NuGetGallery/NuGetGallery.csproj

@@ -429,6 +429,7 @@
     <Compile Include="Controllers\PagesController.cs" />
     <Compile Include="DataServices\NormalizeVersionInterceptor.cs" />
     <Compile Include="DataServices\RewriteBaseUrlMessageInspector.cs" />
+    <Compile Include="DataServices\SearchHijacker.cs" />
     <Compile Include="Diagnostics\AuthenticationGlimpseTab.cs" />
     <Compile Include="Diagnostics\ConcurrentInMemoryPersistenceStore.cs" />
     <Compile Include="Diagnostics\DiagnosticsService.cs" />

src/NuGetGallery/Services/ISearchService.cs

@@ -13,6 +13,16 @@ public interface ISearchService
         Task<SearchResults> Search(SearchFilter filter);
     }
 
+    public interface IRawSearchService
+    {
+        /// <summary>
+        /// Executes a raw lucene query against the search index
+        /// </summary>
+        /// <param name="filter">The query to execute, with the search term interpreted as a raw lucene query</param>
+        /// <returns>The results of the query</returns>
+        Task<SearchResults> RawSearch(SearchFilter filter);
+    }
+
     public class SearchResults
     {
         public int Hits { get; private set; }

src/NuGetGallery/Services/SearchFilter.cs

@@ -19,5 +19,7 @@ public class SearchFilter
         ///     Determines if only this is a count only query and does not process the source queryable.
         /// </summary>
         public bool CountOnly { get; set; }
+
+        public bool IncludeAllVersions { get; set; }
     }
 }