New Query Pipeline (#14455)

* New Query pipeline

What works:
Non-collection queries
Sync/Async
InMemory/Sqlite/SqlServer
Method/Member translators
Spatial

What does not work:
Cosmos provider - Tests are skipped in normal run so will fix later
Collection queries - Includes/Correlated Subquery is not implemented yet
Subquery lifting - Was dependent on nav rewrite
A lot of small optimizations in expression tree/generated Sql
Null semantics
TPH
Broader list at #14630

Notes:
Old files are not deleted yet. (keeping them for reference for now)
Still target NS2.0
Class names/namespaces are not final.
Doc comments missing

* Disabling tests

Directory.Build.props

@@ -28,11 +28,6 @@
     <PackageProjectUrl>https://docs.microsoft.com/ef/core/</PackageProjectUrl>
   </PropertyGroup>
 
-  <!-- HACK: Work around #15093 -->
-  <PropertyGroup>
-    <NoWarn>$(NoWarn.Replace(';1591', ''))</NoWarn>
-  </PropertyGroup>
-
   <!-- HACK: Work around dotnet/arcade#1373 -->
   <PropertyGroup>
     <NoWarn>$(NoWarn);NU5125</NoWarn>

src/EFCore.Cosmos/Extensions/CosmosServiceCollectionExtensions.cs

@@ -8,6 +8,7 @@
 using Microsoft.EntityFrameworkCore.Cosmos.Metadata.Conventions.Internal;
 using Microsoft.EntityFrameworkCore.Cosmos.Query.ExpressionVisitors.Internal;
 using Microsoft.EntityFrameworkCore.Cosmos.Query.Internal;
+using Microsoft.EntityFrameworkCore.Cosmos.Query.Pipeline;
 using Microsoft.EntityFrameworkCore.Cosmos.Query.Sql;
 using Microsoft.EntityFrameworkCore.Cosmos.Query.Sql.Internal;
 using Microsoft.EntityFrameworkCore.Cosmos.Storage.Internal;
@@ -17,6 +18,7 @@
 using Microsoft.EntityFrameworkCore.Query;
 using Microsoft.EntityFrameworkCore.Query.ExpressionVisitors;
 using Microsoft.EntityFrameworkCore.Query.ExpressionVisitors.Internal;
+using Microsoft.EntityFrameworkCore.Query.Pipeline;
 using Microsoft.EntityFrameworkCore.Storage;
 using Microsoft.EntityFrameworkCore.Utilities;
 
@@ -42,6 +44,12 @@ public static IServiceCollection AddEntityFrameworkCosmos([NotNull] this IServic
                 .TryAdd<IEntityQueryableExpressionVisitorFactory, CosmosEntityQueryableExpressionVisitorFactory>()
                 .TryAdd<IMemberAccessBindingExpressionVisitorFactory, CosmosMemberAccessBindingExpressionVisitorFactory>()
                 .TryAdd<ITypeMappingSource, CosmosTypeMappingSource>()
+
+                // New Query pipeline
+                .TryAdd<IEntityQueryableTranslatorFactory, CosmosEntityQueryableTranslatorFactory>()
+                .TryAdd<IQueryableMethodTranslatingExpressionVisitorFactory, CosmosQueryableMethodTranslatingExpressionVisitorFactory>()
+                .TryAdd<IShapedQueryCompilingExpressionVisitorFactory, CosmosShapedQueryCompilingExpressionVisitorFactory>()
+
                 .TryAddProviderSpecificServices(
                     b => b
                         .TryAddScoped<CosmosClientWrapper, CosmosClientWrapper>()

src/EFCore.Cosmos/Query/Internal/EntityShaper.cs

@@ -117,7 +117,7 @@ var materializationContextParameter
             var materializer
                 = _entityMaterializerSource
                     .CreateMaterializeExpression(
-                        firstEntityType, materializationContextParameter);
+                        firstEntityType, "instance", materializationContextParameter);
 
             if (concreteEntityTypes.Count == 1)
             {
@@ -145,7 +145,8 @@ var blockExpressions
                             .CreateReadValueExpression(
                                 Expression.Call(materializationContextParameter, MaterializationContext.GetValueBufferMethod),
                                 discriminatorProperty.ClrType,
-                                indexMap[discriminatorProperty.GetIndex()])),
+                                indexMap[discriminatorProperty.GetIndex()],
+                                discriminatorProperty)),
                     Expression.IfThenElse(
                         Expression.Equal(discriminatorValueVariable, firstDiscriminatorValue),
                         Expression.Return(returnLabelTarget, materializer),
@@ -196,7 +197,7 @@ var discriminatorValue
                 materializer
                     = _entityMaterializerSource
                         .CreateMaterializeExpression(
-                            concreteEntityType, materializationContextParameter);
+                            concreteEntityType, "instance", materializationContextParameter);
 
                 blockExpressions[1]
                     = Expression.IfThenElse(
@@ -230,7 +231,7 @@ private static object Shape(
             {
                 if (trackingQuery)
                 {
-                    var entry = queryContext.StateManager.TryGetEntry(entityInfo.Key, valueBuffer, throwOnNullKey: true);
+                    var entry = queryContext.StateManager.TryGetEntry(entityInfo.Key, new object[] { }, throwOnNullKey: true, out var _);
                     if (entry != null)
                     {
                         return ShapeNestedEntities(

src/EFCore.Cosmos/Query/Pipeline/CosmosEntityQueryableTranslatorFactory.cs

@@ -0,0 +1,53 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Query.Pipeline;
+
+namespace Microsoft.EntityFrameworkCore.Cosmos.Query.Pipeline
+{
+    public class CosmosEntityQueryableTranslatorFactory : EntityQueryableTranslatorFactory
+    {
+        private readonly IModel _model;
+
+        public CosmosEntityQueryableTranslatorFactory(IModel model)
+        {
+            _model = model;
+        }
+
+        public override EntityQueryableTranslator Create(QueryCompilationContext2 queryCompilationContext)
+        {
+            throw new NotImplementedException();
+        }
+    }
+
+    public class CosmosQueryableMethodTranslatingExpressionVisitorFactory : IQueryableMethodTranslatingExpressionVisitorFactory
+    {
+        public QueryableMethodTranslatingExpressionVisitor Create(QueryCompilationContext2 queryCompilationContext)
+        {
+            throw new NotImplementedException();
+        }
+    }
+
+    public class CosmosShapedQueryCompilingExpressionVisitorFactory : IShapedQueryCompilingExpressionVisitorFactory
+    {
+        private readonly IEntityMaterializerSource _entityMaterializerSource;
+
+        public CosmosShapedQueryCompilingExpressionVisitorFactory(IEntityMaterializerSource entityMaterializerSource)
+        {
+            _entityMaterializerSource = entityMaterializerSource;
+        }
+
+        public ShapedQueryCompilingExpressionVisitor Create(QueryCompilationContext2 queryCompilationContext)
+        {
+            throw new NotImplementedException();
+            //return new CosmosShapedQueryCompilingExpressionVisitor(
+            //    _entityMaterializerSource,
+            //    queryCompilationContext.TrackQueryResults,
+            //    queryCompilationContext.Async);
+        }
+    }
+}

src/EFCore.Design/Design/Internal/AppServiceProviderFactory.cs

@@ -2,7 +2,6 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
-using System.Diagnostics;
 using System.Reflection;
 using JetBrains.Annotations;
 using Microsoft.EntityFrameworkCore.Internal;

src/EFCore.InMemory/Extensions/InMemoryServiceCollectionExtensions.cs

@@ -9,11 +9,13 @@
 using Microsoft.EntityFrameworkCore.InMemory.Metadata.Conventions;
 using Microsoft.EntityFrameworkCore.InMemory.Query.ExpressionVisitors.Internal;
 using Microsoft.EntityFrameworkCore.InMemory.Query.Internal;
+using Microsoft.EntityFrameworkCore.InMemory.Query.Pipeline;
 using Microsoft.EntityFrameworkCore.InMemory.Storage.Internal;
 using Microsoft.EntityFrameworkCore.InMemory.ValueGeneration.Internal;
 using Microsoft.EntityFrameworkCore.Metadata.Conventions.Infrastructure;
 using Microsoft.EntityFrameworkCore.Query;
 using Microsoft.EntityFrameworkCore.Query.ExpressionVisitors;
+using Microsoft.EntityFrameworkCore.Query.Pipeline;
 using Microsoft.EntityFrameworkCore.Storage;
 using Microsoft.EntityFrameworkCore.Utilities;
 using Microsoft.EntityFrameworkCore.ValueGeneration;
@@ -70,6 +72,13 @@ public static IServiceCollection AddEntityFrameworkInMemoryDatabase([NotNull] th
                 .TryAdd<IEntityQueryModelVisitorFactory, InMemoryQueryModelVisitorFactory>()
                 .TryAdd<IEntityQueryableExpressionVisitorFactory, InMemoryEntityQueryableExpressionVisitorFactory>()
                 .TryAdd<IProviderConventionSetBuilder, InMemoryConventionSetBuilder>()
+
+                // New Query pipeline
+                .TryAdd<IShapedQueryCompilingExpressionVisitorFactory, InMemoryShapedQueryCompilingExpressionVisitorFactory>()
+                .TryAdd<IQueryableMethodTranslatingExpressionVisitorFactory, InMemoryQueryableMethodTranslatingExpressionVisitorFactory>()
+                .TryAdd<IEntityQueryableTranslatorFactory, InMemoryEntityQueryableTranslatorFactory>()
+
+
                 .TryAdd<ISingletonOptions, IInMemorySingletonOptions>(p => p.GetService<IInMemorySingletonOptions>())
                 .TryAdd<ITypeMappingSource, InMemoryTypeMappingSource>()
                 .TryAddProviderSpecificServices(

src/EFCore.InMemory/Query/Internal/InMemoryMaterializerFactory.cs

@@ -58,7 +58,7 @@ var concreteEntityTypes
                 return Expression.Lambda<Func<IEntityType, MaterializationContext, object>>(
                     _entityMaterializerSource
                         .CreateMaterializeExpression(
-                            concreteEntityTypes[0], materializationContextParameter),
+                            concreteEntityTypes[0], "instance", materializationContextParameter),
                     entityTypeParameter,
                     materializationContextParameter);
             }
@@ -76,7 +76,7 @@ var blockExpressions
                             returnLabelTarget,
                             _entityMaterializerSource
                                 .CreateMaterializeExpression(
-                                    concreteEntityTypes[0], materializationContextParameter))),
+                                    concreteEntityTypes[0], "instance", materializationContextParameter))),
                     Expression.Label(
                         returnLabelTarget,
                         Expression.Default(returnLabelTarget.Type))
@@ -92,7 +92,7 @@ var blockExpressions
                         Expression.Return(
                             returnLabelTarget,
                             _entityMaterializerSource
-                                .CreateMaterializeExpression(concreteEntityType, materializationContextParameter)),
+                                .CreateMaterializeExpression(concreteEntityType, "instance", materializationContextParameter)),
                         blockExpressions[0]);
             }
 

src/EFCore.InMemory/Query/PipeLine/EntityValuesExpression.cs

@@ -0,0 +1,21 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Linq.Expressions;
+using Microsoft.EntityFrameworkCore.Metadata;
+
+namespace Microsoft.EntityFrameworkCore.InMemory.Query.Pipeline
+{
+    public class EntityProjectionExpression : Expression
+    {
+        public EntityProjectionExpression(IEntityType entityType, int startIndex)
+        {
+            EntityType = entityType;
+            StartIndex = startIndex;
+        }
+
+        public IEntityType EntityType { get; }
+        public int StartIndex { get; }
+    }
+
+}

src/EFCore.InMemory/Query/PipeLine/InMemoryEntityQueryableExpressionVisitor2.cs

@@ -0,0 +1,24 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Query.Pipeline;
+
+namespace Microsoft.EntityFrameworkCore.InMemory.Query.Pipeline
+{
+    public class InMemoryEntityQueryableExpressionVisitor2 : EntityQueryableExpressionVisitor2
+    {
+        private readonly IModel _model;
+
+        public InMemoryEntityQueryableExpressionVisitor2(IModel model)
+        {
+            _model = model;
+        }
+
+        protected override ShapedQueryExpression CreateShapedQueryExpression(Type elementType)
+        {
+            return new InMemoryShapedQueryExpression(_model.FindEntityType(elementType));
+        }
+    }
+}

src/EFCore.InMemory/Query/PipeLine/InMemoryEntityQueryableExpressionVisitorFactory2.cs

@@ -0,0 +1,23 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Query.Pipeline;
+
+namespace Microsoft.EntityFrameworkCore.InMemory.Query.Pipeline
+{
+    public class InMemoryEntityQueryableTranslatorFactory : EntityQueryableTranslatorFactory
+    {
+        private readonly IModel _model;
+
+        public InMemoryEntityQueryableTranslatorFactory(IModel model)
+        {
+            _model = model;
+        }
+
+        public override EntityQueryableTranslator Create(QueryCompilationContext2 queryCompilationContext)
+        {
+            return new InMemoryEntityQueryableTranslator(_model);
+        }
+    }
+}

src/EFCore.InMemory/Query/PipeLine/InMemoryEntityQueryableExpressionVisitors.cs

@@ -0,0 +1,25 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System.Collections.Generic;
+using System.Linq.Expressions;
+using Microsoft.EntityFrameworkCore.Metadata;
+using Microsoft.EntityFrameworkCore.Query.Pipeline;
+
+namespace Microsoft.EntityFrameworkCore.InMemory.Query.Pipeline
+{
+    public class InMemoryEntityQueryableTranslator : EntityQueryableTranslator
+    {
+        private readonly IModel _model;
+
+        public InMemoryEntityQueryableTranslator(IModel model)
+        {
+            _model = model;
+        }
+
+        public override Expression Visit(Expression query)
+        {
+            return new InMemoryEntityQueryableExpressionVisitor2(_model).Visit(query);
+        }
+    }
+}

src/EFCore.InMemory/Query/PipeLine/InMemoryLinqOperatorProvider.cs

@@ -0,0 +1,67 @@
+// Copyright (c) .NET Foundation. All rights reserved.
+// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Reflection;
+using Microsoft.EntityFrameworkCore.Utilities;
+
+namespace Microsoft.EntityFrameworkCore.InMemory.Query.Pipeline
+{
+    public static class InMemoryLinqOperatorProvider
+    {
+        private static MethodInfo GetMethod(string name, int parameterCount = 0)
+            => GetMethods(name, parameterCount).Single();
+
+        private static IEnumerable<MethodInfo> GetMethods(string name, int parameterCount = 0)
+            => typeof(Enumerable).GetTypeInfo().GetDeclaredMethods(name)
+                .Where(mi => mi.GetParameters().Length == parameterCount + 1);
+
+        public static MethodInfo Where = GetMethods(nameof(Enumerable.Where), 1)
+            .Single(mi => mi.GetParameters()[1].ParameterType.GetGenericArguments().Length == 2);
+        public static MethodInfo Select = GetMethods(nameof(Enumerable.Select), 1)
+            .Single(mi => mi.GetParameters()[1].ParameterType.GetGenericArguments().Length == 2);
+
+        public static MethodInfo Join = GetMethod(nameof(Enumerable.Join), 4);
+        public static MethodInfo Contains = GetMethod(nameof(Enumerable.Contains), 1);
+
+        public static MethodInfo OrderBy = GetMethod(nameof(Enumerable.OrderBy), 1);
+        public static MethodInfo OrderByDescending = GetMethod(nameof(Enumerable.OrderByDescending), 1);
+        public static MethodInfo ThenBy = GetMethod(nameof(Enumerable.ThenBy), 1);
+        public static MethodInfo ThenByDescending = GetMethod(nameof(Enumerable.ThenByDescending), 1);
+        public static MethodInfo All = GetMethod(nameof(Enumerable.All), 1);
+        public static MethodInfo Any = GetMethod(nameof(Enumerable.Any));
+        public static MethodInfo AnyPredicate = GetMethod(nameof(Enumerable.Any), 1);
+        public static MethodInfo Count = GetMethod(nameof(Enumerable.Count));
+        public static MethodInfo LongCount = GetMethod(nameof(Enumerable.LongCount));
+        public static MethodInfo CountPredicate = GetMethod(nameof(Enumerable.Count), 1);
+        public static MethodInfo LongCountPredicate = GetMethod(nameof(Enumerable.LongCount), 1);
+        public static MethodInfo Distinct = GetMethod(nameof(Enumerable.Distinct));
+        public static MethodInfo Take = GetMethod(nameof(Enumerable.Take), 1);
+        public static MethodInfo Skip = GetMethod(nameof(Enumerable.Skip), 1);
+
+        public static MethodInfo FirstPredicate = GetMethod(nameof(Enumerable.First), 1);
+        public static MethodInfo FirstOrDefaultPredicate = GetMethod(nameof(Enumerable.FirstOrDefault), 1);
+        public static MethodInfo LastPredicate = GetMethod(nameof(Enumerable.Last), 1);
+        public static MethodInfo LastOrDefaultPredicate = GetMethod(nameof(Enumerable.LastOrDefault), 1);
+        public static MethodInfo SinglePredicate = GetMethod(nameof(Enumerable.Single), 1);
+        public static MethodInfo SingleOrDefaultPredicate = GetMethod(nameof(Enumerable.SingleOrDefault), 1);
+
+        public static MethodInfo GetAggregateMethod(string methodName, Type elementType, int parameterCount = 0)
+        {
+            Check.NotEmpty(methodName, nameof(methodName));
+            Check.NotNull(elementType, nameof(elementType));
+
+            var aggregateMethods = GetMethods(methodName, parameterCount).ToList();
+
+            return
+                aggregateMethods
+                    .Single(
+                        mi => mi.GetParameters().Last().ParameterType.GetGenericArguments().Last() == elementType);
+            //?? aggregateMethods.Single(mi => mi.IsGenericMethod)
+            //    .MakeGenericMethod(elementType);
+        }
+    }
+
+}