diff --git a/src/EFCore.Relational/Query/JsonQueryExpression.cs b/src/EFCore.Relational/Query/JsonQueryExpression.cs index 2aade3bd5f4..83ce3639f27 100644 --- a/src/EFCore.Relational/Query/JsonQueryExpression.cs +++ b/src/EFCore.Relational/Query/JsonQueryExpression.cs @@ -117,8 +117,9 @@ public virtual SqlExpression BindProperty(IProperty property) return new JsonScalarExpression( JsonColumn, - property, newPath, + property.ClrType.UnwrapNullableType(), + property.FindRelationalTypeMapping()!, IsNullable || property.IsNullable); } diff --git a/src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs b/src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs index 536749f75ce..900f54abbc1 100644 --- a/src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs +++ b/src/EFCore.Relational/Query/RelationalQueryableMethodTranslatingExpressionVisitor.cs @@ -2786,7 +2786,7 @@ when InferredTypeMappings.TryGetValue((columnExpression.Table, columnExpression. case ValuesExpression valuesExpression: // By default, the ValuesExpression also contains an ordering by a synthetic increasing _ord. If the containing // SelectExpression doesn't project it out or require it (limit/offset), strip that out. - // TODO: Strictly-speaking, this doesn't belong in this visitor which is about applying type mappings + // TODO: Strictly-speaking, stripping the ordering doesn't belong in this visitor which is about applying type mappings return ApplyTypeMappingsOnValuesExpression( valuesExpression, stripOrdering: _currentSelectExpression is { Limit: null, Offset: null } diff --git a/src/EFCore.Relational/Query/SqlExpressionFactory.cs b/src/EFCore.Relational/Query/SqlExpressionFactory.cs index 6dc46ca1058..53d23cbdfdf 100644 --- a/src/EFCore.Relational/Query/SqlExpressionFactory.cs +++ b/src/EFCore.Relational/Query/SqlExpressionFactory.cs @@ -10,6 +10,7 @@ namespace Microsoft.EntityFrameworkCore.Query; /// public class SqlExpressionFactory : ISqlExpressionFactory { + private readonly IRelationalTypeMappingSource _typeMappingSource; private readonly RelationalTypeMapping _boolTypeMapping; /// @@ -19,7 +20,8 @@ public class SqlExpressionFactory : ISqlExpressionFactory public SqlExpressionFactory(SqlExpressionFactoryDependencies dependencies) { Dependencies = dependencies; - _boolTypeMapping = dependencies.TypeMappingSource.FindMapping(typeof(bool), dependencies.Model)!; + _typeMappingSource = dependencies.TypeMappingSource; + _boolTypeMapping = _typeMappingSource.FindMapping(typeof(bool), dependencies.Model)!; } /// @@ -38,7 +40,7 @@ public SqlExpressionFactory(SqlExpressionFactoryDependencies dependencies) && sqlUnaryExpression.Type == typeof(object) ? sqlUnaryExpression.Operand : ApplyTypeMapping( - sqlExpression, Dependencies.TypeMappingSource.FindMapping(sqlExpression.Type, Dependencies.Model)); + sqlExpression, _typeMappingSource.FindMapping(sqlExpression.Type, Dependencies.Model)); /// [return: NotNullIfNotNull("sqlExpression")] @@ -60,6 +62,13 @@ public SqlExpressionFactory(SqlExpressionFactoryDependencies dependencies) ColumnExpression e => e.ApplyTypeMapping(typeMapping), DistinctExpression e => ApplyTypeMappingOnDistinct(e, typeMapping), InExpression e => ApplyTypeMappingOnIn(e), + + // We only do type inference for JSON scalar expression which represent a single array indexing operation; we can infer the + // array's mapping from the element or vice versa, allowing e.g. parameter primitive collections to get inferred when an + // an indexer is used over them and then compared to a column. + // But we can't infer anything for other Path forms of JsonScalarExpression (e.g. a property lookup). + JsonScalarExpression { Path: [{ ArrayIndex: not null }] } e => ApplyTypeMappingOnJsonScalar(e, typeMapping), + LikeExpression e => ApplyTypeMappingOnLike(e), ScalarSubqueryExpression e => e.ApplyTypeMapping(typeMapping), SqlBinaryExpression e => ApplyTypeMappingOnSqlBinary(e, typeMapping), @@ -68,6 +77,7 @@ public SqlExpressionFactory(SqlExpressionFactoryDependencies dependencies) SqlFunctionExpression e => e.ApplyTypeMapping(typeMapping), SqlParameterExpression e => e.ApplyTypeMapping(typeMapping), SqlUnaryExpression e => ApplyTypeMappingOnSqlUnary(e, typeMapping), + _ => sqlExpression }; } @@ -82,7 +92,7 @@ private SqlExpression ApplyTypeMappingOnLike(LikeExpression likeExpression) likeExpression.Match, likeExpression.Pattern) : ExpressionExtensions.InferTypeMapping( likeExpression.Match, likeExpression.Pattern, likeExpression.EscapeChar)) - ?? Dependencies.TypeMappingSource.FindMapping(likeExpression.Match.Type, Dependencies.Model); + ?? _typeMappingSource.FindMapping(likeExpression.Match.Type, Dependencies.Model); return new LikeExpression( ApplyTypeMapping(likeExpression.Match, inferredTypeMapping), @@ -184,9 +194,9 @@ private SqlExpression ApplyTypeMappingOnSqlBinary( { inferredTypeMapping = ExpressionExtensions.InferTypeMapping(left, right) // We avoid object here since the result does not get typeMapping from outside. - ?? (left.Type != typeof(object) - ? Dependencies.TypeMappingSource.FindMapping(left.Type, Dependencies.Model) - : Dependencies.TypeMappingSource.FindMapping(right.Type, Dependencies.Model)); + ?? _typeMappingSource.FindMapping( + left.Type != typeof(object) ? left.Type : right.Type, + Dependencies.Model); resultType = typeof(bool); resultTypeMapping = _boolTypeMapping; break; @@ -236,7 +246,7 @@ private SqlExpression ApplyTypeMappingOnIn(InExpression inExpression) : inExpression.Subquery != null ? ExpressionExtensions.InferTypeMapping(inExpression.Item, inExpression.Subquery.Projection[0].Expression) : inExpression.Item.TypeMapping) - ?? Dependencies.TypeMappingSource.FindMapping(inExpression.Item.Type, Dependencies.Model); + ?? _typeMappingSource.FindMapping(inExpression.Item.Type, Dependencies.Model); var item = ApplyTypeMapping(inExpression.Item, itemTypeMapping); if (inExpression.Values != null) @@ -253,6 +263,41 @@ private SqlExpression ApplyTypeMappingOnIn(InExpression inExpression) : inExpression; } + private SqlExpression ApplyTypeMappingOnJsonScalar( + JsonScalarExpression jsonScalarExpression, + RelationalTypeMapping? typeMapping) + { + if (jsonScalarExpression is not { Json: var array, Path: [{ ArrayIndex: { } index }] }) + { + return jsonScalarExpression; + } + + // The index expression isn't inferred and is always just an int. Apply the default type mapping to it. + var indexWithTypeMapping = ApplyDefaultTypeMapping(index); + var newPath = indexWithTypeMapping == index ? jsonScalarExpression.Path : new[] { new PathSegment(indexWithTypeMapping) }; + + // If a type mapping is being applied from the outside, it applies to the element resulting from the array indexing operation; + // we can infer the array's type mapping from it. Otherwise there's nothing to do but apply the default type mapping to the array. + if (typeMapping is null) + { + return new JsonScalarExpression( + ApplyDefaultTypeMapping(array), + newPath, + jsonScalarExpression.Type, + _typeMappingSource.FindMapping(jsonScalarExpression.Type), + jsonScalarExpression.IsNullable); + } + + // TODO: blocked on #30730: we need to be able to construct a JSON collection type mapping based on the element's. + // For now, hacking to apply the default type mapping instead. + return new JsonScalarExpression( + ApplyDefaultTypeMapping(array), // Hack, until #30730 + newPath, + jsonScalarExpression.Type, + typeMapping, + jsonScalarExpression.IsNullable); + } + /// public virtual SqlBinaryExpression? MakeBinary( ExpressionType operatorType, @@ -350,7 +395,7 @@ public virtual SqlFunctionExpression Coalesce(SqlExpression left, SqlExpression var resultType = right.Type; var inferredTypeMapping = typeMapping ?? ExpressionExtensions.InferTypeMapping(left, right) - ?? Dependencies.TypeMappingSource.FindMapping(resultType, Dependencies.Model); + ?? _typeMappingSource.FindMapping(resultType, Dependencies.Model); var typeMappedArguments = new List { @@ -405,7 +450,7 @@ public virtual CaseExpression Case(SqlExpression? operand, IReadOnlyList wc.Test.Type)) - .Where(t => t != typeof(object)).Select(t => Dependencies.TypeMappingSource.FindMapping(t, Dependencies.Model)) + .Where(t => t != typeof(object)).Select(t => _typeMappingSource.FindMapping(t, Dependencies.Model)) .FirstOrDefault(); var resultTypeMapping = elseResult?.TypeMapping @@ -543,7 +588,7 @@ public virtual ExistsExpression Exists(SelectExpression subquery, bool negated) /// public virtual InExpression In(SqlExpression item, SqlExpression values, bool negated) { - var typeMapping = item.TypeMapping ?? Dependencies.TypeMappingSource.FindMapping(item.Type, Dependencies.Model); + var typeMapping = item.TypeMapping ?? _typeMappingSource.FindMapping(item.Type, Dependencies.Model); item = ApplyTypeMapping(item, typeMapping); values = ApplyTypeMapping(values, typeMapping); diff --git a/src/EFCore.Relational/Query/SqlExpressions/JsonScalarExpression.cs b/src/EFCore.Relational/Query/SqlExpressions/JsonScalarExpression.cs index 920f28efb73..ca809ed4b29 100644 --- a/src/EFCore.Relational/Query/SqlExpressions/JsonScalarExpression.cs +++ b/src/EFCore.Relational/Query/SqlExpressions/JsonScalarExpression.cs @@ -17,28 +17,20 @@ public class JsonScalarExpression : SqlExpression /// /// Creates a new instance of the class. /// - /// A column containg JSON value. - /// A property representing the result of this expression. + /// An expression representing a JSON value. + /// The of the expression. + /// The associated with the expression. /// A list of path segments leading to the scalar from the root of the JSON stored in the column. /// A value indicating whether the expression is nullable. public JsonScalarExpression( - ColumnExpression jsonColumn, - IProperty property, - IReadOnlyList path, - bool nullable) - : this(jsonColumn, path, property.ClrType.UnwrapNullableType(), property.FindRelationalTypeMapping()!, nullable) - { - } - - internal JsonScalarExpression( - ColumnExpression jsonColumn, + SqlExpression json, IReadOnlyList path, Type type, - RelationalTypeMapping typeMapping, + RelationalTypeMapping? typeMapping, bool nullable) : base(type, typeMapping) { - JsonColumn = jsonColumn; + Json = json; Path = path; IsNullable = nullable; } @@ -46,7 +38,7 @@ internal JsonScalarExpression( /// /// The column containing the JSON value. /// - public virtual ColumnExpression JsonColumn { get; } + public virtual SqlExpression Json { get; } /// /// The list of path segments leading to the scalar from the root of the JSON stored in the column. @@ -61,46 +53,99 @@ internal JsonScalarExpression( /// protected override Expression VisitChildren(ExpressionVisitor visitor) { - var jsonColumn = (ColumnExpression)visitor.Visit(JsonColumn); - var jsonColumnMadeNullable = jsonColumn.IsNullable && !JsonColumn.IsNullable; + var newJson = (SqlExpression)visitor.Visit(Json); + + var nullable = IsNullable; + if (newJson is ColumnExpression jsonColumnExpression) + { + nullable |= jsonColumnExpression.IsNullable; + } + + PathSegment[]? newPath = null; + + for (var i = 0; i < Path.Count; i++) + { + var segment = Path[i]; + PathSegment newSegment; + + if (segment.PropertyName is not null) + { + // PropertyName segments are (currently) constants, nothing to visit. + newSegment = segment; + } + else + { + var newArrayIndex = (SqlExpression)visitor.Visit(segment.ArrayIndex)!; + if (newArrayIndex == segment.ArrayIndex) + { + newSegment = segment; + } + else + { + newSegment = new PathSegment(newArrayIndex); + + if (newPath is null) + { + newPath = new PathSegment[Path.Count]; + for (var j = 0; j < i; i++) + { + newPath[j] = Path[j]; + } + } + } + } + + if (newPath is not null) + { + newPath[i] = newSegment; + } + } // TODO Call update: Issue#28887 - return jsonColumn != JsonColumn - ? new JsonScalarExpression( - jsonColumn, - Path, + return newJson == Json && newPath is null + ? this + : new JsonScalarExpression( + newJson, + newPath ?? Path, Type, TypeMapping!, - IsNullable || jsonColumnMadeNullable) - : this; + nullable); } + /// + /// Applies supplied type mapping to this expression. + /// + /// A relational type mapping to apply. + /// A new expression which has supplied type mapping. + public virtual SqlExpression ApplyTypeMapping(RelationalTypeMapping? typeMapping) + => new JsonScalarExpression(Json, Path, Type, typeMapping, IsNullable); + /// /// Creates a new expression that is like this one, but using the supplied children. If all of the children are the same, it will /// return this expression. /// - /// The property of the result. + /// The property of the result. /// This expression if no children changed, or an expression with the updated children. - public virtual JsonScalarExpression Update(ColumnExpression jsonColumn) - => jsonColumn != JsonColumn - ? new JsonScalarExpression(jsonColumn, Path, Type, TypeMapping!, IsNullable) + public virtual JsonScalarExpression Update(SqlExpression json) + => json != Json + ? new JsonScalarExpression(json, Path, Type, TypeMapping!, IsNullable) : this; /// protected override void Print(ExpressionPrinter expressionPrinter) { expressionPrinter.Append("JsonScalarExpression(column: "); - expressionPrinter.Visit(JsonColumn); + expressionPrinter.Visit(Json); expressionPrinter.Append($""", "{string.Join(".", Path.Select(e => e.ToString()))}")"""); } /// public override bool Equals(object? obj) => obj is JsonScalarExpression jsonScalarExpression - && JsonColumn.Equals(jsonScalarExpression.JsonColumn) + && Json.Equals(jsonScalarExpression.Json) && Path.SequenceEqual(jsonScalarExpression.Path); /// public override int GetHashCode() - => HashCode.Combine(base.GetHashCode(), JsonColumn, Path); + => HashCode.Combine(base.GetHashCode(), Json, Path); } diff --git a/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs b/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs index 81764432028..d0815c0bb99 100644 --- a/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs +++ b/src/EFCore.Relational/Query/SqlExpressions/SelectExpression.cs @@ -1739,7 +1739,7 @@ static IReadOnlyList GetMappedKeyProperties(IKey key) static bool JsonEntityContainedIn(JsonScalarExpression sourceExpression, JsonQueryExpression targetExpression) { - if (sourceExpression.JsonColumn != targetExpression.JsonColumn) + if (sourceExpression.Json != targetExpression.JsonColumn) { return false; } diff --git a/src/EFCore.SqlServer/Query/Internal/SqlServerQuerySqlGenerator.cs b/src/EFCore.SqlServer/Query/Internal/SqlServerQuerySqlGenerator.cs index 0ac94604470..94a0ef89df7 100644 --- a/src/EFCore.SqlServer/Query/Internal/SqlServerQuerySqlGenerator.cs +++ b/src/EFCore.SqlServer/Query/Internal/SqlServerQuerySqlGenerator.cs @@ -396,7 +396,7 @@ protected override Expression VisitJsonScalar(JsonScalarExpression jsonScalarExp var path = jsonScalarExpression.Path; if (path.Count == 0) { - Visit(jsonScalarExpression.JsonColumn); + Visit(jsonScalarExpression.Json); return jsonScalarExpression; } @@ -412,7 +412,7 @@ protected override Expression VisitJsonScalar(JsonScalarExpression jsonScalarExp Sql.Append(jsonScalarExpression.TypeMapping is StringTypeMapping ? "JSON_VALUE(" : "CAST(JSON_VALUE("); } - Visit(jsonScalarExpression.JsonColumn); + Visit(jsonScalarExpression.Json); Sql.Append(", '$"); foreach (var pathSegment in jsonScalarExpression.Path) diff --git a/src/EFCore.SqlServer/Query/Internal/SqlServerQueryableMethodTranslatingExpressionVisitor.cs b/src/EFCore.SqlServer/Query/Internal/SqlServerQueryableMethodTranslatingExpressionVisitor.cs index 604c95e619a..f511809fb63 100644 --- a/src/EFCore.SqlServer/Query/Internal/SqlServerQueryableMethodTranslatingExpressionVisitor.cs +++ b/src/EFCore.SqlServer/Query/Internal/SqlServerQueryableMethodTranslatingExpressionVisitor.cs @@ -133,7 +133,9 @@ protected override ShapedQueryExpression TranslateCollection( var openJsonExpression = new TableValuedFunctionExpression(tableAlias, "OpenJson", new[] { sqlExpression }); // TODO: When we have metadata to determine if the element is nullable, pass that here to SelectExpression - var selectExpression = new SelectExpression(openJsonExpression, columnName: "value", columnType: elementClrType, columnTypeMapping: elementTypeMapping, isColumnNullable: null); + var selectExpression = new SelectExpression( + openJsonExpression, columnName: "value", columnType: elementClrType, columnTypeMapping: elementTypeMapping, + isColumnNullable: null); if (elementTypeMapping is { StoreType: not "nvarchar(max)" }) { @@ -191,6 +193,94 @@ protected override ShapedQueryExpression TranslateCollection( return new ShapedQueryExpression(selectExpression, shaperExpression); } + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + protected override ShapedQueryExpression? TranslateElementAtOrDefault( + ShapedQueryExpression source, + Expression index, + bool returnDefault) + { + if (!returnDefault + && source.QueryExpression is SelectExpression + { + Tables: + [ + TableValuedFunctionExpression + { + Name: "OpenJson", Schema: null, IsBuiltIn: true, Arguments: [var jsonArrayColumn] + } openJsonExpression + ], + GroupBy: [], + Having: null, + IsDistinct: false, + Orderings: [ + { + Expression: SqlUnaryExpression + { + OperatorType: ExpressionType.Convert, + Operand: ColumnExpression { Name: "key" } orderingColumn + }, + IsAscending: true + }], + Limit: null, + Offset: null + } selectExpression + && orderingColumn.Table == openJsonExpression + && TranslateExpression(index) is { } translatedIndex) + { + // Index on JSON array + + // Extract the column projected out of the source, and simplify the subquery to a simple JsonScalarExpression + var shaperExpression = source.ShaperExpression; + if (shaperExpression is UnaryExpression { NodeType: ExpressionType.Convert } unaryExpression + && unaryExpression.Operand.Type.IsNullableType() + && unaryExpression.Operand.Type.UnwrapNullableType() == unaryExpression.Type) + { + shaperExpression = unaryExpression.Operand; + } + + if (shaperExpression is ProjectionBindingExpression projectionBindingExpression + && selectExpression.GetProjection(projectionBindingExpression) is SqlExpression projection) + { + // OpenJson's value column is an nvarchar(max); if this is a collection column whose type mapping is know, the projection + // contains a CAST node which we unwrap + var projectionColumn = projection switch + { + ColumnExpression c => c, + SqlUnaryExpression { OperatorType: ExpressionType.Convert, Operand: ColumnExpression c } => c, + _ => null + }; + + if (projectionColumn is not null) + { + SqlExpression translation = new JsonScalarExpression( + jsonArrayColumn, + new PathSegment[] { new(translatedIndex) }, + projection.Type, + projection.TypeMapping, + projectionColumn.IsNullable); + + // If we have a type mapping (i.e. translating over a column rather than a parameter), apply any necessary server-side + // conversions. + // TODO: This should be part of #30677 + // OpenJson's value column has type nvarchar(max); apply a CAST() unless that's the inferred element type mapping + if (projectionColumn.TypeMapping is { StoreType: not "nvarchar(max)"} typeMapping) + { + translation = _sqlExpressionFactory.Convert(translation, typeMapping.ClrType, typeMapping); + } + + return source.UpdateQueryExpression(_sqlExpressionFactory.Select(translation)); + } + } + } + + return base.TranslateElementAtOrDefault(source, index, returnDefault); + } + /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to /// the same compatibility standards as public APIs. It may be changed or removed without notice in @@ -414,6 +504,7 @@ protected virtual TableValuedFunctionExpression ApplyTypeMappingsOnOpenJsonExpre // since we'll always have the same instance of the type mapping returned from the type mapping source. Also remove // CollectionToJsonStringConverter.Equals etc. // TODO: Note: NpgsqlTypeMappingSource exposes FindContainerMapping() for this purpose. + // #30730 if (_typeMappingSource.FindMapping(typeof(string)) is not SqlServerStringTypeMapping parameterTypeMapping) { throw new InvalidOperationException("Type mapping for 'string' could not be found or was not a SqlServerStringTypeMapping"); @@ -436,6 +527,7 @@ protected virtual TableValuedFunctionExpression ApplyTypeMappingsOnOpenJsonExpre /// doing so can result in application failures when updating to a new Entity Framework Core release. /// public virtual SqlExpression ApplyTypeMappingOnColumn(ColumnExpression columnExpression, RelationalTypeMapping typeMapping) + // TODO: this should be part of #30677 // OpenJson's value column has type nvarchar(max); apply a CAST() unless that's the inferred element type mapping => typeMapping.StoreType is "nvarchar(max)" ? columnExpression diff --git a/src/EFCore.Sqlite.Core/Query/Internal/SqliteQuerySqlGenerator.cs b/src/EFCore.Sqlite.Core/Query/Internal/SqliteQuerySqlGenerator.cs index 66db40fa00c..6ebd4a515c4 100644 --- a/src/EFCore.Sqlite.Core/Query/Internal/SqliteQuerySqlGenerator.cs +++ b/src/EFCore.Sqlite.Core/Query/Internal/SqliteQuerySqlGenerator.cs @@ -130,11 +130,11 @@ protected override Expression VisitJsonScalar(JsonScalarExpression jsonScalarExp var path = jsonScalarExpression.Path; if (path.Count == 0) { - Visit(jsonScalarExpression.JsonColumn); + Visit(jsonScalarExpression.Json); return jsonScalarExpression; } - Visit(jsonScalarExpression.JsonColumn); + Visit(jsonScalarExpression.Json); var inJsonpathString = false; diff --git a/src/EFCore.Sqlite.Core/Query/Internal/SqliteQueryableMethodTranslatingExpressionVisitor.cs b/src/EFCore.Sqlite.Core/Query/Internal/SqliteQueryableMethodTranslatingExpressionVisitor.cs index d0728006d3e..ccd80eb9f4a 100644 --- a/src/EFCore.Sqlite.Core/Query/Internal/SqliteQueryableMethodTranslatingExpressionVisitor.cs +++ b/src/EFCore.Sqlite.Core/Query/Internal/SqliteQueryableMethodTranslatingExpressionVisitor.cs @@ -205,6 +205,72 @@ protected override ShapedQueryExpression TranslateCollection( return new ShapedQueryExpression(selectExpression, shaperExpression); } + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + protected override ShapedQueryExpression? TranslateElementAtOrDefault( + ShapedQueryExpression source, + Expression index, + bool returnDefault) + { + if (!returnDefault + && source.QueryExpression is SelectExpression + { + Tables: + [ + TableValuedFunctionExpression + { + Name: "json_each", Schema: null, IsBuiltIn: true, Arguments: [var jsonArrayColumn] + } jsonEachExpression + ], + GroupBy: [], + Having: null, + IsDistinct: false, + Orderings: [{ Expression: ColumnExpression { Name: "key" } orderingColumn, IsAscending: true }], + Limit: null, + Offset: null + } selectExpression + && orderingColumn.Table == jsonEachExpression + && TranslateExpression(index) is { } translatedIndex) + { + // Index on JSON array + + // Extract the column projected out of the source, and simplify the subquery to a simple JsonScalarExpression + var shaperExpression = source.ShaperExpression; + if (shaperExpression is UnaryExpression { NodeType: ExpressionType.Convert } unaryExpression + && unaryExpression.Operand.Type.IsNullableType() + && unaryExpression.Operand.Type.UnwrapNullableType() == unaryExpression.Type) + { + shaperExpression = unaryExpression.Operand; + } + + if (shaperExpression is ProjectionBindingExpression projectionBindingExpression + && selectExpression.GetProjection(projectionBindingExpression) is ColumnExpression projectionColumn) + { + SqlExpression translation = new JsonScalarExpression( + jsonArrayColumn, + new[] { new PathSegment(translatedIndex) }, + projectionColumn.Type, + projectionColumn.TypeMapping, + projectionColumn.IsNullable); + + // If we have a type mapping (i.e. translating over a column rather than a parameter), apply any necessary server-side + // conversions. + if (projectionColumn.TypeMapping is not null) + { + translation = ApplyTypeMappingOnColumn(translation, projectionColumn.TypeMapping, projectionColumn.IsNullable); + } + + return source.UpdateQueryExpression(_sqlExpressionFactory.Select(translation)); + } + } + + return base.TranslateElementAtOrDefault(source, index, returnDefault); + } + private static Type GetProviderType(SqlExpression expression) => expression.TypeMapping?.Converter?.ProviderClrType ?? expression.TypeMapping?.ClrType @@ -293,7 +359,7 @@ when InferredTypeMappings.TryGetValue((jsonEachExpression, "value"), out var typ case ColumnExpression { Name: "value" } columnExpression when _currentSelectInferredTypeMappings is not null && _currentSelectInferredTypeMappings.TryGetValue(columnExpression.Table, out var inferredTypeMapping): - return ApplyTypeMappingOnColumn(columnExpression, inferredTypeMapping); + return ApplyTypeMappingOnColumn(columnExpression, inferredTypeMapping, columnExpression.IsNullable); default: return base.VisitExtension(expression); @@ -318,7 +384,7 @@ protected virtual TableValuedFunctionExpression ApplyTypeMappingsOnJsonEachExpre } // TODO: We shouldn't need to manually construct the JSON string type mapping this way; we need to be able to provide the - // TODO: element's store type mapping as input to _typeMappingSource.FindMapping. + // TODO: element's store type mapping as input to _typeMappingSource.FindMapping. #30730 if (_typeMappingSource.FindMapping(typeof(string)) is not SqliteStringTypeMapping parameterTypeMapping) { throw new InvalidOperationException("Type mapping for 'string' could not be found or was not a SqliteStringTypeMapping"); @@ -331,27 +397,21 @@ protected virtual TableValuedFunctionExpression ApplyTypeMappingsOnJsonEachExpre return jsonEachExpression.Update(new[] { parameterExpression.ApplyTypeMapping(parameterTypeMapping) }); } + } - /// - /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to - /// the same compatibility standards as public APIs. It may be changed or removed without notice in - /// any release. You should only use it directly in your code with extreme caution and knowing that - /// doing so can result in application failures when updating to a new Entity Framework Core release. - /// - public virtual SqlExpression ApplyTypeMappingOnColumn(ColumnExpression columnExpression, RelationalTypeMapping typeMapping) - => typeMapping switch - { - // TODO: These server-side conversions need to be managed on the type mapping + private static SqlExpression ApplyTypeMappingOnColumn(SqlExpression expression, RelationalTypeMapping typeMapping, bool isNullable) + => typeMapping switch + { + // TODO: These server-side conversions need to be managed on the type mapping, #30677 - // The "standard" JSON timestamp representation is ISO8601, with a T between date and time; but SQLite's representation has - // no T. Apply a conversion on the value coming out of json_each. - SqliteDateTimeTypeMapping => _sqlExpressionFactory.Function( - "datetime", new[] { columnExpression }, nullable: true, new[] { true }, typeof(DateTime), typeMapping), + // The "standard" JSON timestamp representation is ISO8601, with a T between date and time; but SQLite's representation has + // no T. Apply a conversion on the value coming out of json_each. + SqliteDateTimeTypeMapping => new SqlFunctionExpression( + "datetime", new[] { expression }, isNullable, new[] { true }, typeof(DateTime), typeMapping), - SqliteGuidTypeMapping => _sqlExpressionFactory.Function( - "upper", new[] { columnExpression }, nullable: true, new[] { true }, typeof(Guid), typeMapping), + SqliteGuidTypeMapping => new SqlFunctionExpression( + "upper", new[] { expression }, isNullable, new[] { true }, typeof(Guid), typeMapping), - _ => columnExpression - }; - } + _ => expression + }; } diff --git a/test/EFCore.Specification.Tests/Query/PrimitiveCollectionsQueryTestBase.cs b/test/EFCore.Specification.Tests/Query/PrimitiveCollectionsQueryTestBase.cs index 199ae52ce07..aff25e2494a 100644 --- a/test/EFCore.Specification.Tests/Query/PrimitiveCollectionsQueryTestBase.cs +++ b/test/EFCore.Specification.Tests/Query/PrimitiveCollectionsQueryTestBase.cs @@ -334,9 +334,26 @@ public virtual Task Inline_collection_index_Column(bool async) ss => ss.Set().Where(c => (c.Int <= 2 ? new[] { 1, 2, 3 }[c.Int] : -1) == 1), entryCount: 1); + // The JsonScalarExpression (ints[c.Int]) should get inferred from the column on the other side (c.Int), and that should propagate to + // ints [ConditionalTheory] [MemberData(nameof(IsAsyncData))] - public virtual Task Parameter_collection_index_Column(bool async) + public virtual Task Parameter_collection_index_Column_equal_Column(bool async) + { + var ints = new[] { 0, 2, 3 }; + + return AssertQuery( + async, + ss => ss.Set().Where(c => ints[c.Int] == c.Int), + ss => ss.Set().Where(c => (c.Int <= 2 ? ints[c.Int] : -1) == c.Int), + entryCount: 1); + } + + // Since the JsonScalarExpression (ints[c.Int]) is being compared to a constant, there's nothing to infer the type mapping from. + // ints should get the default type mapping for based on its CLR type. + [ConditionalTheory] + [MemberData(nameof(IsAsyncData))] + public virtual Task Parameter_collection_index_Column_equal_constant(bool async) { var ints = new[] { 1, 2, 3 }; @@ -494,7 +511,6 @@ public virtual async Task Parameter_collection_in_subquery_Count_as_compiled_que // subquery as its table, and not directly to the parameter's table. // This creates an initially untyped ColumnExpression referencing the pushed-down subquery; it must also be inferred. // Note that this must be a compiled query, since with normal queries the Skip(1) gets client-evaluated. - // TODO: var compiledQuery = EF.CompileQuery( (PrimitiveCollectionsContext context, int[] ints) => context.Set().Where(p => ints.Skip(1).Count(i => i > p.Id) == 1).Count()); diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQueryOldSqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQueryOldSqlServerTest.cs index 5003eeb4c76..7bb8aadb05c 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQueryOldSqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQueryOldSqlServerTest.cs @@ -319,8 +319,8 @@ ORDER BY [v].[_ord] """); } - public override Task Parameter_collection_index_Column(bool async) - => AssertTranslationFailed(() => base.Parameter_collection_index_Column(async)); + public override Task Parameter_collection_index_Column_equal_constant(bool async) + => AssertTranslationFailed(() => base.Parameter_collection_index_Column_equal_constant(async)); public override Task Column_collection_ElementAt(bool async) => AssertTranslationFailed(() => base.Column_collection_ElementAt(async)); diff --git a/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQuerySqlServerTest.cs b/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQuerySqlServerTest.cs index 8a06980d06e..4e12c199e0a 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQuerySqlServerTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Query/PrimitiveCollectionsQuerySqlServerTest.cs @@ -423,11 +423,7 @@ public override async Task Column_collection_index_int(bool async) """ SELECT [p].[Id], [p].[Bool], [p].[Bools], [p].[DateTime], [p].[DateTimes], [p].[Enum], [p].[Enums], [p].[Int], [p].[Ints], [p].[NullableInt], [p].[NullableInts], [p].[String], [p].[Strings] FROM [PrimitiveCollectionsEntity] AS [p] -WHERE ( - SELECT CAST([i].[value] AS int) - FROM OpenJson([p].[Ints]) AS [i] - ORDER BY CAST([i].[key] AS int) - OFFSET 1 ROWS FETCH NEXT 1 ROWS ONLY) = 10 +WHERE CAST(JSON_VALUE([p].[Ints], '$[1]') AS int) = 10 """); } @@ -436,14 +432,10 @@ public override async Task Column_collection_index_string(bool async) await base.Column_collection_index_string(async); AssertSql( -""" + """ SELECT [p].[Id], [p].[Bool], [p].[Bools], [p].[DateTime], [p].[DateTimes], [p].[Enum], [p].[Enums], [p].[Int], [p].[Ints], [p].[NullableInt], [p].[NullableInts], [p].[String], [p].[Strings] FROM [PrimitiveCollectionsEntity] AS [p] -WHERE ( - SELECT [s].[value] - FROM OpenJson([p].[Strings]) AS [s] - ORDER BY CAST([s].[key] AS int) - OFFSET 1 ROWS FETCH NEXT 1 ROWS ONLY) = N'10' +WHERE JSON_VALUE([p].[Strings], '$[1]') = N'10' """); } @@ -455,11 +447,7 @@ public override async Task Column_collection_index_datetime(bool async) """ SELECT [p].[Id], [p].[Bool], [p].[Bools], [p].[DateTime], [p].[DateTimes], [p].[Enum], [p].[Enums], [p].[Int], [p].[Ints], [p].[NullableInt], [p].[NullableInts], [p].[String], [p].[Strings] FROM [PrimitiveCollectionsEntity] AS [p] -WHERE ( - SELECT CAST([d].[value] AS datetime2) - FROM OpenJson([p].[DateTimes]) AS [d] - ORDER BY CAST([d].[key] AS int) - OFFSET 1 ROWS FETCH NEXT 1 ROWS ONLY) = '2020-01-10T12:30:00.0000000Z' +WHERE CAST(JSON_VALUE([p].[DateTimes], '$[1]') AS datetime2) = '2020-01-10T12:30:00.0000000Z' """); } @@ -471,11 +459,7 @@ public override async Task Column_collection_index_beyond_end(bool async) """ SELECT [p].[Id], [p].[Bool], [p].[Bools], [p].[DateTime], [p].[DateTimes], [p].[Enum], [p].[Enums], [p].[Int], [p].[Ints], [p].[NullableInt], [p].[NullableInts], [p].[String], [p].[Strings] FROM [PrimitiveCollectionsEntity] AS [p] -WHERE ( - SELECT CAST([i].[value] AS int) - FROM OpenJson([p].[Ints]) AS [i] - ORDER BY CAST([i].[key] AS int) - OFFSET 999 ROWS FETCH NEXT 1 ROWS ONLY) = 10 +WHERE CAST(JSON_VALUE([p].[Ints], '$[999]') AS int) = 10 """); } @@ -495,9 +479,23 @@ ORDER BY [v].[_ord] """); } - public override async Task Parameter_collection_index_Column(bool async) + public override async Task Parameter_collection_index_Column_equal_Column(bool async) + { + await base.Parameter_collection_index_Column_equal_Column(async); + + AssertSql( +""" +@__ints_0='[0,2,3]' (Size = 4000) + +SELECT [p].[Id], [p].[Bool], [p].[Bools], [p].[DateTime], [p].[DateTimes], [p].[Enum], [p].[Enums], [p].[Int], [p].[Ints], [p].[NullableInt], [p].[NullableInts], [p].[String], [p].[Strings] +FROM [PrimitiveCollectionsEntity] AS [p] +WHERE CAST(JSON_VALUE(@__ints_0, '$[' + CAST([p].[Int] AS nvarchar(max)) + ']') AS int) = [p].[Int] +"""); + } + + public override async Task Parameter_collection_index_Column_equal_constant(bool async) { - await base.Parameter_collection_index_Column(async); + await base.Parameter_collection_index_Column_equal_constant(async); AssertSql( """ @@ -505,11 +503,7 @@ public override async Task Parameter_collection_index_Column(bool async) SELECT [p].[Id], [p].[Bool], [p].[Bools], [p].[DateTime], [p].[DateTimes], [p].[Enum], [p].[Enums], [p].[Int], [p].[Ints], [p].[NullableInt], [p].[NullableInts], [p].[String], [p].[Strings] FROM [PrimitiveCollectionsEntity] AS [p] -WHERE ( - SELECT CAST([i].[value] AS int) AS [value] - FROM OpenJson(@__ints_0) AS [i] - ORDER BY CAST([i].[key] AS int) - OFFSET [p].[Int] ROWS FETCH NEXT 1 ROWS ONLY) = 1 +WHERE CAST(JSON_VALUE(@__ints_0, '$[' + CAST([p].[Int] AS nvarchar(max)) + ']') AS int) = 1 """); } @@ -521,11 +515,7 @@ public override async Task Column_collection_ElementAt(bool async) """ SELECT [p].[Id], [p].[Bool], [p].[Bools], [p].[DateTime], [p].[DateTimes], [p].[Enum], [p].[Enums], [p].[Int], [p].[Ints], [p].[NullableInt], [p].[NullableInts], [p].[String], [p].[Strings] FROM [PrimitiveCollectionsEntity] AS [p] -WHERE ( - SELECT CAST([i].[value] AS int) - FROM OpenJson([p].[Ints]) AS [i] - ORDER BY CAST([i].[key] AS int) - OFFSET 1 ROWS FETCH NEXT 1 ROWS ONLY) = 10 +WHERE CAST(JSON_VALUE([p].[Ints], '$[1]') AS int) = 10 """); } diff --git a/test/EFCore.Sqlite.FunctionalTests/Query/PrimitiveCollectionsQuerySqliteTest.cs b/test/EFCore.Sqlite.FunctionalTests/Query/PrimitiveCollectionsQuerySqliteTest.cs index 53f39ceca8e..8fc5012b386 100644 --- a/test/EFCore.Sqlite.FunctionalTests/Query/PrimitiveCollectionsQuerySqliteTest.cs +++ b/test/EFCore.Sqlite.FunctionalTests/Query/PrimitiveCollectionsQuerySqliteTest.cs @@ -1,6 +1,8 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using Microsoft.Data.Sqlite; + namespace Microsoft.EntityFrameworkCore.Query; public class PrimitiveCollectionsQuerySqliteTest : PrimitiveCollectionsQueryTestBase< @@ -10,7 +12,7 @@ public PrimitiveCollectionsQuerySqliteTest(PrimitiveCollectionsQuerySqlServerFix : base(fixture) { Fixture.TestSqlLoggerFactory.Clear(); - //Fixture.TestSqlLoggerFactory.SetTestOutputHelper(testOutputHelper); + // Fixture.TestSqlLoggerFactory.SetTestOutputHelper(testOutputHelper); } public override async Task Inline_collection_of_ints_Contains(bool async) @@ -409,11 +411,7 @@ public override async Task Column_collection_index_int(bool async) """ SELECT "p"."Id", "p"."Bool", "p"."Bools", "p"."DateTime", "p"."DateTimes", "p"."Enum", "p"."Enums", "p"."Int", "p"."Ints", "p"."NullableInt", "p"."NullableInts", "p"."String", "p"."Strings" FROM "PrimitiveCollectionsEntity" AS "p" -WHERE ( - SELECT "i"."value" - FROM json_each("p"."Ints") AS "i" - ORDER BY "i"."key" - LIMIT 1 OFFSET 1) = 10 +WHERE "p"."Ints" ->> 1 = 10 """); } @@ -425,11 +423,7 @@ public override async Task Column_collection_index_string(bool async) """ SELECT "p"."Id", "p"."Bool", "p"."Bools", "p"."DateTime", "p"."DateTimes", "p"."Enum", "p"."Enums", "p"."Int", "p"."Ints", "p"."NullableInt", "p"."NullableInts", "p"."String", "p"."Strings" FROM "PrimitiveCollectionsEntity" AS "p" -WHERE ( - SELECT "s"."value" - FROM json_each("p"."Strings") AS "s" - ORDER BY "s"."key" - LIMIT 1 OFFSET 1) = '10' +WHERE "p"."Strings" ->> 1 = '10' """); } @@ -441,11 +435,7 @@ public override async Task Column_collection_index_datetime(bool async) """ SELECT "p"."Id", "p"."Bool", "p"."Bools", "p"."DateTime", "p"."DateTimes", "p"."Enum", "p"."Enums", "p"."Int", "p"."Ints", "p"."NullableInt", "p"."NullableInts", "p"."String", "p"."Strings" FROM "PrimitiveCollectionsEntity" AS "p" -WHERE ( - SELECT datetime("d"."value") AS "value" - FROM json_each("p"."DateTimes") AS "d" - ORDER BY "d"."key" - LIMIT 1 OFFSET 1) = '2020-01-10 12:30:00' +WHERE datetime("p"."DateTimes" ->> 1) = '2020-01-10 12:30:00' """); } @@ -457,45 +447,52 @@ public override async Task Column_collection_index_beyond_end(bool async) """ SELECT "p"."Id", "p"."Bool", "p"."Bools", "p"."DateTime", "p"."DateTimes", "p"."Enum", "p"."Enums", "p"."Int", "p"."Ints", "p"."NullableInt", "p"."NullableInts", "p"."String", "p"."Strings" FROM "PrimitiveCollectionsEntity" AS "p" -WHERE ( - SELECT "i"."value" - FROM json_each("p"."Ints") AS "i" - ORDER BY "i"."key" - LIMIT 1 OFFSET 999) = 10 +WHERE "p"."Ints" ->> 999 = 10 """); } - [ConditionalTheory(Skip = "Sqlite issue, should be taken care of by #30724")] public override async Task Inline_collection_index_Column(bool async) { - await base.Inline_collection_index_Column(async); + // SQLite doesn't support correlated subqueries where the outer column is used as the LIMIT/OFFSET (see OFFSET "p"."Int" below) + await Assert.ThrowsAsync(() => base.Inline_collection_index_Column(async)); AssertSql( """ -SELECT [p].[Id], [p].[Bool], [p].[Bools], [p].[DateTime], [p].[DateTimes], [p].[Enum], [p].[Enums], [p].[Int], [p].[Ints], [p].[NullableInt], [p].[NullableInts], [p].[String], [p].[Strings] -FROM [PrimitiveCollectionsEntity] AS [p] +SELECT "p"."Id", "p"."Bool", "p"."Bools", "p"."DateTime", "p"."DateTimes", "p"."Enum", "p"."Enums", "p"."Int", "p"."Ints", "p"."NullableInt", "p"."NullableInts", "p"."String", "p"."Strings" +FROM "PrimitiveCollectionsEntity" AS "p" WHERE ( - SELECT [v].[Value] - FROM (VALUES (0, CAST(1 AS int)), (1, 2), (2, 3)) AS [v]([_ord], [Value]) - ORDER BY [v].[_ord] - OFFSET [p].[Int] ROWS FETCH NEXT 1 ROWS ONLY) = 1 + SELECT "v"."Value" + FROM (SELECT 0 AS "_ord", CAST(1 AS INTEGER) AS "Value" UNION ALL VALUES (1, 2), (2, 3)) AS "v" + ORDER BY "v"."_ord" + LIMIT 1 OFFSET "p"."Int") = 1 """); } - [ConditionalTheory(Skip = "Sqlite issue, should be taken care of by #30724")] - public override async Task Parameter_collection_index_Column(bool async) + public override async Task Parameter_collection_index_Column_equal_Column(bool async) { - await base.Parameter_collection_index_Column(async); + await base.Parameter_collection_index_Column_equal_Column(async); AssertSql( - """ -SELECT [p].[Id], [p].[Bool], [p].[Bools], [p].[DateTime], [p].[DateTimes], [p].[Enum], [p].[Enums], [p].[Int], [p].[Ints], [p].[NullableInt], [p].[NullableInts], [p].[String], [p].[Strings] -FROM [PrimitiveCollectionsEntity] AS [p] -WHERE ( - SELECT [v].[Value] - FROM (VALUES (0, CAST(1 AS int)), (1, 2), (2, 3)) AS [v]([_ord], [Value]) - ORDER BY [v].[_ord] - OFFSET [p].[Int] ROWS FETCH NEXT 1 ROWS ONLY) = 1 +""" +@__ints_0='[0,2,3]' (Size = 7) + +SELECT "p"."Id", "p"."Bool", "p"."Bools", "p"."DateTime", "p"."DateTimes", "p"."Enum", "p"."Enums", "p"."Int", "p"."Ints", "p"."NullableInt", "p"."NullableInts", "p"."String", "p"."Strings" +FROM "PrimitiveCollectionsEntity" AS "p" +WHERE @__ints_0 ->> "p"."Int" = "p"."Int" +"""); + } + + public override async Task Parameter_collection_index_Column_equal_constant(bool async) + { + await base.Parameter_collection_index_Column_equal_constant(async); + + AssertSql( +""" +@__ints_0='[1,2,3]' (Size = 7) + +SELECT "p"."Id", "p"."Bool", "p"."Bools", "p"."DateTime", "p"."DateTimes", "p"."Enum", "p"."Enums", "p"."Int", "p"."Ints", "p"."NullableInt", "p"."NullableInts", "p"."String", "p"."Strings" +FROM "PrimitiveCollectionsEntity" AS "p" +WHERE @__ints_0 ->> "p"."Int" = 1 """); } @@ -507,11 +504,7 @@ public override async Task Column_collection_ElementAt(bool async) """ SELECT "p"."Id", "p"."Bool", "p"."Bools", "p"."DateTime", "p"."DateTimes", "p"."Enum", "p"."Enums", "p"."Int", "p"."Ints", "p"."NullableInt", "p"."NullableInts", "p"."String", "p"."Strings" FROM "PrimitiveCollectionsEntity" AS "p" -WHERE ( - SELECT "i"."value" - FROM json_each("p"."Ints") AS "i" - ORDER BY "i"."key" - LIMIT 1 OFFSET 1) = 10 +WHERE "p"."Ints" ->> 1 = 10 """); }