Experimental support for the Azure SQL JSON type (#34401)

* Experimental support for the Azure SQL json type

Fixes #28452
Fixes #32150

Remaining work:

- Test reverse engineering from an existing database
- Output a warning when the native JSON type is used
- Replace the ToJson overload with HasColumnType()
- Move the type mapping visitation to another visitor

Known issues:

- Various issues communicated with the SQL team--see TODO:SQLJSON
- Testing is disabled until we have an appropriate server and driver to test against

* Revert changes needed for testing.

* Fix SQLite tests

* EF code updates

* More review updates.

* More updates,

* Disable Cosmos tests on C.I.

src/EFCore.Relational/Design/AnnotationCodeGenerator.cs

@@ -243,6 +243,18 @@ public virtual IReadOnlyList<MethodCallCodeFragment> GenerateFluentApiCalls(
 #pragma warning restore CS0618
         }
 
+        if (annotations.TryGetValue(RelationalAnnotationNames.ContainerColumnType, out var containerColumnTypeAnnotation)
+            && containerColumnTypeAnnotation is { Value: string containerColumnType }
+            && entityType.IsOwned())
+        {
+            methodCallCodeFragments.Add(
+                new MethodCallCodeFragment(
+                    nameof(RelationalOwnedNavigationBuilderExtensions.HasColumnType),
+                    containerColumnType));
+
+            annotations.Remove(RelationalAnnotationNames.ContainerColumnType);
+        }
+
         methodCallCodeFragments.AddRange(GenerateFluentApiCallsHelper(entityType, annotations, GenerateFluentApi));
 
         return methodCallCodeFragments;

src/EFCore.Relational/Extensions/RelationalEntityTypeExtensions.cs

@@ -1606,9 +1606,46 @@ public static void SetContainerColumnName(this IMutableEntityType entityType, st
     /// <param name="entityType">The entity type to get the container column name for.</param>
     /// <returns>The container column name to which the entity type is mapped.</returns>
     public static string? GetContainerColumnName(this IReadOnlyEntityType entityType)
-        => entityType.FindAnnotation(RelationalAnnotationNames.ContainerColumnName)?.Value is string columnName
-            ? columnName
-            : (entityType.FindOwnership()?.PrincipalEntityType.GetContainerColumnName());
+        => entityType.FindAnnotation(RelationalAnnotationNames.ContainerColumnName)?.Value as string
+            ?? entityType.FindOwnership()?.PrincipalEntityType.GetContainerColumnName();
+
+    /// <summary>
+    ///     Sets the column type to use for the container column to which the entity type is mapped.
+    /// </summary>
+    /// <param name="entityType">The entity type.</param>
+    /// <param name="columnType">The database column type.</param>
+    public static void SetContainerColumnType(this IMutableEntityType entityType, string? columnType)
+        => entityType.SetOrRemoveAnnotation(RelationalAnnotationNames.ContainerColumnType, columnType);
+
+    /// <summary>
+    ///     Sets the column type to use for the container column to which the entity type is mapped.
+    /// </summary>
+    /// <param name="entityType">The entity type.</param>
+    /// <param name="columnType">The database column type.</param>
+    /// <param name="fromDataAnnotation">Indicates whether the configuration was specified using a data annotation.</param>
+    /// <returns>The configured value.</returns>
+    public static string? SetContainerColumnType(
+        this IConventionEntityType entityType,
+        string? columnType,
+        bool fromDataAnnotation = false)
+        => (string?)entityType.SetAnnotation(RelationalAnnotationNames.ContainerColumnType, columnType, fromDataAnnotation)?.Value;
+
+    /// <summary>
+    ///     Gets the <see cref="ConfigurationSource" /> for the container column type.
+    /// </summary>
+    /// <param name="entityType">The entity type.</param>
+    /// <returns>The <see cref="ConfigurationSource" />.</returns>
+    public static ConfigurationSource? GetContainerColumnTypeConfigurationSource(this IConventionEntityType entityType)
+        => entityType.FindAnnotation(RelationalAnnotationNames.ContainerColumnType)
+            ?.GetConfigurationSource();
+
+    /// <summary>
+    ///     Gets the column type to use for the container column to which the entity type is mapped.
+    /// </summary>
+    /// <param name="entityType">The entity type.</param>
+    /// <returns>The database column type.</returns>
+    public static string? GetContainerColumnType(this IReadOnlyEntityType entityType)
+        => entityType.FindAnnotation(RelationalAnnotationNames.ContainerColumnType)?.Value as string;
 
     /// <summary>
     ///     Sets the type mapping for the container column to which the entity type is mapped.

src/EFCore.Relational/Extensions/RelationalOwnedNavigationBuilderExtensions.cs

@@ -23,12 +23,7 @@ public static class RelationalOwnedNavigationBuilderExtensions
     /// <param name="builder">The builder for the owned navigation being configured.</param>
     /// <returns>The same builder instance so that multiple calls can be chained.</returns>
     public static OwnedNavigationBuilder ToJson(this OwnedNavigationBuilder builder)
-    {
-        var navigationName = builder.Metadata.GetNavigation(pointsToPrincipal: false)!.Name;
-        builder.ToJson(navigationName);
-
-        return builder;
-    }
+        => builder.ToJson(builder.Metadata.GetNavigation(pointsToPrincipal: false)!.Name);
 
     /// <summary>
     ///     Configures a relationship where this entity type and the entities that it owns are mapped to a JSON column in the database.
@@ -45,12 +40,7 @@ public static OwnedNavigationBuilder<TOwnerEntity, TDependentEntity> ToJson<TOwn
         this OwnedNavigationBuilder<TOwnerEntity, TDependentEntity> builder)
         where TOwnerEntity : class
         where TDependentEntity : class
-    {
-        var navigationName = builder.Metadata.GetNavigation(pointsToPrincipal: false)!.Name;
-        builder.ToJson(navigationName);
-
-        return builder;
-    }
+        => (OwnedNavigationBuilder<TOwnerEntity, TDependentEntity>)((OwnedNavigationBuilder)builder).ToJson();
 
     /// <summary>
     ///     Configures a relationship where this entity type and the entities that it owns are mapped to a JSON column in the database.
@@ -68,11 +58,7 @@ public static OwnedNavigationBuilder<TOwnerEntity, TDependentEntity> ToJson<TOwn
         string? jsonColumnName)
         where TOwnerEntity : class
         where TDependentEntity : class
-    {
-        builder.OwnedEntityType.SetContainerColumnName(jsonColumnName);
-
-        return builder;
-    }
+        => (OwnedNavigationBuilder<TOwnerEntity, TDependentEntity>)((OwnedNavigationBuilder)builder).ToJson(jsonColumnName);
 
     /// <summary>
     ///     Configures a relationship where this entity type and the entities that it owns are mapped to a JSON column in the database.
@@ -94,6 +80,40 @@ public static OwnedNavigationBuilder ToJson(
         return builder;
     }
 
+    /// <summary>
+    ///     Set the relational database column type to be used to store the document represented by this owned entity.
+    /// </summary>
+    /// <remarks>
+    ///     This method should only be specified for the outer-most owned entity in the given ownership structure and
+    ///     only when mapping the column to a database document type.
+    /// </remarks>
+    /// <param name="builder">The builder for the owned navigation being configured.</param>
+    /// <param name="columnType">The database type for the column, or <see langword="null"/> to use the database default.</param>
+    /// <returns>The same builder instance so that multiple calls can be chained.</returns>
+    public static OwnedNavigationBuilder<TOwnerEntity, TDependentEntity> HasColumnType<TOwnerEntity, TDependentEntity>(
+        this OwnedNavigationBuilder<TOwnerEntity, TDependentEntity> builder,
+        string? columnType)
+        where TOwnerEntity : class
+        where TDependentEntity : class
+        => (OwnedNavigationBuilder<TOwnerEntity, TDependentEntity>)((OwnedNavigationBuilder)builder).HasColumnType(columnType);
+
+    /// <summary>
+    ///     Set the relational database column type to be used to store the document represented by this owned entity.
+    /// </summary>
+    /// <remarks>
+    ///     This method should only be specified for the outer-most owned entity in the given ownership structure and
+    ///     only when mapping the column to a database document type.
+    /// </remarks>
+    /// <param name="builder">The builder for the owned navigation being configured.</param>
+    /// <param name="columnType">The database type for the column, or <see langword="null"/> to use the database default.</param>
+    /// <returns>The same builder instance so that multiple calls can be chained.</returns>
+    public static OwnedNavigationBuilder HasColumnType(this OwnedNavigationBuilder builder, string? columnType)
+    {
+        builder.OwnedEntityType.SetContainerColumnType(columnType);
+
+        return builder;
+    }
+
     /// <summary>
     ///     Configures the navigation of an entity mapped to a JSON column, mapping the navigation to a specific JSON property,
     ///     rather than using the navigation name.

src/EFCore.Relational/Extensions/RelationalTypeBaseExtensions.cs

@@ -368,6 +368,17 @@ public static bool IsMappedToJson(this IReadOnlyTypeBase typeBase)
             ? entityType.GetContainerColumnName()
             : ((IReadOnlyComplexType)typeBase).GetContainerColumnName();
 
+
+    /// <summary>
+    ///     Gets the column type to use for the container column to which the type is mapped.
+    /// </summary>
+    /// <param name="typeBase">The type.</param>
+    /// <returns>The database column type.</returns>
+    public static string? GetContainerColumnType(this IReadOnlyTypeBase typeBase)
+        => typeBase is IReadOnlyEntityType entityType
+            ? entityType.GetContainerColumnType()
+            : null;
+
     /// <summary>
     ///     Gets the value of JSON property name used for the given entity mapped to a JSON column.
     /// </summary>

src/EFCore.Relational/Infrastructure/RelationalModelValidator.cs

@@ -2592,6 +2592,23 @@ protected virtual void ValidateJsonEntities(
         IModel model,
         IDiagnosticsLogger<DbLoggerCategory.Model.Validation> logger)
     {
+        foreach (var entityType in model.GetEntityTypes())
+        {
+            if (entityType[RelationalAnnotationNames.ContainerColumnType] != null)
+            {
+                if (entityType.FindOwnership()?.PrincipalEntityType.IsOwned() == true)
+                {
+                    throw new InvalidOperationException(RelationalStrings.ContainerTypeOnNestedOwnedEntityType(entityType.DisplayName()));
+                }
+
+                if (!entityType.IsOwned()
+                    || entityType.GetContainerColumnName() == null)
+                {
+                    throw new InvalidOperationException(RelationalStrings.ContainerTypeOnNonContainer(entityType.DisplayName()));
+                }
+            }
+        }
+
         var tables = BuildSharedTableEntityMap(model.GetEntityTypes());
         foreach (var (table, mappedTypes) in tables)
         {

src/EFCore.Relational/Metadata/Internal/RelationalModel.cs

@@ -286,12 +286,14 @@ private static void AddDefaultMappings(
                 includesDerivedTypes: entityType.GetDirectlyDerivedTypes().Any()
                     ? !isTpc && mappedType == entityType
                     : null);
+
             var containerColumnName = mappedType.GetContainerColumnName();
+            var containerColumnType = mappedType.GetContainerColumnType();
             if (!string.IsNullOrEmpty(containerColumnName))
             {
                 CreateContainerColumn(
-                    defaultTable, containerColumnName, mappedType, relationalTypeMappingSource,
-                    static (c, t, m) => new JsonColumnBase(c, m.StoreType, t, m));
+                    defaultTable, containerColumnName, containerColumnType, mappedType, relationalTypeMappingSource,
+                    static (colName, colType, table, mapping) => new JsonColumnBase(colName, colType ?? mapping.StoreType, table, mapping));
             }
             else
             {
@@ -492,11 +494,12 @@ private static void CreateTableMapping(
         };
 
         var containerColumnName = mappedType.GetContainerColumnName();
+        var containerColumnType = mappedType.GetContainerColumnType();
         if (!string.IsNullOrEmpty(containerColumnName))
         {
             CreateContainerColumn(
-                table, containerColumnName, (IEntityType)mappedType, relationalTypeMappingSource,
-                static (c, t, m) => new JsonColumn(c, m.StoreType, (Table)t, m));
+                table, containerColumnName, containerColumnType, (IEntityType)mappedType, relationalTypeMappingSource,
+                static (colName, colType, table, mapping) => new JsonColumn(colName, colType ?? mapping.StoreType, (Table)table, mapping));
         }
         else
         {
@@ -567,18 +570,19 @@ private static void CreateTableMapping(
     private static void CreateContainerColumn<TColumnMappingBase>(
         TableBase tableBase,
         string containerColumnName,
+        string? containerColumnType,
         IEntityType mappedType,
         IRelationalTypeMappingSource relationalTypeMappingSource,
-        Func<string, TableBase, RelationalTypeMapping, ColumnBase<TColumnMappingBase>> createColumn)
+        Func<string, string?, TableBase, RelationalTypeMapping, ColumnBase<TColumnMappingBase>> createColumn)
         where TColumnMappingBase : class, IColumnMappingBase
     {
         var ownership = mappedType.GetForeignKeys().Single(fk => fk.IsOwnership);
         if (!ownership.PrincipalEntityType.IsMappedToJson())
         {
             Check.DebugAssert(tableBase.FindColumn(containerColumnName) == null, $"Table does not have column '{containerColumnName}'.");
 
-            var jsonColumnTypeMapping = relationalTypeMappingSource.FindMapping(typeof(JsonElement), mappedType.Model)!;
-            var jsonColumn = createColumn(containerColumnName, tableBase, jsonColumnTypeMapping);
+            var jsonColumnTypeMapping = relationalTypeMappingSource.FindMapping(typeof(JsonElement), storeTypeName: containerColumnType)!;
+            var jsonColumn = createColumn(containerColumnName, containerColumnType, tableBase, jsonColumnTypeMapping);
             tableBase.Columns.Add(containerColumnName, jsonColumn);
             jsonColumn.IsNullable = !ownership.IsRequiredDependent || !ownership.IsUnique;
 
@@ -684,11 +688,12 @@ private static void CreateViewMapping(
         };
 
         var containerColumnName = mappedType.GetContainerColumnName();
+        var containerColumnType = mappedType.GetContainerColumnType();
         if (!string.IsNullOrEmpty(containerColumnName))
         {
             CreateContainerColumn(
-                view, containerColumnName, mappedType, relationalTypeMappingSource,
-                static (c, t, m) => new JsonViewColumn(c, m.StoreType, (View)t, m));
+                view, containerColumnName, containerColumnType, mappedType, relationalTypeMappingSource,
+                static (colName, colType, table, mapping) => new JsonViewColumn(colName, colType ?? mapping.StoreType, (View)table, mapping));
         }
         else
         {

src/EFCore.Relational/Metadata/RelationalAnnotationNames.cs

@@ -324,6 +324,11 @@ public static class RelationalAnnotationNames
     /// </summary>
     public const string ContainerColumnName = Prefix + "ContainerColumnName";
 
+    /// <summary>
+    ///     The column type for the container column to which the object is mapped.
+    /// </summary>
+    public const string ContainerColumnType = Prefix + nameof(ContainerColumnType);
+
     /// <summary>
     ///     The name for the annotation specifying container column type mapping.
     /// </summary>
@@ -408,6 +413,7 @@ public static class RelationalAnnotationNames
         ModelDependencies,
         FieldValueGetter,
         ContainerColumnName,
+        ContainerColumnType,
         #pragma warning disable CS0618 // Type or member is obsolete
         ContainerColumnTypeMapping,
         #pragma warning restore CS0618 // Type or member is obsolete

src/EFCore.Relational/Properties/RelationalStrings.resx

@@ -187,6 +187,12 @@
   <data name="ConflictingTypeMappingsInferredForColumn" xml:space="preserve">
     <value>Conflicting type mappings were inferred for column '{column}'.</value>
   </data>
+  <data name="ContainerTypeOnNestedOwnedEntityType" xml:space="preserve">
+    <value>The entity type '{entityType}' has a container column type configured, but is nested in another owned type. The container column type can only be specified on a top-level owned type mapped to a container.</value>
+  </data>
+  <data name="ContainerTypeOnNonContainer" xml:space="preserve">
+    <value>The entity type '{entityType}' has a container column type configured, but is not mapped to a container column, such as for JSON. The container column type can only be specified on a top-level owned type mapped to a container.</value>
+  </data>
   <data name="CreateIndexOperationWithInvalidSortOrder" xml:space="preserve">
     <value>{numSortOrderProperties} values were provided in CreateIndexOperations.IsDescending, but the operation has {numColumns} columns.</value>
   </data>
@@ -514,6 +520,9 @@
   <data name="JsonCantNavigateToParentEntity" xml:space="preserve">
     <value>Can't navigate from JSON-mapped entity '{jsonEntity}' to its parent entity '{parentEntity}' using navigation '{navigation}'. Entities mapped to JSON can only navigate to their children.</value>
   </data>
+  <data name="JsonEmptyString" xml:space="preserve">
+    <value>The database returned the empty string when a JSON object was expected.</value>
+  </data>
   <data name="JsonEntityMappedToDifferentTableOrViewThanOwner" xml:space="preserve">
     <value>Entity '{jsonType}' is mapped to JSON and also to a table or view '{tableOrViewName}', but its owner '{ownerType}' is mapped to a different table or view '{ownerTableOrViewName}'. Every entity mapped to JSON must also map to the same table or view as its owner.</value>
   </data>

src/EFCore.SqlServer/Diagnostics/Internal/SqlServerLoggingDefinitions.cs

@@ -194,4 +194,12 @@ public class SqlServerLoggingDefinitions : RelationalLoggingDefinitions
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
     public EventDefinitionBase? LogMissingViewDefinitionRights;
+
+    /// <summary>
+    ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
+    ///     the same compatibility standards as public APIs. It may be changed or removed without notice in
+    ///     any release. You should only use it directly in your code with extreme caution and knowing that
+    ///     doing so can result in application failures when updating to a new Entity Framework Core release.
+    /// </summary>
+    public EventDefinitionBase? LogJsonTypeExperimental;
 }