Block level variance (#17120)

* Block level variance - initial commit

* Remove TODOs

* Only convert RTEs with blocks

* Fix JSON paths for block level property validation

* Rename Properties to Values

* Correct the JSON path of block level validation errors

* Make it possible to skip content migration + ensure backwards compat for the new block format

* Partial culture variance publishing at property level

* UDI to key conversion for block editors - draft, WIP, do NOT merge 😄  (#16970)

* Convert block UDIs to GUIDs

* Fix merge

* Fix merge issues

* Rework nested layout item key parsing for backwards compatibility

* Clean-up

* Reverse block layout item key calculation

* Review

* Use IOptions to skip content migrations

* Remove "published" from data editor feature naming, as it can be used in other contexts too

* Parallel migration

* Don't use deprecated constructor

* Ensure that layout follows structure for partial publishing

* Block Grid element level variance + tests (incl. refactor of element level variation tests)

* Rollback unintended changes to Program.cs

* Fix bad casing

* Minor formatting

* RTE element level variance + tests

* Remove obsoleted constructors

* Use Umbraco.RichText instead of Umbraco.TinyMCE as layout alias for blocks in the RTE

* Fix bad merge

* Temporary fix for new cache in integration tests

* Add EditorAlias to block level properties

* Remove the unintended PropertyEditorAlias output for block values

* Add EditorAlias to Datatype Item model

* Update OpenApi.json

* Introduce "expose" for blocks

* Strict (explicit) handling for Expose

* Improve handling of document and element level variance changes

* Refactor variance alignment for published rendering

* Block UDI to Key conversion should also register as a conversion

* Convert newly added RTE unit test to new RTE blocks format

* Minor review changes

* Run memory intensive tests on Linux only

* Add tests proving that AllowEditInvariantFromNonDefault has effect for block level variance too

* Fix the Platform annotations

* Removed Platform annotations for tests.

* Fix merge

* Obsolete old PublishCulture extension

* More fixing bad merge

---------

Co-authored-by: Niels Lyngsø <[email protected]>
Co-authored-by: nikolajlauridsen <[email protected]>

src/Umbraco.Cms.Api.Management/Mapping/Item/ItemTypeMapDefinition.cs

@@ -48,6 +48,7 @@ private static void Map(IDataType source, DataTypeItemResponseModel target, Mapp
         target.Name = source.Name ?? string.Empty;
         target.Id = source.Key;
         target.EditorUiAlias = source.EditorUiAlias;
+        target.EditorAlias = source.EditorAlias;
         target.IsDeletable = source.IsDeletableDataType();
     }
 

src/Umbraco.Cms.Api.Management/OpenApi.json

@@ -35987,6 +35987,7 @@
       },
       "DataTypeItemResponseModel": {
         "required": [
+          "editorAlias",
           "id",
           "isDeletable",
           "name"
@@ -36004,6 +36005,9 @@
             "type": "string",
             "nullable": true
           },
+          "editorAlias": {
+            "type": "string"
+          },
           "isDeletable": {
             "type": "boolean"
           }

src/Umbraco.Cms.Api.Management/ViewModels/DataType/Item/DataTypeItemResponseModel.cs

@@ -6,5 +6,7 @@ public class DataTypeItemResponseModel : NamedItemResponseModelBase
 {
     public string? EditorUiAlias { get; set; }
 
+    public string EditorAlias { get; set; } = string.Empty;
+
     public bool IsDeletable { get; set; }
 }

src/Umbraco.Core/Models/Blocks/BlockEditorDataConverter.cs

@@ -62,13 +62,63 @@ public BlockEditorData<TValue, TLayout> Deserialize(string json)
 
     public BlockEditorData<TValue, TLayout> Convert(TValue? value)
     {
-        if (value?.GetLayouts() is not IEnumerable<TLayout> layouts)
+        if (value is not null)
+        {
+            var converted = ConvertOriginalBlockFormat(value.ContentData);
+            if (converted)
+            {
+                ConvertOriginalBlockFormat(value.SettingsData);
+                AmendExpose(value);
+            }
+        }
+
+        TLayout[]? layouts = value?.GetLayouts()?.ToArray();
+        if (layouts is null)
         {
             return BlockEditorData<TValue, TLayout>.Empty;
         }
 
         IEnumerable<ContentAndSettingsReference> references = GetBlockReferences(layouts);
 
-        return new BlockEditorData<TValue, TLayout>(references, value);
+        return new BlockEditorData<TValue, TLayout>(references, value!);
+    }
+
+    // this method is only meant to have any effect when migrating block editor values
+    // from the original format to the new, variant enabled format
+    private void AmendExpose(TValue value)
+        => value.Expose = value.ContentData.Select(cd => new BlockItemVariation(cd.Key, null, null)).ToList();
+
+    // this method is only meant to have any effect when migrating block editor values
+    // from the original format to the new, variant enabled format
+    private bool ConvertOriginalBlockFormat(List<BlockItemData> blockItemDatas)
+    {
+        var converted = false;
+        foreach (BlockItemData blockItemData in blockItemDatas)
+        {
+            // only overwrite the Properties collection if none have been added at this point
+            if (blockItemData.Values.Any() is false && blockItemData.RawPropertyValues.Any())
+            {
+                blockItemData.Values = blockItemData
+                    .RawPropertyValues
+                    .Select(item => new BlockPropertyValue { Alias = item.Key, Value = item.Value })
+                    .ToList();
+                converted = true;
+            }
+
+            // no matter what, clear the RawPropertyValues collection so it is not saved back to the DB
+            blockItemData.RawPropertyValues.Clear();
+
+            // assign the correct Key if only a UDI is set
+            if (blockItemData.Key == Guid.Empty && blockItemData.Udi is GuidUdi guidUdi)
+            {
+                blockItemData.Key = guidUdi.Guid;
+                converted = true;
+            }
+
+            // no matter what, clear the UDI value so it's not saved back to the DB
+            blockItemData.Udi = null;
+        }
+
+        return converted;
     }
 }

src/Umbraco.Core/Models/Blocks/BlockGridEditorDataConverter.cs

@@ -1,8 +1,6 @@
 // Copyright (c) Umbraco.
 // See LICENSE for more details.
 
-using Microsoft.Extensions.DependencyInjection;
-using Umbraco.Cms.Core.DependencyInjection;
 using Umbraco.Cms.Core.Serialization;
 
 namespace Umbraco.Cms.Core.Models.Blocks;
@@ -12,12 +10,6 @@ namespace Umbraco.Cms.Core.Models.Blocks;
 /// </summary>
 public class BlockGridEditorDataConverter : BlockEditorDataConverter<BlockGridValue, BlockGridLayoutItem>
 {
-    [Obsolete("Use the constructor that takes IJsonSerializer. Will be removed in V15.")]
-    public BlockGridEditorDataConverter()
-        : this(StaticServiceProvider.Instance.GetRequiredService<IJsonSerializer>())
-    {
-    }
-
     public BlockGridEditorDataConverter(IJsonSerializer jsonSerializer)
         : base(jsonSerializer)
     {
@@ -27,7 +19,7 @@ protected override IEnumerable<ContentAndSettingsReference> GetBlockReferences(I
     {
         IList<ContentAndSettingsReference> ExtractContentAndSettingsReferences(BlockGridLayoutItem item)
         {
-            var references = new List<ContentAndSettingsReference> { new(item.ContentUdi, item.SettingsUdi) };
+            var references = new List<ContentAndSettingsReference> { new(item.ContentKey, item.SettingsKey) };
             references.AddRange(item.Areas.SelectMany(area => area.Items.SelectMany(ExtractContentAndSettingsReferences)));
             return references;
         }

src/Umbraco.Core/Models/Blocks/BlockGridItem.cs

@@ -23,21 +23,45 @@ public class BlockGridItem : IBlockReference<IPublishedElement, IPublishedElemen
         /// <exception cref="ArgumentNullException">contentUdi
         /// or
         /// content</exception>
+        [Obsolete("Use constructor that accepts GUIDs instead. Will be removed in V18.")]
         public BlockGridItem(Udi contentUdi, IPublishedElement content, Udi settingsUdi, IPublishedElement settings)
+            : this(
+                (contentUdi as GuidUdi)?.Guid ?? throw new ArgumentException(nameof(contentUdi)),
+                content,
+                (settingsUdi as GuidUdi)?.Guid,
+                settings)
         {
-            ContentUdi = contentUdi ?? throw new ArgumentNullException(nameof(contentUdi));
+        }
+
+        public BlockGridItem(Guid contentKey, IPublishedElement content, Guid? settingsKey, IPublishedElement? settings)
+        {
+            ContentKey = contentKey;
+            ContentUdi = new GuidUdi(Constants.UdiEntityType.Element, contentKey);
             Content = content ?? throw new ArgumentNullException(nameof(content));
-            SettingsUdi = settingsUdi;
+            SettingsKey = settingsKey;
+            SettingsUdi = settingsKey.HasValue
+                ? new GuidUdi(Constants.UdiEntityType.Element, settingsKey.Value)
+                : null;
             Settings = settings;
         }
 
+        /// <summary>
+        /// Gets the content key.
+        /// </summary>
+        public Guid ContentKey { get; set; }
+
+        /// <summary>
+        /// Gets the settings key.
+        /// </summary>
+        public Guid? SettingsKey { get; set; }
+
         /// <summary>
         /// Gets the content UDI.
         /// </summary>
         /// <value>
         /// The content UDI.
         /// </value>
-        [DataMember(Name = "contentUdi")]
+        [Obsolete("Use ContentKey instead. Will be removed in V18.")]
         public Udi ContentUdi { get; }
 
         /// <summary>
@@ -46,7 +70,6 @@ public BlockGridItem(Udi contentUdi, IPublishedElement content, Udi settingsUdi,
         /// <value>
         /// The content.
         /// </value>
-        [DataMember(Name = "content")]
         public IPublishedElement Content { get; }
 
         /// <summary>
@@ -55,46 +78,40 @@ public BlockGridItem(Udi contentUdi, IPublishedElement content, Udi settingsUdi,
         /// <value>
         /// The settings UDI.
         /// </value>
-        [DataMember(Name = "settingsUdi")]
-        public Udi SettingsUdi { get; }
+        [Obsolete("Use SettingsKey instead. Will be removed in V18.")]
+        public Udi? SettingsUdi { get; }
 
         /// <summary>
         /// Gets the settings.
         /// </summary>
         /// <value>
         /// The settings.
         /// </value>
-        [DataMember(Name = "settings")]
-        public IPublishedElement Settings { get; }
+        public IPublishedElement? Settings { get; }
 
         /// <summary>
         /// The number of rows this item should span
         /// </summary>
-        [DataMember(Name = "rowSpan")]
         public int RowSpan { get; set; }
 
         /// <summary>
         /// The number of columns this item should span
         /// </summary>
-        [DataMember(Name = "columnSpan")]
         public int ColumnSpan { get; set; }
 
         /// <summary>
         /// The grid areas within this item
         /// </summary>
-        [DataMember(Name = "areas")]
         public IEnumerable<BlockGridArea> Areas { get; set; } = Array.Empty<BlockGridArea>();
 
         /// <summary>
         /// The number of columns available for the areas to span
         /// </summary>
-        [DataMember(Name = "areaGridColumns")]
         public int? AreaGridColumns { get; set; }
 
         /// <summary>
         /// The number of columns in the root grid
         /// </summary>
-        [DataMember(Name = "gridColumns")]
         public int? GridColumns { get; set; }
     }
 
@@ -112,12 +129,19 @@ public class BlockGridItem<T> : BlockGridItem
         /// <param name="content">The content.</param>
         /// <param name="settingsUdi">The settings UDI.</param>
         /// <param name="settings">The settings.</param>
+        [Obsolete("Use constructor that accepts GUIDs instead. Will be removed in V18.")]
         public BlockGridItem(Udi contentUdi, T content, Udi settingsUdi, IPublishedElement settings)
             : base(contentUdi, content, settingsUdi, settings)
         {
             Content = content;
         }
 
+        public BlockGridItem(Guid contentKey, T content, Guid? settingsKey, IPublishedElement? settings)
+            : base(contentKey, content, settingsKey, settings)
+        {
+            Content = content;
+        }
+
         /// <summary>
         /// Gets the content.
         /// </summary>
@@ -143,18 +167,25 @@ public class BlockGridItem<TContent, TSettings> : BlockGridItem<TContent>
         /// <param name="content">The content.</param>
         /// <param name="settingsUdi">The settings udi.</param>
         /// <param name="settings">The settings.</param>
+        [Obsolete("Use constructor that accepts GUIDs instead. Will be removed in V18.")]
         public BlockGridItem(Udi contentUdi, TContent content, Udi settingsUdi, TSettings settings)
             : base(contentUdi, content, settingsUdi, settings)
         {
             Settings = settings;
         }
 
+        public BlockGridItem(Guid contentKey, TContent content, Guid? settingsKey, TSettings? settings)
+            : base(contentKey, content, settingsKey, settings)
+        {
+            Settings = settings;
+        }
+
         /// <summary>
         /// Gets the settings.
         /// </summary>
         /// <value>
         /// The settings.
         /// </value>
-        public new TSettings Settings { get; }
+        public new TSettings? Settings { get; }
     }
 }

src/Umbraco.Core/Models/Blocks/BlockGridLayoutItem.cs

@@ -6,12 +6,8 @@ namespace Umbraco.Cms.Core.Models.Blocks;
 /// <summary>
 /// Used for deserializing the block grid layout
 /// </summary>
-public class BlockGridLayoutItem : IBlockLayoutItem
+public class BlockGridLayoutItem : BlockLayoutItemBase
 {
-    public Udi? ContentUdi { get; set; }
-
-    public Udi? SettingsUdi { get; set; }
-
     public int? ColumnSpan { get; set; }
 
     public int? RowSpan { get; set; }
@@ -21,10 +17,25 @@ public class BlockGridLayoutItem : IBlockLayoutItem
     public BlockGridLayoutItem()
     { }
 
+    [Obsolete("Use constructor that accepts GUIDs instead. Will be removed in V18.")]
     public BlockGridLayoutItem(Udi contentUdi)
-        => ContentUdi = contentUdi;
+        : base(contentUdi)
+    {
+    }
 
+    [Obsolete("Use constructor that accepts GUIDs instead. Will be removed in V18.")]
     public BlockGridLayoutItem(Udi contentUdi, Udi settingsUdi)
-        : this(contentUdi)
-        => SettingsUdi = settingsUdi;
+        : base(contentUdi, settingsUdi)
+    {
+    }
+
+    public BlockGridLayoutItem(Guid contentKey)
+        : base(contentKey)
+    {
+    }
+
+    public BlockGridLayoutItem(Guid contentKey, Guid settingsKey)
+        : base(contentKey, settingsKey)
+    {
+    }
 }

src/Umbraco.Core/Models/Blocks/BlockGridValue.cs

@@ -1,3 +1,5 @@
+using System.Text.Json.Serialization;
+
 namespace Umbraco.Cms.Core.Models.Blocks;
 
 /// <summary>
@@ -19,5 +21,6 @@ public BlockGridValue(IEnumerable<BlockGridLayoutItem> layouts)
         => Layout[PropertyEditorAlias] = layouts;
 
     /// <inheritdoc />
+    [JsonIgnore]
     public override string PropertyEditorAlias => Constants.PropertyEditors.Aliases.BlockGrid;
 }