diff --git a/uSync.BackOffice/Services/uSyncService_Single.cs b/uSync.BackOffice/Services/uSyncService_Single.cs index 6113ee54..0ccfe59e 100644 --- a/uSync.BackOffice/Services/uSyncService_Single.cs +++ b/uSync.BackOffice/Services/uSyncService_Single.cs @@ -12,6 +12,7 @@ using uSync.BackOffice.SyncHandlers; using uSync.BackOffice.SyncHandlers.Interfaces; using uSync.Core; +using uSync.Core.Dependency; using static Umbraco.Cms.Core.Constants; @@ -332,9 +333,7 @@ private SyncHandlerOptions HandlerOptionsFromPaged(uSyncPagedImportOptions optio public IList LoadOrderedNodes(string folder) { var files = _syncFileService.GetFiles(folder, $"*.{_uSyncConfig.Settings.DefaultExtension}", true); - var nodes = new List(); - foreach (var file in files) { nodes.Add(new OrderedNodeInfo(file, _syncFileService.LoadXElement(file))); @@ -345,6 +344,44 @@ public IList LoadOrderedNodes(string folder) .ToList(); } + public IList LoadOrderedNodes(ISyncHandler handler, string handlerFolder) + { + if (handler is not ISyncGraphableHandler graphableHandler) + return LoadOrderedNodes(handlerFolder); + + var files = _syncFileService.GetFiles(handlerFolder, $"*.{_uSyncConfig.Settings.DefaultExtension}", true); + var nodes = new Dictionary(); + var graph = new List>(); + + foreach (var file in files) + { + var node = _syncFileService.LoadXElement(file); + if (node == null) continue; + + var key = node.GetKey(); + + nodes.Add(key, new OrderedNodeInfo(file, _syncFileService.LoadXElement(file))); + + graph.AddRange(graphableHandler.GetGraphIds(node) + .Select(x => GraphEdge.Create(key, x))); + } + + var cleanGraph = graph.Where(x => x.Node != x.Edge).ToList(); + var sortedList = nodes.Keys.TopologicalSort(cleanGraph); + + if (sortedList == null) + return nodes.Values.OrderBy(x => x.Node.GetLevel()).ToList(); + + var results = new List(); + foreach(var key in sortedList) + { + if (nodes.ContainsKey(key)) + results.Add(nodes[key]); + } + return results; + + } + /// /// calculate the percentage progress we are making between a range. /// diff --git a/uSync.BackOffice/SyncHandlers/Handlers/ContentTypeHandler.cs b/uSync.BackOffice/SyncHandlers/Handlers/ContentTypeHandler.cs index 15300956..857768a7 100644 --- a/uSync.BackOffice/SyncHandlers/Handlers/ContentTypeHandler.cs +++ b/uSync.BackOffice/SyncHandlers/Handlers/ContentTypeHandler.cs @@ -24,7 +24,7 @@ namespace uSync.BackOffice.SyncHandlers.Handlers /// [SyncHandler(uSyncConstants.Handlers.ContentTypeHandler, "DocTypes", "ContentTypes", uSyncConstants.Priorites.ContentTypes, IsTwoPass = true, Icon = "icon-item-arrangement", EntityType = UdiEntityType.DocumentType)] - public class ContentTypeHandler : SyncHandlerContainerBase, ISyncHandler, ISyncPostImportHandler, + public class ContentTypeHandler : SyncHandlerContainerBase, ISyncHandler, ISyncPostImportHandler, ISyncGraphableHandler, INotificationHandler>, INotificationHandler>, INotificationHandler>, diff --git a/uSync.BackOffice/SyncHandlers/Handlers/MediaTypeHandler.cs b/uSync.BackOffice/SyncHandlers/Handlers/MediaTypeHandler.cs index 483bce87..70de821d 100644 --- a/uSync.BackOffice/SyncHandlers/Handlers/MediaTypeHandler.cs +++ b/uSync.BackOffice/SyncHandlers/Handlers/MediaTypeHandler.cs @@ -24,7 +24,7 @@ namespace uSync.BackOffice.SyncHandlers.Handlers /// [SyncHandler(uSyncConstants.Handlers.MediaTypeHandler, "Media Types", "MediaTypes", uSyncConstants.Priorites.MediaTypes, IsTwoPass = true, Icon = "icon-thumbnails", EntityType = UdiEntityType.MediaType)] - public class MediaTypeHandler : SyncHandlerContainerBase, ISyncHandler, + public class MediaTypeHandler : SyncHandlerContainerBase, ISyncHandler, ISyncGraphableHandler, INotificationHandler>, INotificationHandler>, INotificationHandler>, diff --git a/uSync.BackOffice/SyncHandlers/Handlers/MemberTypeHandler.cs b/uSync.BackOffice/SyncHandlers/Handlers/MemberTypeHandler.cs index 9f3bcf8f..01fc0e6b 100644 --- a/uSync.BackOffice/SyncHandlers/Handlers/MemberTypeHandler.cs +++ b/uSync.BackOffice/SyncHandlers/Handlers/MemberTypeHandler.cs @@ -24,7 +24,7 @@ namespace uSync.BackOffice.SyncHandlers.Handlers /// [SyncHandler(uSyncConstants.Handlers.MemberTypeHandler, "Member Types", "MemberTypes", uSyncConstants.Priorites.MemberTypes, IsTwoPass = true, Icon = "icon-users", EntityType = UdiEntityType.MemberType)] - public class MemberTypeHandler : SyncHandlerContainerBase, ISyncHandler, + public class MemberTypeHandler : SyncHandlerContainerBase, ISyncHandler, ISyncGraphableHandler, INotificationHandler>, INotificationHandler>, INotificationHandler>, diff --git a/uSync.BackOffice/SyncHandlers/Interfaces/ISyncGraphableHandler.cs b/uSync.BackOffice/SyncHandlers/Interfaces/ISyncGraphableHandler.cs new file mode 100644 index 00000000..26738023 --- /dev/null +++ b/uSync.BackOffice/SyncHandlers/Interfaces/ISyncGraphableHandler.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Xml.Linq; + +namespace uSync.BackOffice.SyncHandlers +{ + /// + /// graphable handler - handler can use a Topological graph + /// to sort items into the most efficient order + /// + public interface ISyncGraphableHandler + { + /// + /// return + /// + /// + /// + public IEnumerable GetGraphIds(XElement node); + } +} diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs index a9a10092..dc85067b 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs @@ -188,30 +188,41 @@ protected override IEnumerable GetChildItems(IEntity parent) // almost everything does this - but languages can't so we need to // let the language Handler override this. - - /// - /// Get all child items beneath a given item - /// - virtual protected IEnumerable GetChildItems(int parent) + + virtual protected IEnumerable GetChildItems(int parent, UmbracoObjectTypes objectType) { - if (this.itemObjectType == UmbracoObjectTypes.Unknown) - return Enumerable.Empty(); - - var cacheKey = $"{GetCacheKeyBase()}_parent_{parent}"; + var cacheKey = $"{GetCacheKeyBase()}_parent_{parent}_{objectType}"; return runtimeCache.GetCacheItem(cacheKey, () => { + // logger.LogDebug("Cache miss [{key}]", cacheKey); if (parent == -1) { - return entityService.GetChildren(parent, this.itemObjectType); + return entityService.GetChildren(parent, objectType); } else { // If you ask for the type then you get more info, and there is extra db calls to // load it, so GetChildren without the object type is quicker. - return entityService.GetChildren(parent); + + // but we need to know that we only get our type so we then filter. + var guidType = ObjectTypes.GetGuid(objectType); + return entityService.GetChildren(parent).Where(x => x.NodeObjectType == guidType); } }, null); + + } + + /// + /// Get all child items beneath a given item + /// + virtual protected IEnumerable GetChildItems(int parent) + { + if (this.itemObjectType == UmbracoObjectTypes.Unknown) + return Enumerable.Empty(); + + return GetChildItems(parent, this.itemObjectType); + } /// diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerContainerBase.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerContainerBase.cs index 4cf001da..39922632 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerContainerBase.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerContainerBase.cs @@ -1,10 +1,17 @@ -using Microsoft.Extensions.Logging; +using Microsoft.CodeAnalysis.Operations; +using Microsoft.Extensions.Logging; +using NPoco.RowMappers; + +using NUglify.JavaScript.Syntax; + +using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Xml.Linq; +using Umbraco.Cms.Core; using Umbraco.Cms.Core.Cache; using Umbraco.Cms.Core.Models; using Umbraco.Cms.Core.Models.Entities; @@ -16,6 +23,7 @@ using uSync.BackOffice.Configuration; using uSync.BackOffice.Services; using uSync.Core; +using uSync.Core.Dependency; using uSync.Core.Serialization; namespace uSync.BackOffice.SyncHandlers @@ -58,9 +66,10 @@ protected SyncHandlerContainerBase( protected IEnumerable CleanFolders(string folder, int parent) { var actions = new List(); - var folders = GetFolders(parent); + var folders = GetChildItems(parent, this.itemContainerType); foreach (var fdlr in folders) { + logger.LogDebug("Checking Container: {folder} for any childItems [{type}]", fdlr.Id, fdlr?.GetType()?.Name ?? "Unknown"); actions.AddRange(CleanFolders(folder, fdlr.Id)); if (!HasChildren(fdlr)) @@ -69,7 +78,11 @@ protected IEnumerable CleanFolders(string folder, int parent) var name = fdlr.Id.ToString(); if (fdlr is IEntitySlim slim) { + // if this item isn't an container type, don't delete. + if (ObjectTypes.GetUmbracoObjectType(slim.NodeObjectType) != this.itemContainerType) continue; + name = slim.Name; + logger.LogDebug("Folder has no children {name} {type}", name, slim.NodeObjectType); } actions.Add(uSyncAction.SetAction(true, name, typeof(EntityContainer).Name, ChangeType.Delete, "Empty Container")); @@ -80,6 +93,10 @@ protected IEnumerable CleanFolders(string folder, int parent) return actions; } + private bool IsContainer(Guid guid) + => guid == Constants.ObjectTypes.DataTypeContainer + || guid == Constants.ObjectTypes.MediaTypeContainer + || guid == Constants.ObjectTypes.DocumentTypeContainer; /// /// delete a container @@ -113,13 +130,13 @@ protected IEnumerable UpdateFolder(int folderId, string folder, Han } var actions = new List(); - var folders = GetFolders(folderId); + var folders = GetChildItems(folderId, this.itemContainerType); foreach (var fdlr in folders) { actions.AddRange(UpdateFolder(fdlr.Id, folder, config)); } - var items = GetChildItems(folderId); + var items = GetChildItems(folderId, this.itemObjectType); foreach (var item in items) { var obj = GetFromService(item.Id); @@ -149,9 +166,15 @@ protected IEnumerable UpdateFolder(int folderId, string folder, Han /// public virtual void Handle(EntityContainerSavedNotification notification) { - if (_mutexService.IsPaused) return; + // we are not handling saves, we assume a rename, is just that + // if a rename does happen as part of a save then its only + // going to be part of an import, and that will rename the rest of the items + // + // performance wise this is a big improvement, for very little/no impact - ProcessContainerChanges(notification.SavedEntities); + // if (!ShouldProcessEvent()) return; + // logger.LogDebug("Container(s) saved [{count}]", notification.SavedEntities.Count()); + // ProcessContainerChanges(notification.SavedEntities); } /// @@ -159,7 +182,7 @@ public virtual void Handle(EntityContainerSavedNotification notification) /// public virtual void Handle(EntityContainerRenamedNotification notification) { - if (_mutexService.IsPaused) return; + if (!ShouldProcessEvent()) return; ProcessContainerChanges(notification.Entities); } @@ -167,6 +190,7 @@ private void ProcessContainerChanges(IEnumerable containers) { foreach (var folder in containers) { + logger.LogDebug("Processing container change : {name} [{id}]", folder.Name, folder.Id); if (folder.ContainedObjectType == this.itemObjectType.GetGuid()) { UpdateFolder(folder.Id, Path.Combine(rootFolder, this.DefaultFolder), DefaultConfig); @@ -194,5 +218,86 @@ protected override bool DoItemsMatch(XElement node, TObject item) if (base.DoItemsMatch(node, item)) return true; return node.GetAlias().InvariantEquals(GetItemAlias(item)); } + + /// + /// for containers, we are building a dependency graph. + /// + protected override IList GetLevelOrderedFiles(string folder, IList actions) + { + var files = syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}"); + + var nodes = new Dictionary(); + var graph = new List>(); + + foreach (var file in files) + { + var node = LoadNode(file); + if (node == null) continue; + + var key = node.GetKey(); + nodes.Add(key, new LeveledFile + { + Alias = node.GetAlias(), + File = file, + Level = node.GetLevel(), + }); + + // you can have circular dependencies in structure :( + // graph.AddRange(GetStructure(node).Select(x => GraphEdge.Create(key, x))); + + graph.AddRange(GetCompositions(node).Select(x => GraphEdge.Create(key, x))); + } + + var cleanGraph = graph.Where(x => x.Node != x.Edge).ToList(); + var sortedList = nodes.Keys.TopologicalSort(cleanGraph); + + if (sortedList == null) + return nodes.Values.OrderBy(x => x.Level).ToList(); + + var result = new List(); + foreach(var key in sortedList) + { + if (nodes.ContainsKey(key)) + result.Add(nodes[key]); + } + return result; + + } + + public IEnumerable GetGraphIds(XElement node) + { + return GetCompositions(node); + } + + private IEnumerable GetStructure(XElement node) + { + + var structure = node.Element("Structure"); + if (structure == null) return Enumerable.Empty(); + + return GetKeys(structure); + } + + private IEnumerable GetCompositions(XElement node) + { + var compositionNode = node.Element("Info")?.Element("Compositions"); + if (compositionNode == null) return Enumerable.Empty(); + + return GetKeys(compositionNode); + } + + private IEnumerable GetKeys(XElement node) + { + if (node != null) + { + foreach (var item in node.Elements()) + { + var key = item.Attribute("Key").ValueOrDefault(Guid.Empty); + if (key == Guid.Empty) continue; + + yield return key; + } + } + } } } diff --git a/uSync.BackOffice/SyncHandlers/SyncHandlerLevelBase.cs b/uSync.BackOffice/SyncHandlers/SyncHandlerLevelBase.cs index 3d274fc6..3bb89254 100644 --- a/uSync.BackOffice/SyncHandlers/SyncHandlerLevelBase.cs +++ b/uSync.BackOffice/SyncHandlers/SyncHandlerLevelBase.cs @@ -130,7 +130,7 @@ protected override IEnumerable ImportFolder(string folder, HandlerS /// /// Get all the files in a folder and return them sorted by their level /// - private IList GetLevelOrderedFiles(string folder, IList actions) + protected virtual IList GetLevelOrderedFiles(string folder, IList actions) { List nodes = new List(); @@ -162,14 +162,14 @@ private IList GetLevelOrderedFiles(string folder, IList x.Level).ToList(); } - private class LeveledFile + protected class LeveledFile { public string Alias { get; set; } public int Level { get; set; } public string File { get; set; } } - private XElement LoadNode(string path) + protected XElement LoadNode(string path) { syncFileService.EnsureFileExists(path); diff --git a/uSync.Core/Dependency/DependencyGraph.cs b/uSync.Core/Dependency/DependencyGraph.cs new file mode 100644 index 00000000..f1e25443 --- /dev/null +++ b/uSync.Core/Dependency/DependencyGraph.cs @@ -0,0 +1,64 @@ +using System; +using System.Collections.Generic; +using System.Linq; + +namespace uSync.Core.Dependency; + +/// +/// builds a graph of dependencies, so things can be installed in order. +/// +public static class DependencyGraph +{ + public static List TopologicalSort(this ICollection nodes, ICollection> edges) + where T : IEquatable + { + var sortedList = new List(); + + // all items where they don't have a dependency on themselves. + var queue = new Queue( + nodes.Where(x => edges.All(e => e.Node.Equals(x) == false))); + + + while (queue.Any()) + { + // remove this item add it to the queue. + var next = queue.Dequeue(); + sortedList.Add(next); + + // look for any edges for this queue. + foreach(var edge in edges.Where(e => e.Edge.Equals(next)).ToList()) + { + var dependency = edge.Node; + edges.Remove(edge); + + if (edges.All(x => x.Node.Equals(dependency) == false)) + { + queue.Enqueue(dependency); + } + } + } + + if (edges.Any()) + { + return null; + } + + return sortedList; + + } +} + +public class GraphEdge + where T : IEquatable +{ + public T Node { get; set; } + public T Edge { get; set; } + +} + +public class GraphEdge +{ + public static GraphEdge Create(T node, T edge) where T : IEquatable + => new GraphEdge { Node = node, Edge = edge }; +} + diff --git a/uSync.Core/Serialization/Serializers/ContentTypeSerializer.cs b/uSync.Core/Serialization/Serializers/ContentTypeSerializer.cs index 0baf4e35..d888c71c 100644 --- a/uSync.Core/Serialization/Serializers/ContentTypeSerializer.cs +++ b/uSync.Core/Serialization/Serializers/ContentTypeSerializer.cs @@ -125,6 +125,9 @@ protected override SyncAttempt DeserializeCore(XElement node, Sync // templates details.AddRange(DeserializeTemplates(item, node, options)); + // compositions + details.AddRange(DeserializeCompositions(item, node)); + return DeserializedResult(item, details, options); } @@ -151,7 +154,11 @@ public override SyncAttempt DeserializeSecondPass(IContentType ite SetSafeAliasValue(item, node, false); + // we can do this here, hopefully its not needed + // as we graph sort at the start, + // so it should say 'no changes' on a second pass. details.AddRange(DeserializeCompositions(item, node)); + details.AddRange(DeserializeStructure(item, node)); // When doing this reflectiony - it doesn't set is dirty. diff --git a/uSync.Core/Serialization/Serializers/DataTypeSerializer.cs b/uSync.Core/Serialization/Serializers/DataTypeSerializer.cs index f3306582..07aeb7a6 100644 --- a/uSync.Core/Serialization/Serializers/DataTypeSerializer.cs +++ b/uSync.Core/Serialization/Serializers/DataTypeSerializer.cs @@ -306,7 +306,7 @@ public override IDataType FindItem(string alias) => _dataTypeService.GetDataType(alias); protected override EntityContainer FindContainer(Guid key) - => _dataTypeService.GetContainer(key); + => key == Guid.Empty ? null : _dataTypeService.GetContainer(key); protected override IEnumerable FindContainers(string folder, int level) => _dataTypeService.GetContainers(folder, level); diff --git a/uSync.Core/Serialization/SyncContainerSerializerBase.cs b/uSync.Core/Serialization/SyncContainerSerializerBase.cs index 308e0a45..63bb143a 100644 --- a/uSync.Core/Serialization/SyncContainerSerializerBase.cs +++ b/uSync.Core/Serialization/SyncContainerSerializerBase.cs @@ -63,7 +63,7 @@ protected override Attempt FindOrCreate(XElement node) var folderKey = folder.Attribute(uSyncConstants.Xml.Key).ValueOrDefault(Guid.Empty); - logger.LogDebug("Searching for Parent by folder {folderKey}", folderKey); + logger.LogDebug("Searching for Parent by folder {folderKey} {folderValue}", folderKey, folder.Value); var container = FindFolder(folderKey, folder.Value); if (container != null) diff --git a/uSync.Tests/Extensions/SortingTests.cs b/uSync.Tests/Extensions/SortingTests.cs new file mode 100644 index 00000000..a8a05e3b --- /dev/null +++ b/uSync.Tests/Extensions/SortingTests.cs @@ -0,0 +1,89 @@ +using System; +using System.Collections.Generic; + +using NUnit.Framework; + +using Umbraco.Cms.Core.WebAssets; + +using uSync.Core.Dependency; + +namespace uSync.Tests.Extensions; + +[TestFixture] +public class SortingTests +{ + private HashSet _nodes = new HashSet + { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 + }; + + private List _guidNodes = new List + { + Guid.Parse("{5E37F691-FF91-45DA-9F53-8641FD9FE233}"), // 0 + Guid.Parse("{5A35701C-349C-4AAE-BBCE-964B5C196989}"), // 1 + Guid.Parse("{C70BE6CF-4923-4E2B-8742-B90FB7BBAFCB}"), // 2 + Guid.Parse("{DE77C37C-DBC7-4806-8B0F-332BC38A87A4}"), // 3 + Guid.Parse("{3C90BAD8-09F2-486E-BAA7-474F0D423C4D}") // 4 + }; + + [Test] + public void GraphSortGuidNodes() + { + var graph = new List>(); + + graph.Add(GraphEdge.Create(_guidNodes[2], _guidNodes[1])); + graph.Add(GraphEdge.Create(_guidNodes[2], _guidNodes[4])); + + var result = _guidNodes.TopologicalSort(graph); + + var expected = new List + { + Guid.Parse("{5E37F691-FF91-45DA-9F53-8641FD9FE233}"), // 0 + Guid.Parse("{5A35701C-349C-4AAE-BBCE-964B5C196989}"), // 1 + Guid.Parse("{DE77C37C-DBC7-4806-8B0F-332BC38A87A4}"), // 3 + Guid.Parse("{3C90BAD8-09F2-486E-BAA7-474F0D423C4D}"), // 4 + Guid.Parse("{C70BE6CF-4923-4E2B-8742-B90FB7BBAFCB}"), // 2 + }; + + Assert.AreEqual(expected, result); + } + + [Test] + public void GraphSortNodes() + { + var graph = new HashSet>( + new[] + { + GraphEdge.Create(2,4), + GraphEdge.Create(4,7), + GraphEdge.Create(5,8), + GraphEdge.Create(6,9), + }); + + + var result = _nodes.TopologicalSort(graph); + var expected = new List { + 1, 3, 7, 8, 9, 10, 4, 5, 6, 2 + }; + + Assert.AreEqual(expected, result); + } + + [Test] + public void CircularDependencyReturnsNull() + { + var graph = new HashSet>( + new[] + { + GraphEdge.Create(2,4), + GraphEdge.Create(4,7), + GraphEdge.Create(5,8), + GraphEdge.Create(6,9), + GraphEdge.Create(7,2), // 7 can't depend on 2 and it depends on 4 which depends on 7 + }); + + var result = _nodes.TopologicalSort(graph); + + Assert.IsNull(result); + } +}