V12/content type sorting (#555)

* Topographical sorting of compositions . (#542)

* move composition to first pass.

* Comment up the graph handler interface.

* Fix - don't report delete on clean for non-containers (as we don't actually delete them).

uSync.BackOffice/Services/uSyncService_Single.cs

@@ -12,6 +12,7 @@
 using uSync.BackOffice.SyncHandlers;
 using uSync.BackOffice.SyncHandlers.Interfaces;
 using uSync.Core;
+using uSync.Core.Dependency;
 
 using static Umbraco.Cms.Core.Constants;
 
@@ -332,9 +333,7 @@ private SyncHandlerOptions HandlerOptionsFromPaged(uSyncPagedImportOptions optio
         public IList<OrderedNodeInfo> LoadOrderedNodes(string folder)
         {
             var files = _syncFileService.GetFiles(folder, $"*.{_uSyncConfig.Settings.DefaultExtension}", true);
-
             var nodes = new List<OrderedNodeInfo>();
-
             foreach (var file in files)
             {
                 nodes.Add(new OrderedNodeInfo(file, _syncFileService.LoadXElement(file)));
@@ -345,6 +344,44 @@ public IList<OrderedNodeInfo> LoadOrderedNodes(string folder)
                 .ToList();
         }
 
+        public IList<OrderedNodeInfo> LoadOrderedNodes(ISyncHandler handler, string handlerFolder)
+        {
+            if (handler is not ISyncGraphableHandler graphableHandler)
+                return LoadOrderedNodes(handlerFolder);
+
+            var files = _syncFileService.GetFiles(handlerFolder, $"*.{_uSyncConfig.Settings.DefaultExtension}", true);
+            var nodes = new Dictionary<Guid, OrderedNodeInfo>();
+            var graph = new List<GraphEdge<Guid>>();
+
+            foreach (var file in files)
+            {
+                var node = _syncFileService.LoadXElement(file);
+                if (node == null) continue;
+
+                var key = node.GetKey();
+
+                nodes.Add(key, new OrderedNodeInfo(file, _syncFileService.LoadXElement(file)));
+
+                graph.AddRange(graphableHandler.GetGraphIds(node)
+                    .Select(x => GraphEdge.Create(key, x)));
+            }
+
+            var cleanGraph = graph.Where(x => x.Node != x.Edge).ToList();
+            var sortedList = nodes.Keys.TopologicalSort(cleanGraph);
+
+            if (sortedList == null)
+                return nodes.Values.OrderBy(x => x.Node.GetLevel()).ToList();
+
+            var results = new List<OrderedNodeInfo>();
+            foreach(var key in sortedList)
+            {
+                if (nodes.ContainsKey(key))
+                    results.Add(nodes[key]);
+            }
+            return results;
+
+        }
+
         /// <summary>
         ///  calculate the percentage progress we are making between a range. 
         /// </summary>

uSync.BackOffice/SyncHandlers/Handlers/ContentTypeHandler.cs

@@ -24,7 +24,7 @@ namespace uSync.BackOffice.SyncHandlers.Handlers
     /// </summary>
     [SyncHandler(uSyncConstants.Handlers.ContentTypeHandler, "DocTypes", "ContentTypes", uSyncConstants.Priorites.ContentTypes,
             IsTwoPass = true, Icon = "icon-item-arrangement", EntityType = UdiEntityType.DocumentType)]
-    public class ContentTypeHandler : SyncHandlerContainerBase<IContentType, IContentTypeService>, ISyncHandler, ISyncPostImportHandler,
+    public class ContentTypeHandler : SyncHandlerContainerBase<IContentType, IContentTypeService>, ISyncHandler, ISyncPostImportHandler, ISyncGraphableHandler,
         INotificationHandler<SavedNotification<IContentType>>,
         INotificationHandler<DeletedNotification<IContentType>>,
         INotificationHandler<MovedNotification<IContentType>>,

uSync.BackOffice/SyncHandlers/Handlers/MediaTypeHandler.cs

@@ -24,7 +24,7 @@ namespace uSync.BackOffice.SyncHandlers.Handlers
     /// </summary>
     [SyncHandler(uSyncConstants.Handlers.MediaTypeHandler, "Media Types", "MediaTypes", uSyncConstants.Priorites.MediaTypes,
         IsTwoPass = true, Icon = "icon-thumbnails", EntityType = UdiEntityType.MediaType)]
-    public class MediaTypeHandler : SyncHandlerContainerBase<IMediaType, IMediaTypeService>, ISyncHandler,
+    public class MediaTypeHandler : SyncHandlerContainerBase<IMediaType, IMediaTypeService>, ISyncHandler, ISyncGraphableHandler,
         INotificationHandler<SavedNotification<IMediaType>>,
         INotificationHandler<DeletedNotification<IMediaType>>,
         INotificationHandler<MovedNotification<IMediaType>>,

uSync.BackOffice/SyncHandlers/Handlers/MemberTypeHandler.cs

@@ -24,7 +24,7 @@ namespace uSync.BackOffice.SyncHandlers.Handlers
     /// </summary>
     [SyncHandler(uSyncConstants.Handlers.MemberTypeHandler, "Member Types", "MemberTypes", uSyncConstants.Priorites.MemberTypes,
         IsTwoPass = true, Icon = "icon-users", EntityType = UdiEntityType.MemberType)]
-    public class MemberTypeHandler : SyncHandlerContainerBase<IMemberType, IMemberTypeService>, ISyncHandler,
+    public class MemberTypeHandler : SyncHandlerContainerBase<IMemberType, IMemberTypeService>, ISyncHandler, ISyncGraphableHandler,
         INotificationHandler<SavedNotification<IMemberType>>,
         INotificationHandler<MovedNotification<IMemberType>>,
         INotificationHandler<DeletedNotification<IMemberType>>,

uSync.BackOffice/SyncHandlers/Interfaces/ISyncGraphableHandler.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Xml.Linq;
+
+namespace uSync.BackOffice.SyncHandlers
+{
+    /// <summary>
+    ///  graphable handler - handler can use a Topological graph
+    ///  to sort items into the most efficient order 
+    /// </summary>
+    public interface ISyncGraphableHandler
+    {
+        /// <summary>
+        ///  return
+        /// </summary>
+        /// <param name="node"></param>
+        /// <returns></returns>
+        public IEnumerable<Guid> GetGraphIds(XElement node);
+    }
+}

uSync.BackOffice/SyncHandlers/SyncHandlerBase.cs

@@ -188,30 +188,41 @@ protected override IEnumerable<IEntity> GetChildItems(IEntity parent)
 
         // almost everything does this - but languages can't so we need to 
         // let the language Handler override this. 
-
-        /// <summary>
-        ///  Get all child items beneath a given item
-        /// </summary>
-        virtual protected IEnumerable<IEntity> GetChildItems(int parent)
+
+        virtual protected IEnumerable<IEntity> GetChildItems(int parent, UmbracoObjectTypes objectType)
         {
-            if (this.itemObjectType == UmbracoObjectTypes.Unknown)
-                return Enumerable.Empty<IEntity>();
-
-            var cacheKey = $"{GetCacheKeyBase()}_parent_{parent}";
+            var cacheKey = $"{GetCacheKeyBase()}_parent_{parent}_{objectType}";
 
             return runtimeCache.GetCacheItem(cacheKey, () =>
             {
+                // logger.LogDebug("Cache miss [{key}]", cacheKey);
                 if (parent == -1)
                 {
-                    return entityService.GetChildren(parent, this.itemObjectType);
+                    return entityService.GetChildren(parent, objectType);
                 }
                 else
                 {
                     // If you ask for the type then you get more info, and there is extra db calls to 
                     // load it, so GetChildren without the object type is quicker. 
-                    return entityService.GetChildren(parent);
+
+                    // but we need to know that we only get our type so we then filter.
+                    var guidType = ObjectTypes.GetGuid(objectType);
+                    return entityService.GetChildren(parent).Where(x => x.NodeObjectType == guidType);
                 }
             }, null);
+
+        }
+
+        /// <summary>
+        ///  Get all child items beneath a given item
+        /// </summary>
+        virtual protected IEnumerable<IEntity> GetChildItems(int parent)
+        {
+            if (this.itemObjectType == UmbracoObjectTypes.Unknown)
+                return Enumerable.Empty<IEntity>();
+
+            return GetChildItems(parent, this.itemObjectType);
+
         }
 
         /// <summary>

uSync.BackOffice/SyncHandlers/SyncHandlerContainerBase.cs

@@ -1,10 +1,17 @@
-using Microsoft.Extensions.Logging;
+using Microsoft.CodeAnalysis.Operations;
+using Microsoft.Extensions.Logging;
 
+using NPoco.RowMappers;
+
+using NUglify.JavaScript.Syntax;
+
+using System;
 using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Xml.Linq;
 
+using Umbraco.Cms.Core;
 using Umbraco.Cms.Core.Cache;
 using Umbraco.Cms.Core.Models;
 using Umbraco.Cms.Core.Models.Entities;
@@ -16,6 +23,7 @@
 using uSync.BackOffice.Configuration;
 using uSync.BackOffice.Services;
 using uSync.Core;
+using uSync.Core.Dependency;
 using uSync.Core.Serialization;
 
 namespace uSync.BackOffice.SyncHandlers
@@ -58,9 +66,10 @@ protected SyncHandlerContainerBase(
         protected IEnumerable<uSyncAction> CleanFolders(string folder, int parent)
         {
             var actions = new List<uSyncAction>();
-            var folders = GetFolders(parent);
+            var folders = GetChildItems(parent, this.itemContainerType);
             foreach (var fdlr in folders)
             {
+                logger.LogDebug("Checking Container: {folder} for any childItems [{type}]", fdlr.Id, fdlr?.GetType()?.Name ?? "Unknown");
                 actions.AddRange(CleanFolders(folder, fdlr.Id));
 
                 if (!HasChildren(fdlr))
@@ -69,7 +78,11 @@ protected IEnumerable<uSyncAction> CleanFolders(string folder, int parent)
                     var name = fdlr.Id.ToString();
                     if (fdlr is IEntitySlim slim)
                     {
+                        // if this item isn't an container type, don't delete. 
+                        if (ObjectTypes.GetUmbracoObjectType(slim.NodeObjectType) != this.itemContainerType) continue;
+
                         name = slim.Name;
+                        logger.LogDebug("Folder has no children {name} {type}", name, slim.NodeObjectType);
                     }
 
                     actions.Add(uSyncAction.SetAction(true, name, typeof(EntityContainer).Name, ChangeType.Delete, "Empty Container"));
@@ -80,6 +93,10 @@ protected IEnumerable<uSyncAction> CleanFolders(string folder, int parent)
             return actions;
         }
 
+        private bool IsContainer(Guid guid)
+            => guid == Constants.ObjectTypes.DataTypeContainer
+            || guid == Constants.ObjectTypes.MediaTypeContainer
+            || guid == Constants.ObjectTypes.DocumentTypeContainer;
 
         /// <summary>
         /// delete a container
@@ -113,13 +130,13 @@ protected IEnumerable<uSyncAction> UpdateFolder(int folderId, string folder, Han
             }
 
             var actions = new List<uSyncAction>();
-            var folders = GetFolders(folderId);
+            var folders = GetChildItems(folderId, this.itemContainerType);
             foreach (var fdlr in folders)
             {
                 actions.AddRange(UpdateFolder(fdlr.Id, folder, config));
             }
 
-            var items = GetChildItems(folderId);
+            var items = GetChildItems(folderId, this.itemObjectType);
             foreach (var item in items)
             {
                 var obj = GetFromService(item.Id);
@@ -149,24 +166,31 @@ protected IEnumerable<uSyncAction> UpdateFolder(int folderId, string folder, Han
         /// <param name="notification"></param>
         public virtual void Handle(EntityContainerSavedNotification notification)
         {
-            if (_mutexService.IsPaused) return;
+            // we are not handling saves, we assume a rename, is just that
+            // if a rename does happen as part of a save then its only 
+            // going to be part of an import, and that will rename the rest of the items
+            // 
+            // performance wise this is a big improvement, for very little/no impact
 
-            ProcessContainerChanges(notification.SavedEntities);
+            // if (!ShouldProcessEvent()) return;
+            // logger.LogDebug("Container(s) saved [{count}]", notification.SavedEntities.Count());
+            // ProcessContainerChanges(notification.SavedEntities);
         }
 
         /// <summary>
         ///  handler renames of containers.
         /// </summary>
         public virtual void Handle(EntityContainerRenamedNotification notification)
         {
-            if (_mutexService.IsPaused) return;
+            if (!ShouldProcessEvent()) return;
             ProcessContainerChanges(notification.Entities);
         }
 
         private void ProcessContainerChanges(IEnumerable<EntityContainer> containers)
         {
             foreach (var folder in containers)
             {
+                logger.LogDebug("Processing container change : {name} [{id}]", folder.Name, folder.Id);
                 if (folder.ContainedObjectType == this.itemObjectType.GetGuid())
                 {
                     UpdateFolder(folder.Id, Path.Combine(rootFolder, this.DefaultFolder), DefaultConfig);
@@ -194,5 +218,86 @@ protected override bool DoItemsMatch(XElement node, TObject item)
             if (base.DoItemsMatch(node, item)) return true;
             return node.GetAlias().InvariantEquals(GetItemAlias(item));
         }
+
+        /// <summary>
+        ///  for containers, we are building a dependency graph.
+        /// </summary>
+        protected override IList<LeveledFile> GetLevelOrderedFiles(string folder, IList<uSyncAction> actions)
+        {
+            var files = syncFileService.GetFiles(folder, $"*.{this.uSyncConfig.Settings.DefaultExtension}");
+
+            var nodes = new Dictionary<Guid, LeveledFile>();
+            var graph = new List<GraphEdge<Guid>>();
+
+            foreach (var file in files)
+            {
+                var node = LoadNode(file);
+                if (node == null) continue;
+
+                var key = node.GetKey();
+                nodes.Add(key, new LeveledFile
+                {
+                    Alias = node.GetAlias(),
+                    File = file,
+                    Level = node.GetLevel(),
+                });
+
+                // you can have circular dependencies in structure :( 
+                // graph.AddRange(GetStructure(node).Select(x => GraphEdge.Create(key, x)));
+
+                graph.AddRange(GetCompositions(node).Select(x => GraphEdge.Create(key, x)));
+            }
+
+            var cleanGraph = graph.Where(x => x.Node != x.Edge).ToList();
+            var sortedList = nodes.Keys.TopologicalSort(cleanGraph);
+
+            if (sortedList == null)
+                return nodes.Values.OrderBy(x => x.Level).ToList();
+
+            var result = new List<LeveledFile>();
+            foreach(var key in sortedList)
+            {
+                if (nodes.ContainsKey(key))
+                    result.Add(nodes[key]);
+            }
+            return result;
+
+        }
+
+        public IEnumerable<Guid> GetGraphIds(XElement node)
+        {
+            return GetCompositions(node);
+        }
+
+        private IEnumerable<Guid> GetStructure(XElement node)
+        {
+
+            var structure = node.Element("Structure");
+            if (structure == null) return Enumerable.Empty<Guid>();
+
+            return GetKeys(structure);
+        }
+
+        private IEnumerable<Guid> GetCompositions(XElement node)
+        {
+            var compositionNode = node.Element("Info")?.Element("Compositions");
+            if (compositionNode == null) return Enumerable.Empty<Guid>();
+
+            return GetKeys(compositionNode);
+        }
+
+        private IEnumerable<Guid> GetKeys(XElement node)
+        {
+            if (node != null)
+            {
+                foreach (var item in node.Elements())
+                {
+                    var key = item.Attribute("Key").ValueOrDefault(Guid.Empty);
+                    if (key == Guid.Empty) continue;
+
+                    yield return key;
+                }
+            }
+        }
     }
 }

uSync.BackOffice/SyncHandlers/SyncHandlerLevelBase.cs

@@ -130,7 +130,7 @@ protected override IEnumerable<uSyncAction> ImportFolder(string folder, HandlerS
         /// <summary>
         ///  Get all the files in a folder and return them sorted by their level 
         /// </summary>
-        private IList<LeveledFile> GetLevelOrderedFiles(string folder, IList<uSyncAction> actions)
+        protected virtual IList<LeveledFile> GetLevelOrderedFiles(string folder, IList<uSyncAction> actions)
         {
             List<LeveledFile> nodes = new List<LeveledFile>();
 
@@ -162,14 +162,14 @@ private IList<LeveledFile> GetLevelOrderedFiles(string folder, IList<uSyncAction
             return nodes.OrderBy(x => x.Level).ToList();
         }
 
-        private class LeveledFile
+        protected class LeveledFile
         {
             public string Alias { get; set; }
             public int Level { get; set; }
             public string File { get; set; }
         }
 
-        private XElement LoadNode(string path)
+        protected XElement LoadNode(string path)
         {
             syncFileService.EnsureFileExists(path);