Merge pull request #9859 from nzdev/v8/bugfix/fix-cold-boot

Reduce cold boot times by loading content and media only once on cold boot

src/Umbraco.Core/Sync/DatabaseServerMessenger.cs

@@ -28,7 +28,7 @@ namespace Umbraco.Core.Sync
     // but only processes instructions coming from remote servers,
     // thus ensuring that instructions run only once
     //
-    public class DatabaseServerMessenger : ServerMessengerBase
+    public class DatabaseServerMessenger : ServerMessengerBase, ISyncBootStateAccessor
     {
         private readonly IRuntimeState _runtime;
         private readonly ManualResetEvent _syncIdle;
@@ -172,15 +172,39 @@ private void Initialize(IUmbracoDatabase database)
             lock (_locko)
             {
                 if (_released) return;
+                var coldboot = IsColdBoot(database);
 
-                var coldboot = false;
-                if (_lastId < 0) // never synced before
+                if (coldboot)
                 {
-                    // we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new
-                    // server and it will need to rebuild it's own caches, eg Lucene or the xml cache file.
-                    Logger.Warn<DatabaseServerMessenger>("No last synced Id found, this generally means this is a new server/install."
-                        + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in"
-                        + " the database and maintain cache updates based on that Id.");
+                    // go get the last id in the db and store it
+                    // note: do it BEFORE initializing otherwise some instructions might get lost
+                    // when doing it before, some instructions might run twice - not an issue
+                    var maxId = database.ExecuteScalar<int>("SELECT MAX(id) FROM umbracoCacheInstruction");
+
+                    //if there is a max currently, or if we've never synced
+                    if (maxId > 0 || _lastId < 0)
+                        SaveLastSynced(maxId);
+
+                    // execute initializing callbacks
+                    if (Options.InitializingCallbacks != null)
+                        foreach (var callback in Options.InitializingCallbacks)
+                            callback();
+                }
+
+                _initialized = true;
+            }
+        }
+
+        private bool IsColdBoot(IUmbracoDatabase database)
+        {
+            var coldboot = false;
+            if (_lastId < 0) // never synced before
+            {
+                // we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new
+                // server and it will need to rebuild it's own caches, eg Lucene or the xml cache file.
+                Logger.Warn<DatabaseServerMessenger>("No last synced Id found, this generally means this is a new server/install."
+                    + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in"
+                    + " the database and maintain cache updates based on that Id.");
 
                     coldboot = true;
                 }
@@ -198,29 +222,11 @@ private void Initialize(IUmbracoDatabase database)
                             + " to the latest found in the database and maintain cache updates based on that Id.",
                             count, Options.MaxProcessingInstructionCount);
 
-                        coldboot = true;
-                    }
-                }
-
-                if (coldboot)
-                {
-                    // go get the last id in the db and store it
-                    // note: do it BEFORE initializing otherwise some instructions might get lost
-                    // when doing it before, some instructions might run twice - not an issue
-                    var maxId = database.ExecuteScalar<int>("SELECT MAX(id) FROM umbracoCacheInstruction");
-
-                    //if there is a max currently, or if we've never synced
-                    if (maxId > 0 || _lastId < 0)
-                        SaveLastSynced(maxId);
-
-                    // execute initializing callbacks
-                    if (Options.InitializingCallbacks != null)
-                        foreach (var callback in Options.InitializingCallbacks)
-                            callback();
+                    coldboot = true;
                 }
-
-                _initialized = true;
             }
+
+            return coldboot;
         }
 
         /// <summary>
@@ -548,6 +554,30 @@ private string GetDistCacheFilePath(IGlobalSettings globalSettings)
 
         #endregion
 
+        public SyncBootState GetSyncBootState()
+        {
+            try
+            {
+                ReadLastSynced(); // get _lastId
+                using (var scope = ScopeProvider.CreateScope())
+                {
+                    EnsureInstructions(scope.Database);
+                    bool isColdBoot = IsColdBoot(scope.Database);
+
+                    if (isColdBoot)
+                    {
+                        return SyncBootState.ColdBoot;
+                    }
+                    return SyncBootState.HasSyncState;
+                }
+            }
+            catch(Exception ex)
+            {
+                Logger.Warn<DatabaseServerMessenger>("Error determining Sync Boot State", ex);
+                return SyncBootState.Unknown;
+            }
+        }
+
         #region Notify refreshers
 
         private static ICacheRefresher GetRefresher(Guid id)

src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs

@@ -0,0 +1,20 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Umbraco.Core.Sync
+{
+    /// <summary>
+    /// Retrieve the state of the sync service
+    /// </summary>
+    public interface ISyncBootStateAccessor
+    {
+        /// <summary>
+        /// Get the boot state
+        /// </summary>
+        /// <returns></returns>
+        SyncBootState GetSyncBootState();
+    }
+}

src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs

@@ -0,0 +1,19 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Umbraco.Core.Sync
+{
+    /// <summary>
+    /// Boot state implementation for when umbraco is not in the run state
+    /// </summary>
+    public class NonRuntimeLevelBootStateAccessor : ISyncBootStateAccessor
+    {
+        public SyncBootState GetSyncBootState()
+        {
+            return SyncBootState.Unknown;
+        }
+    }
+}

src/Umbraco.Core/Sync/SyncBootState.cs

@@ -0,0 +1,24 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Umbraco.Core.Sync
+{
+    public enum SyncBootState
+    {
+        /// <summary>
+        /// Unknown state. Treat as HasSyncState
+        /// </summary>
+        Unknown = 0,
+        /// <summary>
+        /// Cold boot. No Sync state
+        /// </summary>
+        ColdBoot = 1,
+        /// <summary>
+        /// Warm boot. Sync state present
+        /// </summary>
+        HasSyncState = 2
+    }
+}

src/Umbraco.Core/Umbraco.Core.csproj

@@ -179,6 +179,9 @@
     <Compile Include="Migrations\Upgrade\V_8_7_0\MissingDictionaryIndex.cs" />
     <Compile Include="Services\IInstallationService.cs" />
     <Compile Include="Services\IUpgradeService.cs" />
+    <Compile Include="Sync\ISyncBootStateAccessor.cs" />
+    <Compile Include="Sync\NonRuntimeLevelBootStateAccessor.cs" />
+    <Compile Include="Sync\SyncBootState.cs" />
     <Compile Include="SystemLock.cs" />
     <Compile Include="Attempt.cs" />
     <Compile Include="AttemptOfTResult.cs" />

src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs

@@ -17,6 +17,7 @@
 using Umbraco.Core.Services;
 using Umbraco.Core.Services.Changes;
 using Umbraco.Core.Strings;
+using Umbraco.Core.Sync;
 using Umbraco.Tests.TestHelpers;
 using Umbraco.Tests.Testing.Objects;
 using Umbraco.Tests.Testing.Objects.Accessors;
@@ -155,7 +156,8 @@ private void Init(Func<IEnumerable<ContentNodeKit>> kits)
                 globalSettings,
                 Mock.Of<IEntityXmlSerializer>(),
                 Mock.Of<IPublishedModelFactory>(),
-                new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }));
+                new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }),
+                new TestSyncBootStateAccessor(SyncBootState.HasSyncState));
 
             // invariant is the current default
             _variationAccesor.VariationContext = new VariationContext();

src/Umbraco.Tests/PublishedContent/NuCacheTests.cs

@@ -17,6 +17,7 @@
 using Umbraco.Core.Services;
 using Umbraco.Core.Services.Changes;
 using Umbraco.Core.Strings;
+using Umbraco.Core.Sync;
 using Umbraco.Tests.TestHelpers;
 using Umbraco.Tests.Testing.Objects;
 using Umbraco.Tests.Testing.Objects.Accessors;
@@ -201,7 +202,8 @@ private void Init()
                 globalSettings,
                 Mock.Of<IEntityXmlSerializer>(),
                 Mock.Of<IPublishedModelFactory>(),
-                new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }));
+                new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }),
+                new TestSyncBootStateAccessor(SyncBootState.HasSyncState));
 
             // invariant is the current default
             _variationAccesor.VariationContext = new VariationContext();

src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs

@@ -99,7 +99,8 @@ protected override IPublishedSnapshotService CreatePublishedSnapshotService()
                 Factory.GetInstance<IGlobalSettings>(),
                 Factory.GetInstance<IEntityXmlSerializer>(),
                 Mock.Of<IPublishedModelFactory>(),
-                new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }));
+                new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }),
+                new TestSyncBootStateAccessor(SyncBootState.HasSyncState));
         }
 
         protected UmbracoContext GetUmbracoContextNu(string url, int templateId = 1234, RouteData routeData = null, bool setSingleton = false, IUmbracoSettingsSection umbracoSettings = null, IEnumerable<IUrlProvider> urlProviders = null)

src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs

@@ -17,6 +17,7 @@
 using Umbraco.Core.Services;
 using Umbraco.Core.Strings;
 using Umbraco.Core.Sync;
+using Umbraco.Tests.TestHelpers;
 using Umbraco.Tests.TestHelpers.Entities;
 using Umbraco.Tests.Testing;
 using Umbraco.Web.PublishedCache;
@@ -70,7 +71,8 @@ protected override IPublishedSnapshotService CreatePublishedSnapshotService()
                 Factory.GetInstance<IGlobalSettings>(),
                 Factory.GetInstance<IEntityXmlSerializer>(),
                 Mock.Of<IPublishedModelFactory>(),
-                new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }));
+                new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }),
+                new TestSyncBootStateAccessor(SyncBootState.HasSyncState));
         }
 
         public class LocalServerMessenger : ServerMessengerBase

src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Umbraco.Core.Sync;
+
+namespace Umbraco.Tests.TestHelpers
+{
+    class TestSyncBootStateAccessor : ISyncBootStateAccessor
+    {
+        private readonly SyncBootState _syncBootState;
+
+        public TestSyncBootStateAccessor(SyncBootState syncBootState)
+        {
+            _syncBootState = syncBootState;
+        }
+        public SyncBootState GetSyncBootState()
+        {
+            return _syncBootState;
+        }
+    }
+}

src/Umbraco.Tests/Umbraco.Tests.csproj

@@ -178,6 +178,7 @@
     <Compile Include="Services\RedirectUrlServiceTests.cs" />
     <Compile Include="Templates\HtmlLocalLinkParserTests.cs" />
     <Compile Include="TestHelpers\RandomIdRamDirectory.cs" />
+    <Compile Include="TestHelpers\TestSyncBootStateAccessor.cs" />
     <Compile Include="Testing\Objects\TestDataSource.cs" />
     <Compile Include="Published\PublishedSnapshotTestObjects.cs" />
     <Compile Include="Published\ModelTypeTests.cs" />

src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs

@@ -72,6 +72,7 @@ public override void Compose(Composition composition)
 
             composition.SetDatabaseServerMessengerOptions(GetDefaultOptions);
             composition.SetServerMessenger<BatchedDatabaseServerMessenger>();
+            composition.Register<ISyncBootStateAccessor>(factory=> factory.GetInstance<IServerMessenger>() as BatchedDatabaseServerMessenger, Lifetime.Singleton);
         }
     }
 

src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs

@@ -1,5 +1,6 @@
 using Umbraco.Core;
 using Umbraco.Core.Composing;
+using Umbraco.Core.Sync;
 using Umbraco.Web.PublishedCache.NuCache.DataSource;
 
 namespace Umbraco.Web.PublishedCache.NuCache
@@ -10,6 +11,9 @@ public override void Compose(Composition composition)
         {
             base.Compose(composition);
 
+            //Overriden on Run state in DatabaseServerRegistrarAndMessengerComposer
+            composition.Register<ISyncBootStateAccessor, NonRuntimeLevelBootStateAccessor>(Lifetime.Singleton);
+
             // register the NuCache database data source
             composition.Register<IDataSource, DatabaseDataSource>();
 

src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs

@@ -24,6 +24,7 @@
 using Umbraco.Core.Services.Changes;
 using Umbraco.Core.Services.Implement;
 using Umbraco.Core.Strings;
+using Umbraco.Core.Sync;
 using Umbraco.Web.Cache;
 using Umbraco.Web.Install;
 using Umbraco.Web.PublishedCache.NuCache.DataSource;
@@ -63,6 +64,8 @@ internal class PublishedSnapshotService : PublishedSnapshotServiceBase
         private bool _localContentDbExists;
         private bool _localMediaDbExists;
 
+        private readonly ISyncBootStateAccessor _syncBootStateAccessor;
+
         // define constant - determines whether to use cache when previewing
         // to store eg routes, property converted values, anything - caching
         // means faster execution, but uses memory - not sure if we want it
@@ -81,7 +84,8 @@ public PublishedSnapshotService(PublishedSnapshotServiceOptions options, IMainDo
             IDataSource dataSource, IGlobalSettings globalSettings,
             IEntityXmlSerializer entitySerializer,
             IPublishedModelFactory publishedModelFactory,
-            UrlSegmentProviderCollection urlSegmentProviders)
+            UrlSegmentProviderCollection urlSegmentProviders,
+            ISyncBootStateAccessor syncBootStateAccessor)
             : base(publishedSnapshotAccessor, variationContextAccessor)
         {
             //if (Interlocked.Increment(ref _singletonCheck) > 1)
@@ -99,6 +103,8 @@ public PublishedSnapshotService(PublishedSnapshotServiceOptions options, IMainDo
             _globalSettings = globalSettings;
             _urlSegmentProviders = urlSegmentProviders;
 
+            _syncBootStateAccessor = syncBootStateAccessor;
+
             // we need an Xml serializer here so that the member cache can support XPath,
             // for members this is done by navigating the serialized-to-xml member
             _entitySerializer = entitySerializer;
@@ -217,7 +223,12 @@ private void LoadCachesOnStartup()
         {
             var okContent = false;
             var okMedia = false;
-
+            if (_syncBootStateAccessor.GetSyncBootState() == SyncBootState.ColdBoot)
+            {
+                _logger.Info<PublishedSnapshotService>("Sync Service is in a Cold Boot state. Skip LoadCachesOnStartup as the Sync Service will trigger a full reload");
+                _isReady = true;
+                return;
+            }
             try
             {
                 if (_localContentDbExists)
@@ -233,7 +244,7 @@ private void LoadCachesOnStartup()
                     if (!okMedia)
                         _logger.Warn<PublishedSnapshotService>("Loading media from local db raised warnings, will reload from database.");
                 }
-
+                
                 if (!okContent)
                     LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true));