From 7704d02883306d2b71c82b24f4d424c0ffd062f9 Mon Sep 17 00:00:00 2001 From: Chad Date: Wed, 21 Apr 2021 11:05:37 +1200 Subject: [PATCH] Merge in #9859 to v8/feature/nucache-perf (#10152) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * load only once * Bump version to 8.6.8 * Initial rework of Lock dictionaries * [Issue 5277-146] accessibility - Close 'X' icon next to language drop… (#9264) * [Issue 5277-146] accessibility - Close 'X' icon next to language drop down is identified as "link" - screen reader * add new loacalization key * Fix issue with SqlMainDomLock that cannot use implicit lock timeouts … (#9973) * Fix issue with SqlMainDomLock that cannot use implicit lock timeouts … (#9973) (cherry picked from commit da5351dfcf23daad69fcd73eb74811456ffc34c0) * Adjust unit tests and apply fixes to scope * Add more unit tests, showing current issue * Counting Umbraco.ModelsBuilder and ModelsBuilder.Umbraco namespaces as external providers * Fix dead lock with TypeLoader * Fix errors shown in unit tests * Throw error if all scopes hasn't been disposed * Clean * Fixes and Updates for DB Scope and Ambient Context leaks (#9953) * Adds some scope tests (ported back from netcore) and provides a much better error message, ensure execution context is not flowed to child tasks that shouldn't leak any current ambient context * updates comment * Ensure SqlMainDomLock suppresses execution context too * Since we're awaiting a task in a library method, ConfigureAwait(false) * missing null check Co-authored-by: Elitsa Marinovska * Adds additional error checking and reporting to MainDom/SqlMainDomLock (#9954) Co-authored-by: Elitsa Marinovska * Add copy logic to Media Picker (#9957) * Add copy logic to Media Picker * Add action for copy all * Fix for selectable media item * Wrap calls to map in scopes * Autocomplete scopes * Remove unnecessary aria-hidden attribute from * Remove scope from method that calls another method that has a scope * Fixes #9993 - Cannot save empty image in Grid * Clean * Revert "The Value() method for IPublishedContent was not working with the defaultValue parameter" (#9989) * Use a hashset to keep track of acquired locks This simplifies disposing/checking for locks greatly. * Add images in grid - fixes 9982 (#9987) Co-authored-by: Sebastiaan Janssen * Only create the dicts and hashset when a lock is requested * Clean * Adds a config for configuring the access rules on the content dashboard - by default it granted for all user groups * Adds additional params indicating whether user is admin * Add images in grid - fixes 9982 (#9987) Co-authored-by: Sebastiaan Janssen (cherry picked from commit e2019777fbfc1f9221d040cb9f0b82c57f8552b9) * Bump version to 8.12.2 * #9964 Removed unneeded check for HttpContext * Fix for #9950 - HttpsCheck will now retry using the login background image if inital request returns 301/302. Excessvie Headers check will now check the root url instead of the backoffice * Merge pull request #9994 from umbraco/v8/bugfix/9993 Fixes #9993 - Cannot save empty image in Grid (cherry picked from commit 0ecc933921f2dea9a2a16d6f395b44a039663ec6) * Apply suggestions from review * Fixes #9983 - Getting kicked, if document type has a Umbraco.UserPicker property (#10002) * Fixes #9983 Temporary fix for this issue. using the entityservice like before. * Needed to remove the call to usersResource here as well for displaying the picked items * Don't need usersResource for now * Fixes #9983 - Getting kicked, if document type has a Umbraco.UserPicker property (#10002) * Fixes #9983 Temporary fix for this issue. using the entityservice like before. * Needed to remove the call to usersResource here as well for displaying the picked items * Don't need usersResource for now (cherry picked from commit 45de0a101eaa2b8f16e21a765f32928c7cb968be) * 8539: Allow alias in image cropper (#9266) Co-authored-by: Owain Williams * Wrap dumping dictionaries in a method. * Create method for generating log message And remove forgotten comments. * Fix swedish translation for somethingElse. * Copy member type (#10020) * Add copy dialog for member type * Implement copy action for member type * Create specific localization for content type, media type and member type * Handle "foldersonly" querystring * Add button type attribute * Add a few missing changes of anchor to button element * Null check on scope and options to ensure backward compatibility * Improve performance, readability and handling of FollowInternalRedirects (#9889) * Improve performance, readability and handling of FollowInternalRedirects * Logger didn't like string param Passing string param to _logger.Debug wasn't happy. Changed to pass existing internalRedirectAsInt variable. Co-authored-by: Nathan Woulfe * Update casing of listview layout name * 9097 add contextual password helper (#9256) * update back-office forms * Display tip on reset password page as well * add directive for password tip * integrate directove in login screen * forgot the ng-keyup :-) * adapt tooltip directive to potential different Members and Users password settings * remove watcher Co-authored-by: Nathan Woulfe * Unbind listener Listening for splitViewRequest was only unbound if the split view editor was opened. Not cleaning up the listener caused a memory leak when changing between nodes as the spit view editor was detached but not garbage-collected * Replace icon in date picker with umb-icon component (#10040) * Replace icon in date picker with component * Adjust height of clear button * Update cypress and fix tests * Listview config icons (#10036) * Update icons to use component * Simplify markup and use disabled button * Use move cursor style on sortable handle * Add class for action column * Update setting auto focus * Increase font size of umb-panel-header-icon * Anchor noopener (#10009) * Set rel="noopener" for anchors with target="_blank" * Reverted unwanted changes to Default.cshtml * Align 'Add language' test to netcore * Add new cypress tests * Add indentation * Getting rid of the config file and implementing an appSetting instead * Implementation for IContentDashboardSettings * Cleanup * bool.Try * Taking AllowContentDashboardAccessToAllUsers prop from GlobalSettings to ContentDashboardSettings and saving AccessRulesFromConfig into a backing field * fix support for non run states * Handling multiple values per field in Examine Management * Add Root and Breadcrumbs extension methods for IPublishedContent (#9033) * Fix usage of obsolete CreatorName and WriterName properties * Add generic Root extension method * Add Breadcrumbs extension methods * Orders member type grouping of members alphabetically, matching the listing of member types. * Revert updating deprecated WriterName/CreatorName refs Changing the properties to use the extensions is a good thing (given the props are deprecated), but causes issues deep in tests. I'm reverting that change to fix the tests, and all refs to the deprecated properties should be updated in one sweep, to deal with any other test issues that might crop up. * Handle Invalid format for Upgrade check * Fixes tabbing-mode remains active after closing modal #9790 (#10074) * Allow to pass in boolean to preventEnterSubmit directive (#8639) * Pass in value to preventEnterSubmit directive * Set enabled similar to preventDefault and preventEnterSubmit directives * Update prevent enter submit value * Init value from controller * Use a different default input id prefix for umb-search-filter * Fix typo * Check for truthly value * Revert "Set enabled similar to preventDefault and preventEnterSubmit directives" This reverts commit 536ce855c4545ead82cea77b4013bf9010a8687b. * None pointer events when clicking icon * Use color variable * Fixes tabbing-mode remains active after closing modal #9790 (#10074) (cherry picked from commit c881fa9e7d08c11954e18489827f70cdafceb947) * Null check on scope and options to ensure backward compatibility (cherry picked from commit fe8cd239d2f4c528c1a8a3cf4c50e90bb43cacfc) * Fix validation of step size in integer/numeric field * 9962: Use $allowedEditors instead of allowed (#10086) * 9962: Use $allowedEditors instead of allowed * 9962: Remove redundant statement * fixes #10021 adds ng-form and val-form-manager to the documentation * Improved accessibility of link picker (#10099) * Added support for screeen reader alerts on the embed so that assitive technology knows when a url retrieve has been succesfull. Added labels for the controls Preview reload only triggered if the values for height and width change * Added control ids for the link picker * Add French translation * Accessibility: Alerts the user how many results have been returned on a tree search (#10100) * Added support for screeen reader alerts on the embed so that assitive technology knows when a url retrieve has been succesfull. Added labels for the controls Preview reload only triggered if the values for height and width change * Tree search details the number of search items returned * Add French translations * Updated LightInject to v6.4.0 * Remove HtmlSanitizer once more - see #9803 * Also make sure NuGet installs the correct version of the CodePages dependency * Bump version to 8.13 RC * Fixed copy preserving sort order (#10091) * Revert "Updated LightInject to v6.4.0" This reverts commit fc77252ec756cf90bb74e7fbbe6dd6d75cbdacfc. * Revert "Add copy logic to Media Picker (#9957)" This reverts commit f7c032af65cac83182782c758a3ab79c86b92e70. * Reintroduce old constructor to make non-breaking * Update cypress test to make macros in the grid work again * Attributes could be multiple items, test specifically if `Directory` is an attribute * Accessibility: Adding label fors and control ids for the macro picker (#10101) * Added support for screeen reader alerts on the embed so that assitive technology knows when a url retrieve has been succesfull. Added labels for the controls Preview reload only triggered if the values for height and width change * Added support for label fors for the macro picker and also gave the ,acro search box a title * Now displays a count of the matching macros returned. Please note the language file amends shared with #10100 * Removed src-only class for the display of the count of messages * Updating typo * Removed top-margin from switcher icon * Allow KeepAlive controller Ping method to be requested by non local requests (#10126) * Allow KeepAlive controller Ping method to be requested by non local requests and accept head requests * removed unused references * fix csproj * fix merge * btree serializer optimizations * array pool and nametable optimizations Co-authored-by: Mole Co-authored-by: Sebastiaan Janssen Co-authored-by: Justin Shearer Co-authored-by: Bjarke Berg Co-authored-by: Callum Whyte Co-authored-by: Shannon Co-authored-by: Elitsa Marinovska Co-authored-by: patrickdemooij9 Co-authored-by: Bjarne Fyrstenborg Co-authored-by: Michael Latouche Co-authored-by: Nathan Woulfe Co-authored-by: Markus Johansson Co-authored-by: Jeavon Leopold Co-authored-by: Benjamin Carleski Co-authored-by: Owain Williams Co-authored-by: Jesper Löfgren Co-authored-by: Martin Bentancour Co-authored-by: Ronald Barendse Co-authored-by: Andy Butland Co-authored-by: BeardinaSuit Co-authored-by: Mads Rasmussen Co-authored-by: Rachel Breeze Co-authored-by: Dave de Moel Co-authored-by: ric <60885685+ricbrady@users.noreply.github.com> Co-authored-by: Carole Rennie Logan Co-authored-by: Dennis Öhman --- .../Sync/DatabaseServerMessenger.cs | 88 +++++++++++++------ .../Sync/ISyncBootStateAccessor.cs | 20 +++++ .../Sync/NonRuntimeLevelBootStateAccessor.cs | 19 ++++ src/Umbraco.Core/Sync/SyncBootState.cs | 24 +++++ src/Umbraco.Core/Umbraco.Core.csproj | 3 + .../PublishedContent/NuCacheChildrenTests.cs | 2 + .../PublishedContent/NuCacheTests.cs | 2 + .../Scoping/ScopedNuCacheTests.cs | 1 + .../ContentTypeServiceVariantsTests.cs | 2 + .../TestHelpers/TestSyncBootStateAccessor.cs | 23 +++++ src/Umbraco.Tests/Umbraco.Tests.csproj | 1 + ...aseServerRegistrarAndMessengerComponent.cs | 1 + ...Tree.DictionaryOfPropertyDataSerializer.cs | 8 +- .../JsonContentNestedDataSerializer.cs | 53 ++++++++++- .../PublishedCache/NuCache/NuCacheComposer.cs | 4 + .../NuCache/PublishedSnapshotService.cs | 19 +++- 16 files changed, 232 insertions(+), 38 deletions(-) create mode 100644 src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs create mode 100644 src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs create mode 100644 src/Umbraco.Core/Sync/SyncBootState.cs create mode 100644 src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs mode change 100755 => 100644 src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs diff --git a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs index 49b0d23862cc..ebc77dbdca5d 100644 --- a/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs +++ b/src/Umbraco.Core/Sync/DatabaseServerMessenger.cs @@ -28,7 +28,7 @@ namespace Umbraco.Core.Sync // but only processes instructions coming from remote servers, // thus ensuring that instructions run only once // - public class DatabaseServerMessenger : ServerMessengerBase + public class DatabaseServerMessenger : ServerMessengerBase, ISyncBootStateAccessor { private readonly IRuntimeState _runtime; private readonly ManualResetEvent _syncIdle; @@ -172,15 +172,39 @@ private void Initialize(IUmbracoDatabase database) lock (_locko) { if (_released) return; + var coldboot = IsColdBoot(database); - var coldboot = false; - if (_lastId < 0) // never synced before + if (coldboot) { - // we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new - // server and it will need to rebuild it's own caches, eg Lucene or the xml cache file. - Logger.Warn("No last synced Id found, this generally means this is a new server/install." - + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in" - + " the database and maintain cache updates based on that Id."); + // go get the last id in the db and store it + // note: do it BEFORE initializing otherwise some instructions might get lost + // when doing it before, some instructions might run twice - not an issue + var maxId = database.ExecuteScalar("SELECT MAX(id) FROM umbracoCacheInstruction"); + + //if there is a max currently, or if we've never synced + if (maxId > 0 || _lastId < 0) + SaveLastSynced(maxId); + + // execute initializing callbacks + if (Options.InitializingCallbacks != null) + foreach (var callback in Options.InitializingCallbacks) + callback(); + } + + _initialized = true; + } + } + + private bool IsColdBoot(IUmbracoDatabase database) + { + var coldboot = false; + if (_lastId < 0) // never synced before + { + // we haven't synced - in this case we aren't going to sync the whole thing, we will assume this is a new + // server and it will need to rebuild it's own caches, eg Lucene or the xml cache file. + Logger.Warn("No last synced Id found, this generally means this is a new server/install." + + " The server will build its caches and indexes, and then adjust its last synced Id to the latest found in" + + " the database and maintain cache updates based on that Id."); coldboot = true; } @@ -198,29 +222,11 @@ private void Initialize(IUmbracoDatabase database) + " to the latest found in the database and maintain cache updates based on that Id.", count, Options.MaxProcessingInstructionCount); - coldboot = true; - } - } - - if (coldboot) - { - // go get the last id in the db and store it - // note: do it BEFORE initializing otherwise some instructions might get lost - // when doing it before, some instructions might run twice - not an issue - var maxId = database.ExecuteScalar("SELECT MAX(id) FROM umbracoCacheInstruction"); - - //if there is a max currently, or if we've never synced - if (maxId > 0 || _lastId < 0) - SaveLastSynced(maxId); - - // execute initializing callbacks - if (Options.InitializingCallbacks != null) - foreach (var callback in Options.InitializingCallbacks) - callback(); + coldboot = true; } - - _initialized = true; } + + return coldboot; } /// @@ -548,6 +554,30 @@ private string GetDistCacheFilePath(IGlobalSettings globalSettings) #endregion + public SyncBootState GetSyncBootState() + { + try + { + ReadLastSynced(); // get _lastId + using (var scope = ScopeProvider.CreateScope()) + { + EnsureInstructions(scope.Database); + bool isColdBoot = IsColdBoot(scope.Database); + + if (isColdBoot) + { + return SyncBootState.ColdBoot; + } + return SyncBootState.HasSyncState; + } + } + catch(Exception ex) + { + Logger.Warn("Error determining Sync Boot State", ex); + return SyncBootState.Unknown; + } + } + #region Notify refreshers private static ICacheRefresher GetRefresher(Guid id) diff --git a/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs new file mode 100644 index 000000000000..4b8500f2d924 --- /dev/null +++ b/src/Umbraco.Core/Sync/ISyncBootStateAccessor.cs @@ -0,0 +1,20 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Core.Sync +{ + /// + /// Retrieve the state of the sync service + /// + public interface ISyncBootStateAccessor + { + /// + /// Get the boot state + /// + /// + SyncBootState GetSyncBootState(); + } +} diff --git a/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs new file mode 100644 index 000000000000..70cec6cc9606 --- /dev/null +++ b/src/Umbraco.Core/Sync/NonRuntimeLevelBootStateAccessor.cs @@ -0,0 +1,19 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Core.Sync +{ + /// + /// Boot state implementation for when umbraco is not in the run state + /// + public class NonRuntimeLevelBootStateAccessor : ISyncBootStateAccessor + { + public SyncBootState GetSyncBootState() + { + return SyncBootState.Unknown; + } + } +} diff --git a/src/Umbraco.Core/Sync/SyncBootState.cs b/src/Umbraco.Core/Sync/SyncBootState.cs new file mode 100644 index 000000000000..4abc53abba6a --- /dev/null +++ b/src/Umbraco.Core/Sync/SyncBootState.cs @@ -0,0 +1,24 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Umbraco.Core.Sync +{ + public enum SyncBootState + { + /// + /// Unknown state. Treat as HasSyncState + /// + Unknown = 0, + /// + /// Cold boot. No Sync state + /// + ColdBoot = 1, + /// + /// Warm boot. Sync state present + /// + HasSyncState = 2 + } +} diff --git a/src/Umbraco.Core/Umbraco.Core.csproj b/src/Umbraco.Core/Umbraco.Core.csproj index 116088130428..6b4725c48c76 100755 --- a/src/Umbraco.Core/Umbraco.Core.csproj +++ b/src/Umbraco.Core/Umbraco.Core.csproj @@ -190,6 +190,9 @@ + + + diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs index afba2dcc4f1f..75a20ade6fc0 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheChildrenTests.cs @@ -17,6 +17,7 @@ using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Strings; +using Umbraco.Core.Sync; using Umbraco.Tests.TestHelpers; using Umbraco.Tests.Testing.Objects; using Umbraco.Tests.Testing.Objects.Accessors; @@ -158,6 +159,7 @@ private void Init(Func> kits) Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.HasSyncState), _contentNestedDataSerializerFactory); // invariant is the current default diff --git a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs index eee3500495e6..9feb0d703b00 100644 --- a/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs +++ b/src/Umbraco.Tests/PublishedContent/NuCacheTests.cs @@ -17,6 +17,7 @@ using Umbraco.Core.Services; using Umbraco.Core.Services.Changes; using Umbraco.Core.Strings; +using Umbraco.Core.Sync; using Umbraco.Tests.TestHelpers; using Umbraco.Tests.Testing.Objects; using Umbraco.Tests.Testing.Objects.Accessors; @@ -204,6 +205,7 @@ private void Init() Mock.Of(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.HasSyncState), _contentNestedDataSerializerFactory); // invariant is the current default diff --git a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs index be10db3a9d7f..ad372c00b9ee 100644 --- a/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs +++ b/src/Umbraco.Tests/Scoping/ScopedNuCacheTests.cs @@ -101,6 +101,7 @@ protected override IPublishedSnapshotService CreatePublishedSnapshotService() Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.HasSyncState), nestedContentDataSerializerFactory); } diff --git a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs index aaad60f7e9df..b252738fee00 100644 --- a/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs +++ b/src/Umbraco.Tests/Services/ContentTypeServiceVariantsTests.cs @@ -17,6 +17,7 @@ using Umbraco.Core.Services; using Umbraco.Core.Strings; using Umbraco.Core.Sync; +using Umbraco.Tests.TestHelpers; using Umbraco.Tests.TestHelpers.Entities; using Umbraco.Tests.Testing; using Umbraco.Web.PublishedCache; @@ -73,6 +74,7 @@ protected override IPublishedSnapshotService CreatePublishedSnapshotService() Factory.GetInstance(), Mock.Of(), new UrlSegmentProviderCollection(new[] { new DefaultUrlSegmentProvider() }), + new TestSyncBootStateAccessor(SyncBootState.HasSyncState), nestedContentDataSerializerFactory); } diff --git a/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs b/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs new file mode 100644 index 000000000000..e5f69893814d --- /dev/null +++ b/src/Umbraco.Tests/TestHelpers/TestSyncBootStateAccessor.cs @@ -0,0 +1,23 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Umbraco.Core.Sync; + +namespace Umbraco.Tests.TestHelpers +{ + class TestSyncBootStateAccessor : ISyncBootStateAccessor + { + private readonly SyncBootState _syncBootState; + + public TestSyncBootStateAccessor(SyncBootState syncBootState) + { + _syncBootState = syncBootState; + } + public SyncBootState GetSyncBootState() + { + return _syncBootState; + } + } +} diff --git a/src/Umbraco.Tests/Umbraco.Tests.csproj b/src/Umbraco.Tests/Umbraco.Tests.csproj index 46d2216e8240..4920bcda2ae0 100644 --- a/src/Umbraco.Tests/Umbraco.Tests.csproj +++ b/src/Umbraco.Tests/Umbraco.Tests.csproj @@ -184,6 +184,7 @@ + diff --git a/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs index 2fa9d807795e..26ba0db324d0 100644 --- a/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs +++ b/src/Umbraco.Web/Compose/DatabaseServerRegistrarAndMessengerComponent.cs @@ -72,6 +72,7 @@ public override void Compose(Composition composition) composition.SetDatabaseServerMessengerOptions(GetDefaultOptions); composition.SetServerMessenger(); + composition.Register(factory=> factory.GetInstance() as BatchedDatabaseServerMessenger, Lifetime.Singleton); } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs index 0b15c0ba4b8b..1b96538dd046 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/BTree.DictionaryOfPropertyDataSerializer.cs @@ -13,11 +13,11 @@ internal class DictionaryOfPropertyDataSerializer : SerializerBase, ISerializer< { public IDictionary ReadFrom(Stream stream) { - var dict = new Dictionary(StringComparer.InvariantCultureIgnoreCase); // read properties count var pcount = PrimitiveSerializer.Int32.ReadFrom(stream); + var dict = new Dictionary(pcount,StringComparer.InvariantCultureIgnoreCase); // read each property for (var i = 0; i < pcount; i++) { @@ -28,13 +28,13 @@ public IDictionary ReadFrom(Stream stream) var vcount = PrimitiveSerializer.Int32.ReadFrom(stream); // create pdata and add to the dictionary - var pdatas = new List(); + var pdatas = new PropertyData[vcount]; // for each value, read and add to pdata for (var j = 0; j < vcount; j++) { var pdata = new PropertyData(); - pdatas.Add(pdata); + pdatas[j] =pdata; // everything that can be null is read/written as object // even though - culture and segment should never be null here, as 'null' represents @@ -46,7 +46,7 @@ public IDictionary ReadFrom(Stream stream) pdata.Value = ReadObject(stream); } - dict[key] = pdatas.ToArray(); + dict[key] = pdatas; } return dict; } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs index c4d40f721f30..21cd0bf76399 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/DataSource/JsonContentNestedDataSerializer.cs @@ -1,6 +1,8 @@ using Newtonsoft.Json; using System; +using System.Buffers; using System.Collections.Generic; +using System.IO; using Umbraco.Core.Models; using Umbraco.Core.Serialization; @@ -21,13 +23,20 @@ public class JsonContentNestedDataSerializer : IContentCacheDataSerializer DateTimeZoneHandling = DateTimeZoneHandling.Utc, DateFormatString = "o" }; - + private readonly JsonNameTable _propertyNameTable = new DefaultJsonNameTable(); public ContentCacheDataModel Deserialize(IReadOnlyContentBase content, string stringData, byte[] byteData) { if (stringData == null && byteData != null) throw new NotSupportedException($"{typeof(JsonContentNestedDataSerializer)} does not support byte[] serialization"); - return JsonConvert.DeserializeObject(stringData, _jsonSerializerSettings); + JsonSerializer serializer = JsonSerializer.Create(_jsonSerializerSettings); + using (JsonTextReader reader = new JsonTextReader(new StringReader(stringData))) + { + // reader will get buffer from array pool + reader.ArrayPool = JsonArrayPool.Instance; + reader.PropertyNameTable = _propertyNameTable; + return serializer.Deserialize(reader); + } } public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase content, ContentCacheDataModel model) @@ -39,4 +48,44 @@ public ContentCacheDataSerializationResult Serialize(IReadOnlyContentBase conten return new ContentCacheDataSerializationResult(json, null); } } + public class JsonArrayPool : IArrayPool + { + public static readonly JsonArrayPool Instance = new JsonArrayPool(); + + public char[] Rent(int minimumLength) + { + // get char array from System.Buffers shared pool + return ArrayPool.Shared.Rent(minimumLength); + } + + public void Return(char[] array) + { + // return char array to System.Buffers shared pool + ArrayPool.Shared.Return(array); + } + } + public class AutomaticJsonNameTable : DefaultJsonNameTable + { + int nAutoAdded = 0; + int maxToAutoAdd; + + public AutomaticJsonNameTable(int maxToAdd) + { + this.maxToAutoAdd = maxToAdd; + } + + public override string Get(char[] key, int start, int length) + { + var s = base.Get(key, start, length); + + if (s == null && nAutoAdded < maxToAutoAdd) + { + s = new string(key, start, length); + Add(s); + nAutoAdded++; + } + + return s; + } + } } diff --git a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs index 98d8b91386e5..6dac3b9afb49 100644 --- a/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/NuCacheComposer.cs @@ -1,6 +1,7 @@ using System.Configuration; using Umbraco.Core; using Umbraco.Core.Composing; +using Umbraco.Core.Sync; using Umbraco.Core.PropertyEditors; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -27,6 +28,9 @@ public override void Compose(Composition composition) composition.RegisterUnique(factory => new ContentDataSerializer(new DictionaryOfPropertyDataSerializer())); + //Overriden on Run state in DatabaseServerRegistrarAndMessengerComposer + composition.Register(Lifetime.Singleton); + // register the NuCache database data source composition.RegisterUnique(); diff --git a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs old mode 100755 new mode 100644 index 3a055223a594..5b3980ad0694 --- a/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs +++ b/src/Umbraco.Web/PublishedCache/NuCache/PublishedSnapshotService.cs @@ -21,6 +21,7 @@ using Umbraco.Core.Services.Changes; using Umbraco.Core.Services.Implement; using Umbraco.Core.Strings; +using Umbraco.Core.Sync; using Umbraco.Web.Cache; using Umbraco.Web.Install; using Umbraco.Web.PublishedCache.NuCache.DataSource; @@ -62,6 +63,8 @@ internal class PublishedSnapshotService : PublishedSnapshotServiceBase private bool _localContentDbExists; private bool _localMediaDbExists; + private readonly ISyncBootStateAccessor _syncBootStateAccessor; + // define constant - determines whether to use cache when previewing // to store eg routes, property converted values, anything - caching // means faster execution, but uses memory - not sure if we want it @@ -80,7 +83,10 @@ public PublishedSnapshotService(PublishedSnapshotServiceOptions options, IMainDo IDataSource dataSource, IGlobalSettings globalSettings, IEntityXmlSerializer entitySerializer, IPublishedModelFactory publishedModelFactory, - UrlSegmentProviderCollection urlSegmentProviders, IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, ContentDataSerializer contentDataSerializer = null) + UrlSegmentProviderCollection urlSegmentProviders, + ISyncBootStateAccessor syncBootStateAccessor, + IContentCacheDataSerializerFactory contentCacheDataSerializerFactory, + ContentDataSerializer contentDataSerializer = null) : base(publishedSnapshotAccessor, variationContextAccessor) { //if (Interlocked.Increment(ref _singletonCheck) > 1) @@ -100,6 +106,8 @@ public PublishedSnapshotService(PublishedSnapshotServiceOptions options, IMainDo _contentCacheDataSerializerFactory = contentCacheDataSerializerFactory; _contentDataSerializer = contentDataSerializer; + _syncBootStateAccessor = syncBootStateAccessor; + // we need an Xml serializer here so that the member cache can support XPath, // for members this is done by navigating the serialized-to-xml member _entitySerializer = entitySerializer; @@ -218,7 +226,12 @@ private void LoadCachesOnStartup() { var okContent = false; var okMedia = false; - + if (_syncBootStateAccessor.GetSyncBootState() == SyncBootState.ColdBoot) + { + _logger.Warn("Sync Service is in a Cold Boot state. Skip LoadCachesOnStartup as the Sync Service will trigger a full reload"); + _isReady = true; + return; + } try { if (_localContentDbExists) @@ -234,7 +247,7 @@ private void LoadCachesOnStartup() if (!okMedia) _logger.Warn("Loading media from local db raised warnings, will reload from database."); } - + if (!okContent) LockAndLoadContent(scope => LoadContentFromDatabaseLocked(scope, true));