Detect infinite loop in documents and index nodes

ConsoleApp1/Program.cs

@@ -48,7 +48,7 @@
 
         var col1 = db.Query("col1", Query.All()).ToList().Count;
         var col2 = db.Query("col2", Query.All()).ToList().Count;
-        
+
         Console.WriteLine("Inserted Col1: " + col1);
         Console.WriteLine("Inserted Col2: " + col2);
     }

LiteDB.Tests/Database/Upgrade_Tests.cs

@@ -14,7 +14,7 @@ public void Migrage_From_V4()
         {
             // v5 upgrades only from v4!
 
-            var original = "../../../Utils/Legacy/v4.db";
+            var original = "../../../Resources/v4.db";
             var copy = original.Replace(".db", "-copy.db");
 
             File.Copy(original, copy, true);
@@ -54,7 +54,7 @@ public void Migrage_From_V4_No_FileExtension()
         {
             // v5 upgrades only from v4!
 
-            var original = "../../../Utils/Legacy/v4.db";
+            var original = "../../../Resources/v4.db";
             var copy = original.Replace(".db", "-copy");
 
             File.Copy(original, copy, true);

LiteDB.Tests/Engine/Rebuild_Loop_Tests.cs

@@ -0,0 +1,58 @@
+using FluentAssertions;
+using LiteDB.Engine;
+using System;
+using System.IO;
+using System.Linq;
+
+using Xunit;
+
+#if DEBUG
+namespace LiteDB.Tests.Engine
+{
+    public class Rebuild_Loop_Tests
+    {
+
+        [Fact]
+        public void Rebuild_Detected_Infinite_Loop()
+        {
+            var original = "../../../Resources/Loop.db";
+            var filename = original.Replace(".db", "-copy");
+
+            File.Copy(original, filename, true);
+
+            var settings = new EngineSettings
+            {
+                Filename = filename,
+                Password = "bzj2NplCbVH/bB8fxtjEC7u0unYdKHJVSmdmPgArRBwmmGw0+Wd2tE+b2zRMFcHAzoG71YIn/2Nq1EMqa5JKcQ==",
+                AutoRebuild = true,
+            };
+
+            try
+            {
+                using (var db = new LiteEngine(settings))
+                {
+                    // infinite loop here
+                    var col = db.Query("hubData$AppOperations", Query.All()).ToList();
+
+                    // never run here
+                    Assert.Fail("not expected");
+                }
+            }
+            catch (Exception ex)
+            {
+                Assert.True(ex is LiteException lex && lex.ErrorCode == 999);
+            }
+
+            using (var db = new LiteEngine(settings))
+            {
+                var col = db.Query("hubData$AppOperations", Query.All()).ToList().Count;
+                var errors = db.Query("_rebuild_errors", Query.All()).ToList().Count;
+
+                col.Should().Be(408);
+                errors.Should().Be(0);
+            }
+        }
+    }
+}
+
+#endif

LiteDB.Tests/LiteDB.Tests.csproj

@@ -22,13 +22,8 @@
   </ItemGroup>
 
   <ItemGroup>
-    <None Remove="Utils\Json\person.json" />
-    <None Remove="Utils\Json\zip.json" />
-  </ItemGroup>
-
-  <ItemGroup>
-    <EmbeddedResource Include="Utils\Json\person.json" />
-    <EmbeddedResource Include="Utils\Json\zip.json" />
+    <EmbeddedResource Include="Resources\person.json" />
+    <EmbeddedResource Include="Resources\zip.json" />
   </ItemGroup>
 
   <ItemGroup>

LiteDB.Tests/Utils/DataGen.cs

@@ -11,7 +11,7 @@ public class DataGen
         /// </summary>
         public static IEnumerable<Person> Person()
         {
-            using (var stream = typeof(DataGen).Assembly.GetManifestResourceStream("LiteDB.Tests.Utils.Json.person.json"))
+            using (var stream = typeof(DataGen).Assembly.GetManifestResourceStream("LiteDB.Tests.Resources.person.json"))
             {
                 var reader = new StreamReader(stream);
 
@@ -56,7 +56,7 @@ public static IEnumerable<Person> Person(int start, int end)
         /// </summary>
         public static IEnumerable<Zip> Zip()
         {
-            using (var stream = typeof(DataGen).Assembly.GetManifestResourceStream("LiteDB.Tests.Utils.Json.zip.json"))
+            using (var stream = typeof(DataGen).Assembly.GetManifestResourceStream("LiteDB.Tests.Resources.zip.json"))
             {
                 var reader = new StreamReader(stream);
 

LiteDB/Document/Json/JsonSerializer.cs

@@ -16,33 +16,39 @@ public class JsonSerializer
         /// <summary>
         /// Json serialize a BsonValue into a String
         /// </summary>
-        public static string Serialize(BsonValue value)
+        public static string Serialize(BsonValue value, bool indent = false)
         {
             var sb = new StringBuilder();
 
-            Serialize(value, sb);
+            Serialize(value, sb, indent);
 
             return sb.ToString();
         }
 
         /// <summary>
         /// Json serialize a BsonValue into a TextWriter
         /// </summary>
-        public static void Serialize(BsonValue value, TextWriter writer)
+        public static void Serialize(BsonValue value, TextWriter writer, bool indent = false)
         {
-            var json = new JsonWriter(writer);
+            var json = new JsonWriter(writer)
+            {
+                Pretty = indent
+            };
 
             json.Serialize(value ?? BsonValue.Null);
         }
 
         /// <summary>
         /// Json serialize a BsonValue into a StringBuilder
         /// </summary>
-        public static void Serialize(BsonValue value, StringBuilder sb)
+        public static void Serialize(BsonValue value, StringBuilder sb, bool indent = false)
         {
             using (var writer = new StringWriter(sb))
             {
-                var w = new JsonWriter(writer);
+                var w = new JsonWriter(writer)
+                {
+                    Pretty = indent
+                };
 
                 w.Serialize(value ?? BsonValue.Null);
             }

LiteDB/Engine/Disk/DiskService.cs

@@ -2,8 +2,6 @@
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.IO;
-using System.Linq;
-using System.Runtime;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -128,6 +126,13 @@ public DiskReader GetReader()
             return new DiskReader(_state, _cache, _dataPool, _logPool);
         }
 
+        /// <summary>
+        /// This method calculates the maximum number of items (documents or IndexNodes) that this database can have.
+        /// The result is used to prevent infinite loops in case of problems with pointers
+        /// Each page support max of 255 items. Use 10 pages offset (avoid empty disk)
+        /// </summary>
+        public uint MAX_ITEMS_COUNT => (uint)(((_dataLength + _logLength) / PAGE_SIZE) + 10) * byte.MaxValue;
+
         /// <summary>
         /// When a page are requested as Writable but not saved in disk, must be discard before release
         /// </summary>

LiteDB/Engine/Engine/Delete.cs

@@ -19,8 +19,8 @@ public int Delete(string collection, IEnumerable<BsonValue> ids)
             {
                 var snapshot = transaction.CreateSnapshot(LockMode.Write, collection, false);
                 var collectionPage = snapshot.CollectionPage;
-                var data = new DataService(snapshot);
-                var indexer = new IndexService(snapshot, _header.Pragmas.Collation);
+                var data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT);
+                var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT);
 
                 if (collectionPage == null) return 0;
 

LiteDB/Engine/Engine/Index.cs

@@ -29,8 +29,8 @@ public bool EnsureIndex(string collection, string name, BsonExpression expressio
             {
                 var snapshot = transaction.CreateSnapshot(LockMode.Write, collection, true);
                 var collectionPage = snapshot.CollectionPage;
-                var indexer = new IndexService(snapshot, _header.Pragmas.Collation);
-                var data = new DataService(snapshot);
+                var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT);
+                var data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT);
 
                 // check if index already exists
                 var current = collectionPage.GetCollectionIndex(name);
@@ -108,7 +108,7 @@ public bool DropIndex(string collection, string name)
             {
                 var snapshot = transaction.CreateSnapshot(LockMode.Write, collection, false);
                 var col = snapshot.CollectionPage;
-                var indexer = new IndexService(snapshot, _header.Pragmas.Collation);
+                var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT);
             
                 // no collection, no index
                 if (col == null) return false;

LiteDB/Engine/Engine/Insert.cs

@@ -21,8 +21,8 @@ public int Insert(string collection, IEnumerable<BsonDocument> docs, BsonAutoId
             {
                 var snapshot = transaction.CreateSnapshot(LockMode.Write, collection, true);
                 var count = 0;
-                var indexer = new IndexService(snapshot, _header.Pragmas.Collation);
-                var data = new DataService(snapshot);
+                var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT);
+                var data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT);
 
                 LOG($"insert `{collection}`", "COMMAND");
 

LiteDB/Engine/Engine/Query.cs

@@ -36,6 +36,7 @@ public IBsonDataReader Query(string collection, Query query)
                 _state,
                 _monitor, 
                 _sortDisk, 
+                _disk,
                 _header.Pragmas, 
                 collection, 
                 query, 

LiteDB/Engine/Engine/Rebuild.cs

@@ -60,8 +60,8 @@ internal void RebuildContent(IFileReader reader)
                 {
                     // get snapshot, indexer and data services
                     var snapshot = transaction.CreateSnapshot(LockMode.Write, collection, true);
-                    var indexer = new IndexService(snapshot, _header.Pragmas.Collation);
-                    var data = new DataService(snapshot);
+                    var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT);
+                    var data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT);
 
                     // get all documents from current collection
                     var docs = reader.GetDocuments(collection);

LiteDB/Engine/Engine/Update.cs

@@ -19,8 +19,8 @@ public int Update(string collection, IEnumerable<BsonDocument> docs)
             {
                 var snapshot = transaction.CreateSnapshot(LockMode.Write, collection, false);
                 var collectionPage = snapshot.CollectionPage;
-                var indexer = new IndexService(snapshot, _header.Pragmas.Collation);
-                var data = new DataService(snapshot);
+                var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT);
+                var data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT);
                 var count = 0;
 
                 if (collectionPage == null) return 0;

LiteDB/Engine/Engine/Upsert.cs

@@ -20,8 +20,8 @@ public int Upsert(string collection, IEnumerable<BsonDocument> docs, BsonAutoId
             {
                 var snapshot = transaction.CreateSnapshot(LockMode.Write, collection, true);
                 var collectionPage = snapshot.CollectionPage;
-                var indexer = new IndexService(snapshot, _header.Pragmas.Collation);
-                var data = new DataService(snapshot);
+                var indexer = new IndexService(snapshot, _header.Pragmas.Collation, _disk.MAX_ITEMS_COUNT);
+                var data = new DataService(snapshot, _disk.MAX_ITEMS_COUNT);
                 var count = 0;
 
                 LOG($"upsert `{collection}`", "COMMAND");

LiteDB/Engine/Query/Pipeline/BasePipe.cs

@@ -14,13 +14,15 @@ internal abstract class BasePipe
         protected readonly IDocumentLookup _lookup;
         protected readonly SortDisk _tempDisk;
         protected readonly EnginePragmas _pragmas;
+        protected readonly uint _maxItemsCount;
 
-        public BasePipe(TransactionService transaction, IDocumentLookup lookup, SortDisk tempDisk, EnginePragmas pragmas)
+        public BasePipe(TransactionService transaction, IDocumentLookup lookup, SortDisk tempDisk, EnginePragmas pragmas, uint maxItemsCount)
         {
             _transaction = transaction;
             _lookup = lookup;
             _tempDisk = tempDisk;
             _pragmas = pragmas;
+            _maxItemsCount = maxItemsCount;
         }
 
         /// <summary>
@@ -96,8 +98,8 @@ void DoInclude(BsonDocument value)
 
                     // initialize services
                     snapshot = _transaction.CreateSnapshot(LockMode.Read, last, false);
-                    indexer = new IndexService(snapshot, _pragmas.Collation);
-                    data = new DataService(snapshot);
+                    indexer = new IndexService(snapshot, _pragmas.Collation, _maxItemsCount);
+                    data = new DataService(snapshot, _maxItemsCount);
 
                     lookup = new DatafileLookup(data, _pragmas.UtcDate, null);
 

LiteDB/Engine/Query/Pipeline/GroupByPipe.cs

@@ -10,8 +10,8 @@ namespace LiteDB.Engine
     /// </summary>
     internal class GroupByPipe : BasePipe
     {
-        public GroupByPipe(TransactionService transaction, IDocumentLookup loader, SortDisk tempDisk, EnginePragmas pragmas)
-            : base(transaction, loader, tempDisk, pragmas)
+        public GroupByPipe(TransactionService transaction, IDocumentLookup loader, SortDisk tempDisk, EnginePragmas pragmas, uint maxItemsCount)
+            : base(transaction, loader, tempDisk, pragmas, maxItemsCount)
         {
         }
 

LiteDB/Engine/Query/Pipeline/QueryPipe.cs

@@ -10,8 +10,8 @@ namespace LiteDB.Engine
     /// </summary>
     internal class QueryPipe : BasePipe
     {
-        public QueryPipe(TransactionService transaction, IDocumentLookup loader, SortDisk tempDisk, EnginePragmas pragmas)
-            : base(transaction, loader, tempDisk, pragmas)
+        public QueryPipe(TransactionService transaction, IDocumentLookup loader, SortDisk tempDisk, EnginePragmas pragmas, uint maxItemsCount)
+            : base(transaction, loader, tempDisk, pragmas, maxItemsCount)
         {
         }
 

LiteDB/Engine/Query/QueryExecutor.cs

@@ -14,6 +14,7 @@ internal class QueryExecutor
         private readonly EngineState _state;
         private readonly TransactionMonitor _monitor;
         private readonly SortDisk _sortDisk;
+        private readonly DiskService _disk;
         private readonly EnginePragmas _pragmas;
         private readonly CursorInfo _cursor;
         private readonly string _collection;
@@ -25,6 +26,7 @@ public QueryExecutor(
             EngineState state,
             TransactionMonitor monitor, 
             SortDisk sortDisk, 
+            DiskService disk,
             EnginePragmas pragmas, 
             string collection, 
             Query query, 
@@ -34,6 +36,7 @@ public QueryExecutor(
             _state = state;
             _monitor = monitor;
             _sortDisk = sortDisk;
+            _disk = disk;
             _pragmas = pragmas;
             _collection = collection;
             _query = query;
@@ -99,6 +102,8 @@ IEnumerable<BsonDocument> RunQuery()
 
                 var queryPlan = optimizer.ProcessQuery();
 
+                var plan = queryPlan.GetExecutionPlan();
+
                 // if execution is just to get explan plan, return as single document result
                 if (executionPlan)
                 {
@@ -115,10 +120,10 @@ IEnumerable<BsonDocument> RunQuery()
                 }
 
                 // get node list from query - distinct by dataBlock (avoid duplicate)
-                var nodes = queryPlan.Index.Run(snapshot.CollectionPage, new IndexService(snapshot, _pragmas.Collation));
+                var nodes = queryPlan.Index.Run(snapshot.CollectionPage, new IndexService(snapshot, _pragmas.Collation, _disk.MAX_ITEMS_COUNT));
 
                 // get current query pipe: normal or groupby pipe
-                var pipe = queryPlan.GetPipe(transaction, snapshot, _sortDisk, _pragmas);
+                var pipe = queryPlan.GetPipe(transaction, snapshot, _sortDisk, _pragmas, _disk.MAX_ITEMS_COUNT);
 
                 // start cursor elapsed timer
                 _cursor.Elapsed.Start();

LiteDB/Engine/Query/Structures/QueryPlan.cs

@@ -97,25 +97,25 @@ public QueryPlan(string collection)
         /// <summary>
         /// Select corrent pipe
         /// </summary>
-        public BasePipe GetPipe(TransactionService transaction, Snapshot snapshot, SortDisk tempDisk, EnginePragmas pragmas)
+        public BasePipe GetPipe(TransactionService transaction, Snapshot snapshot, SortDisk tempDisk, EnginePragmas pragmas, uint maxItemsCount)
         {
             if (this.GroupBy == null)
             {
-                return new QueryPipe(transaction, this.GetLookup(snapshot, pragmas), tempDisk, pragmas);
+                return new QueryPipe(transaction, this.GetLookup(snapshot, pragmas, maxItemsCount), tempDisk, pragmas, maxItemsCount);
             }
             else
             {
-                return new GroupByPipe(transaction, this.GetLookup(snapshot, pragmas), tempDisk, pragmas);
+                return new GroupByPipe(transaction, this.GetLookup(snapshot, pragmas, maxItemsCount), tempDisk, pragmas, maxItemsCount);
             }
         }
 
         /// <summary>
         /// Get corrent IDocumentLookup
         /// </summary>
-        public IDocumentLookup GetLookup(Snapshot snapshot, EnginePragmas pragmas)
+        public IDocumentLookup GetLookup(Snapshot snapshot, EnginePragmas pragmas, uint maxItemsCount)
         {
-            var data = new DataService(snapshot);
-            var indexer = new IndexService(snapshot, pragmas.Collation);
+            var data = new DataService(snapshot, maxItemsCount);
+            var indexer = new IndexService(snapshot, pragmas.Collation, maxItemsCount);
 
             // define document loader
             // if index are VirtualIndex - it's also lookup document