Merge pull request #155 from Syndica/19/snapshot-loading-fixes

fix(accounts-db): inc snapshot + status cache verification
Syndica · Jun 11, 2024 · c8b88e0 · c8b88e0
2 parents 0298d20 + 6c2f94d
commit c8b88e0
Show file tree

Hide file tree

Showing 5 changed files with 35 additions and 11 deletions.
diff --git a/src/accountsdb/db.zig b/src/accountsdb/db.zig
@@ -15,6 +15,7 @@ const AccountFileInfo = @import("../accountsdb/snapshots.zig").AccountFileInfo;
 const AccountFile = @import("../accountsdb/accounts_file.zig").AccountFile;
 const FileId = @import("../accountsdb/accounts_file.zig").FileId;
 const AccountInFile = @import("../accountsdb/accounts_file.zig").AccountInFile;
+const Blake3 = std.crypto.hash.Blake3;
 
 const ThreadPool = @import("../sync/thread_pool.zig").ThreadPool;
 
@@ -763,11 +764,24 @@ pub const AccountsDB = struct {
                 } orelse continue;
                 const result = try self.getAccountHashAndLamportsFromRef(max_slot_ref);
 
-                // only include non-zero lamport accounts (for full snapshots)
                 const lamports = result.lamports;
-                if (config == .FullAccountHash and lamports == 0) continue;
+                var account_hash = result.hash;
+                if (lamports == 0) {
+                    switch (config) {
+                        // for full snapshots, only include non-zero lamport accounts
+                        .FullAccountHash => continue,
+                        // zero-lamport accounts for incrementals = hash(pubkey)
+                        .IncrementalAccountHash => Blake3.hash(&key.data, &account_hash.data, .{}),
+                    }
+                } else {
+                    // hashes arent always stored correctly in snapshots
+                    if (account_hash.order(&Hash.default()) == .eq) {
+                        const account = try self.getAccountFromRef(max_slot_ref);
+                        account_hash = account.hash(&key);
+                    }
+                }
 
-                hashes.appendAssumeCapacity(result.hash);
+                hashes.appendAssumeCapacity(account_hash);
                 local_total_lamports += lamports;
             }
 

diff --git a/src/accountsdb/snapshots.zig b/src/accountsdb/snapshots.zig
@@ -676,6 +676,7 @@ pub const StatusCache = struct {
         bincode.free(self.bank_slot_deltas.allocator, self.*);
     }
 
+    /// [verify_slot_deltas](https://github.com/anza-xyz/agave/blob/ed500b5afc77bc78d9890d96455ea7a7f28edbf9/runtime/src/snapshot_bank_utils.rs#L709)
     pub fn validate(
         self: *const StatusCache,
         allocator: std.mem.Allocator,
@@ -710,14 +711,21 @@ pub const StatusCache = struct {
             return error.SlotHistoryMismatch;
         }
         for (slots_seen.keys()) |slot| {
-            if (slot_history.check(slot) != sysvars.SlotCheckResult.Found) {
+            if (slot_history.check(slot) != .Found) {
                 return error.SlotNotFoundInHistory;
             }
         }
-        for (slot_history.oldest()..slot_history.newest()) |slot| {
-            if (!slots_seen.contains(slot)) {
-                return error.SlotNotFoundInStatusCache;
+
+        var slots_checked: u32 = 0;
+        var slot = slot_history.newest();
+        while (slot >= slot_history.oldest() and slots_checked != MAX_CACHE_ENTRIES) {
+            if (slot_history.check(slot) == .Found) {
+                slots_checked += 1;
+                if (!slots_seen.contains(slot)) {
+                    return error.SlotNotFoundInStatusCache;
+                }
             }
+            slot -= 1;
         }
     }
 };

diff --git a/src/common/merkle_tree.zig b/src/common/merkle_tree.zig
@@ -36,7 +36,7 @@ pub const NestedHashTree = struct {
         var i: usize = 0;
         while (i < self.hashes.len) {
             const nested_len = self.hashes[i].items.len;
-            if ((search_index + nested_len) > index) {
+            if (search_index + nested_len > index) {
                 const index_in_nested = index - search_index;
                 return &self.hashes[i].items[index_in_nested];
             } else {

diff --git a/src/core/pubkey.zig b/src/core/pubkey.zig
@@ -81,7 +81,10 @@ pub const Pubkey = extern struct {
     }
 
     pub fn format(self: @This(), comptime _: []const u8, _: std.fmt.FormatOptions, writer: anytype) !void {
-        return writer.print("{s}", .{self.string()});
+        var dest: [44]u8 = undefined;
+        @memset(&dest, 0);
+        const written = encoder.encode(&self.data, &dest) catch return error.EncodingError;
+        return writer.print("{s}", .{dest[0..written]});
     }
 
     pub fn isDefault(self: *const Self) bool {

diff --git a/src/utils/tar.zig b/src/utils/tar.zig
@@ -76,8 +76,7 @@ pub fn parallelUntarToFileSystem(
         thread_pool.deinit();
     }
 
-    logger.infof("using {d} threads to unpack snapshot\n", .{n_threads});
-
+    logger.infof("using {d} threads to unpack snapshot", .{n_threads});
     const tasks = try UnTarTask.init(allocator, n_threads);
     defer allocator.free(tasks);