feat: full memtrie logic for range retain (#12130)

Next step on #12074.
Supporting all cases I came up with where nodes restructuring is
required.
Surprisingly, `squash_node` is enough to call. I only needed to
implement trivial cases which weren't possible for single key deletion.
I implemented the tests first, and majority of them failed before
changing the logic.
Each test is comparing naive approach with `retain_multi_range`.

### Notes
* I'm a bit scared that I didn't realise the need to squash Extension
before. Fortunately, `squash_node` handles that, but if you feel some
cases are not covered here, feel free to post suggestions!
* Reused + copypasted some Robin' tooling to generate interesting nodes
conversions.
* Note that test for reading "extra" child node is not required because
we always read all children.

### Next steps
* A bit more testing
* Similar logic for partial trie
* Generating intervals needed for resharding
* Use that to implement shard switch on chain

core/store/src/trie/mem/loading.rs

@@ -195,6 +195,7 @@ mod tests {
     };
     use crate::trie::mem::loading::load_trie_from_flat_state;
     use crate::trie::mem::lookup::memtrie_lookup;
+    use crate::trie::mem::nibbles_utils::{all_two_nibble_nibbles, multi_hex_to_nibbles};
     use crate::{DBCol, KeyLookupMode, NibbleSlice, ShardTries, Store, Trie, TrieUpdate};
     use near_primitives::congestion_info::CongestionInfo;
     use near_primitives::hash::CryptoHash;
@@ -300,80 +301,49 @@ mod tests {
         check_maybe_parallelize(keys, true);
     }
 
-    fn nibbles(hex: &str) -> Vec<u8> {
-        if hex == "_" {
-            return vec![];
-        }
-        assert!(hex.len() % 2 == 0);
-        hex::decode(hex).unwrap()
-    }
-
-    fn all_nibbles(hexes: &str) -> Vec<Vec<u8>> {
-        hexes.split_whitespace().map(|x| nibbles(x)).collect()
-    }
-
     #[test]
     fn test_memtrie_empty() {
         check(vec![]);
     }
 
     #[test]
     fn test_memtrie_root_is_leaf() {
-        check(all_nibbles("_"));
-        check(all_nibbles("00"));
-        check(all_nibbles("01"));
-        check(all_nibbles("ff"));
-        check(all_nibbles("0123456789abcdef"));
+        check(multi_hex_to_nibbles("_"));
+        check(multi_hex_to_nibbles("00"));
+        check(multi_hex_to_nibbles("01"));
+        check(multi_hex_to_nibbles("ff"));
+        check(multi_hex_to_nibbles("0123456789abcdef"));
     }
 
     #[test]
     fn test_memtrie_root_is_extension() {
-        check(all_nibbles("1234 13 14"));
-        check(all_nibbles("12345678 1234abcd"));
+        check(multi_hex_to_nibbles("1234 13 14"));
+        check(multi_hex_to_nibbles("12345678 1234abcd"));
     }
 
     #[test]
     fn test_memtrie_root_is_branch() {
-        check(all_nibbles("11 22"));
-        check(all_nibbles("12345678 22345678 32345678"));
-        check(all_nibbles("11 22 33 44 55 66 77 88 99 aa bb cc dd ee ff"));
+        check(multi_hex_to_nibbles("11 22"));
+        check(multi_hex_to_nibbles("12345678 22345678 32345678"));
+        check(multi_hex_to_nibbles("11 22 33 44 55 66 77 88 99 aa bb cc dd ee ff"));
     }
 
     #[test]
     fn test_memtrie_root_is_branch_with_value() {
-        check(all_nibbles("_ 11"));
+        check(multi_hex_to_nibbles("_ 11"));
     }
 
     #[test]
     fn test_memtrie_prefix_patterns() {
-        check(all_nibbles("10 21 2210 2221 222210 222221 22222210 22222221"));
-        check(all_nibbles("11111112 11111120 111112 111120 1112 1120 12 20"));
-        check(all_nibbles("11 1111 111111 11111111 1111111111 111111111111"));
-        check(all_nibbles("_ 11 1111 111111 11111111 1111111111 111111111111"));
+        check(multi_hex_to_nibbles("10 21 2210 2221 222210 222221 22222210 22222221"));
+        check(multi_hex_to_nibbles("11111112 11111120 111112 111120 1112 1120 12 20"));
+        check(multi_hex_to_nibbles("11 1111 111111 11111111 1111111111 111111111111"));
+        check(multi_hex_to_nibbles("_ 11 1111 111111 11111111 1111111111 111111111111"));
     }
 
     #[test]
     fn test_full_16ary_trees() {
-        check(all_nibbles(
-            "
-            00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f
-            10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f
-            20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f
-            30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d 3e 3f
-            40 41 42 43 44 45 46 47 48 49 4a 4b 4c 4d 4e 4f
-            50 51 52 53 54 55 56 57 58 59 5a 5b 5c 5d 5e 5f
-            60 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 6f
-            70 71 72 73 74 75 76 77 78 79 7a 7b 7c 7d 7e 7f
-            80 81 82 83 84 85 86 87 88 89 8a 8b 8c 8d 8e 8f
-            90 91 92 93 94 95 96 97 98 99 9a 9b 9c 9d 9e 9f
-            a0 a1 a2 a3 a4 a5 a6 a7 a8 a9 aa ab ac ad ae af
-            b0 b1 b2 b3 b4 b5 b6 b7 b8 b9 ba bb bc bd be bf
-            c0 c1 c2 c3 c4 c5 c6 c7 c8 c9 ca cb cc cd ce cf
-            d0 d1 d2 d3 d4 d5 d6 d7 d8 d9 da db dc dd de df
-            e0 e1 e2 e3 e4 e5 e6 e7 e8 e9 ea eb ec ed ee ef
-            f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff
-        ",
-        ))
+        check(all_two_nibble_nibbles())
     }
 
     #[test]

core/store/src/trie/mem/mod.rs

@@ -7,6 +7,8 @@ pub mod loading;
 mod lookup;
 pub mod mem_tries;
 pub mod metrics;
+#[cfg(test)]
+pub(crate) mod nibbles_utils;
 pub mod node;
 mod parallel_loader;
 pub mod resharding;

core/store/src/trie/mem/nibbles_utils.rs

@@ -0,0 +1,46 @@
+/// Utilties for generating vectors of nibbles from human-readable strings.
+///
+/// Input for a single vector is a hex string, e.g. 5da3593f.
+/// It has even length, as tries support only keys in bytes, thus keys of
+/// odd nibble length do not occur.
+/// Each symbol is interpreted as a nibble (half-byte).
+/// Result is a vector of decoded hexes as nibbles, e.g.
+/// [5, 13, 10, 3, 5, 9, 3, 15].
+
+pub(crate) fn hex_to_nibbles(hex: &str) -> Vec<u8> {
+    if hex == "_" {
+        return vec![];
+    }
+    assert!(hex.len() % 2 == 0);
+    hex::decode(hex).unwrap()
+}
+
+/// Converts a string of hex strings separated by whitespaces into a vector of
+/// vectors of nibbles. For example, "01 02 10" is converted to
+/// [[0, 1], [0, 2], [1, 0]].
+pub(crate) fn multi_hex_to_nibbles(hexes: &str) -> Vec<Vec<u8>> {
+    hexes.split_whitespace().map(|x| hex_to_nibbles(x)).collect()
+}
+
+pub(crate) fn all_two_nibble_nibbles() -> Vec<Vec<u8>> {
+    multi_hex_to_nibbles(
+        "
+        00 01 02 03 04 05 06 07 08 09 0a 0b 0c 0d 0e 0f
+        10 11 12 13 14 15 16 17 18 19 1a 1b 1c 1d 1e 1f
+        20 21 22 23 24 25 26 27 28 29 2a 2b 2c 2d 2e 2f
+        30 31 32 33 34 35 36 37 38 39 3a 3b 3c 3d 3e 3f
+        40 41 42 43 44 45 46 47 48 49 4a 4b 4c 4d 4e 4f
+        50 51 52 53 54 55 56 57 58 59 5a 5b 5c 5d 5e 5f
+        60 61 62 63 64 65 66 67 68 69 6a 6b 6c 6d 6e 6f
+        70 71 72 73 74 75 76 77 78 79 7a 7b 7c 7d 7e 7f
+        80 81 82 83 84 85 86 87 88 89 8a 8b 8c 8d 8e 8f
+        90 91 92 93 94 95 96 97 98 99 9a 9b 9c 9d 9e 9f
+        a0 a1 a2 a3 a4 a5 a6 a7 a8 a9 aa ab ac ad ae af
+        b0 b1 b2 b3 b4 b5 b6 b7 b8 b9 ba bb bc bd be bf
+        c0 c1 c2 c3 c4 c5 c6 c7 c8 c9 ca cb cc cd ce cf
+        d0 d1 d2 d3 d4 d5 d6 d7 d8 d9 da db dc dd de df
+        e0 e1 e2 e3 e4 e5 e6 e7 e8 e9 ea eb ec ed ee ef
+        f0 f1 f2 f3 f4 f5 f6 f7 f8 f9 fa fb fc fd fe ff
+    ",
+    )
+}