From 6e037f06d6d3b1d55d6745d4f55f91e5216034f2 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Tue, 24 Sep 2024 15:09:35 -0400 Subject: [PATCH 1/4] khepri_machine: Upgrade state versions incrementally Refactoring `convert_state/3` this way allows us to write `convert_state1/3` so that it always acts on a single version upgrade. --- src/khepri_machine.erl | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/src/khepri_machine.erl b/src/khepri_machine.erl index 5e016159..529e80f3 100644 --- a/src/khepri_machine.erl +++ b/src/khepri_machine.erl @@ -2334,9 +2334,17 @@ make_virgin_state(Params) -> %% %% @private -convert_state(State, MacVer, MacVer) -> +convert_state(State, OldMacVer, NewMacVer) -> + lists:foldl( + fun(N, State1) -> + OldMacVer1 = N, + NewMacVer1 = erlang:min(N + 1, NewMacVer), + convert_state1(State1, OldMacVer1, NewMacVer1) + end, State, lists:seq(OldMacVer, NewMacVer)). + +convert_state1(State, MacVer, MacVer) -> State; -convert_state(State, 0, 1) -> +convert_state1(State, 0, 1) -> %% To go from version 0 to version 1, we add the `dedups' fields at the %% end of the record. The default value is an empty map. ?assert(khepri_machine_v0:is_state(State)), From d38d0533f2a434162fce771f2d2389ee47bfb1e4 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Mon, 30 Sep 2024 12:15:35 -0400 Subject: [PATCH 2/4] khepri_tree: Remove exposure of the keep-while cond reverse index This type is otherwise completely private to the `khepri_tree` module and type. We'll take advantage of its private nature in the child commits to change its representation. This value was only exposed for a single test. --- src/khepri_machine.erl | 15 +-------------- src/khepri_machine_v0.erl | 2 +- src/khepri_tree.erl | 9 ++++++++- test/keep_while_conditions.erl | 5 ----- test/pattern_tree.erl | 8 ++++---- 5 files changed, 14 insertions(+), 25 deletions(-) diff --git a/src/khepri_machine.erl b/src/khepri_machine.erl index 529e80f3..4ed3f3c9 100644 --- a/src/khepri_machine.erl +++ b/src/khepri_machine.erl @@ -101,7 +101,6 @@ get_tree/1, get_root/1, get_keep_while_conds/1, - get_keep_while_conds_revidx/1, get_triggers/1, get_emitted_triggers/1, get_projections/1, @@ -156,7 +155,7 @@ %% State machine's internal state record. -record(khepri_machine, {config = #config{} :: khepri_machine:machine_config(), - tree = #tree{} :: khepri_tree:tree(), + tree = khepri_tree:new() :: khepri_tree:tree(), triggers = #{} :: khepri_machine:triggers_map(), emitted_triggers = [] :: [khepri_machine:triggered()], projections = khepri_pattern_tree:empty() :: @@ -2160,18 +2159,6 @@ get_keep_while_conds(State) -> #tree{keep_while_conds = KeepWhileConds} = get_tree(State), KeepWhileConds. --spec get_keep_while_conds_revidx(State) -> KeepWhileCondsRevIdx when - State :: khepri_machine:state(), - KeepWhileCondsRevIdx :: khepri_tree:keep_while_conds_revidx(). -%% @doc Returns the `keep_while' conditions reverse index in the tree from the -%% given state. -%% -%% @private - -get_keep_while_conds_revidx(State) -> - #tree{keep_while_conds_revidx = KeepWhileCondsRevIdx} = get_tree(State), - KeepWhileCondsRevIdx. - -spec get_triggers(State) -> Triggers when State :: khepri_machine:state(), Triggers :: khepri_machine:triggers_map(). diff --git a/src/khepri_machine_v0.erl b/src/khepri_machine_v0.erl index 5295c817..7e32f68d 100644 --- a/src/khepri_machine_v0.erl +++ b/src/khepri_machine_v0.erl @@ -29,7 +29,7 @@ -record(khepri_machine, {config = #config{} :: khepri_machine:machine_config(), - tree = #tree{} :: khepri_tree:tree(), + tree = khepri_tree:new() :: khepri_tree:tree(), triggers = #{} :: #{khepri:trigger_id() => #{sproc := khepri_path:native_path(), diff --git a/src/khepri_tree.erl b/src/khepri_tree.erl index 65e99960..af46374f 100644 --- a/src/khepri_tree.erl +++ b/src/khepri_tree.erl @@ -19,7 +19,9 @@ -include("src/khepri_error.hrl"). -include("src/khepri_tree.hrl"). --export([are_keep_while_conditions_met/2, +-export([new/0, + + are_keep_while_conditions_met/2, collect_node_props_cb/3, count_node_cb/3, @@ -74,6 +76,11 @@ %% Tree node functions. %% ------------------------------------------------------------------- +-spec new() -> tree(). + +new() -> + #tree{}. + -spec create_node_record(Payload) -> Node when Payload :: khepri_payload:payload(), Node :: tree_node(). diff --git a/test/keep_while_conditions.erl b/test/keep_while_conditions.erl index 1c5cee7a..7dec366d 100644 --- a/test/keep_while_conditions.erl +++ b/test/keep_while_conditions.erl @@ -18,7 +18,6 @@ %% khepri:get_root/1 is unexported when compiled without `-DTEST'. Likewise %% for: %% - `khepri_machine:get_keep_while_conds/1' -%% - `khepri_machine:get_keep_while_conds_revidx/1' -dialyzer(no_missing_calls). are_keep_while_conditions_met_test() -> @@ -70,7 +69,6 @@ insert_when_keep_while_true_test() -> {S1, Ret, SE} = khepri_machine:apply(?META, Command, S0), Root = khepri_machine:get_root(S1), KeepWhileConds = khepri_machine:get_keep_while_conds(S1), - KeepWhileCondsRevIdx = khepri_machine:get_keep_while_conds_revidx(S1), ?assertEqual( #node{ @@ -90,9 +88,6 @@ insert_when_keep_while_true_test() -> ?assertEqual( #{[baz] => KeepWhile}, KeepWhileConds), - ?assertEqual( - #{[foo] => #{[baz] => ok}}, - KeepWhileCondsRevIdx), ?assertEqual({ok, #{[baz] => #{}}}, Ret), ?assertEqual([], SE). diff --git a/test/pattern_tree.erl b/test/pattern_tree.erl index 65dd5897..99646725 100644 --- a/test/pattern_tree.erl +++ b/test/pattern_tree.erl @@ -70,10 +70,10 @@ fold_finds_all_patterns_matching_a_path_test() -> khepri_tree:insert_or_update_node( Tree0, Path, TreePayload, #{}, #{}), Tree - end, #tree{}, [[stock, wood, <<"oak">>], - [stock, wood, <<"birch">>], - [stock, metal, <<"iron">>], - []]), + end, khepri_tree:new(), [[stock, wood, <<"oak">>], + [stock, wood, <<"birch">>], + [stock, metal, <<"iron">>], + []]), PathPatterns = [[stock, wood, <<"oak">>], %% 1 [stock, wood, <<"birch">>], %% 2 From 13d349bf3ff690b4f42ea36adcb1c291aff8d806 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Mon, 30 Sep 2024 12:27:30 -0400 Subject: [PATCH 3/4] Add a prefix tree type for quick prefix lookups This will be used in the child commit to replace the keep-while conditions reverse index which currently uses a map of paths to watchers. The design is very similar to- but simpler than the existing trees in `khepri_tree` and `khepri_pattern_tree`. --- src/khepri_prefix_tree.erl | 179 +++++++++++++++++++++++++++++++++++++ test/prefix_tree.erl | 109 ++++++++++++++++++++++ 2 files changed, 288 insertions(+) create mode 100644 src/khepri_prefix_tree.erl create mode 100644 test/prefix_tree.erl diff --git a/src/khepri_prefix_tree.erl b/src/khepri_prefix_tree.erl new file mode 100644 index 00000000..3d27b54a --- /dev/null +++ b/src/khepri_prefix_tree.erl @@ -0,0 +1,179 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright © 2024 Broadcom. All Rights Reserved. The term "Broadcom" refers +%% to Broadcom Inc. and/or its subsidiaries. +%% + +-module(khepri_prefix_tree). + +%% This module defines a tree that associates paths with optional payloads and +%% specializes in lookup of tree nodes by a prefixing path. +%% +%% This tree is similar to {@link khepri_pattern_tree} but the path components +%% in this tree must be {@link khepri_path:node_id()}s rather than pattern +%% components. +%% +%% This tree is also similar to the main tree type {@link khepri_tree} but it +%% is simpler: it does not support keep-while conditions or properties for +%% tree nodes. This type is used within {@link khepri_tree} for the reverse +%% index of keep-while conditions. +%% +%% See https://en.wikipedia.org/wiki/Trie. + +-include_lib("stdlib/include/assert.hrl"). + +-include("src/khepri_payload.hrl"). + +-type child_nodes(Payload) :: #{khepri_path:node_id() => tree(Payload)}. + +-record(prefix_tree, {payload = ?NO_PAYLOAD, + child_nodes = #{}}). + +-opaque tree(Payload) :: #prefix_tree{payload :: Payload | ?NO_PAYLOAD, + child_nodes :: child_nodes(Payload)}. + +-export_type([tree/1]). + +-export([empty/0, + is_prefix_tree/1, + from_map/1, + fold_prefixes_of/4, + find_path/2, + update/3]). + +-spec empty() -> tree(_). +%% @doc Returns a new empty tree. +%% +%% @see tree(). + +empty() -> + #prefix_tree{}. + +-spec is_prefix_tree(tree(_)) -> true; + (term()) -> false. +%% @doc Determines whether the given term is a prefix tree. + +is_prefix_tree(#prefix_tree{}) -> + true; +is_prefix_tree(_) -> + false. + +-spec from_map(Map) -> Tree when + Map :: #{khepri_path:native_path() => Payload}, + Tree :: khepri_prefix_tree:tree(Payload), + Payload :: term(). +%% @doc Converts a map of paths to payloads into a prefix tree. + +from_map(Map) when is_map(Map) -> + maps:fold( + fun(Path, Payload, Tree) -> + update( + fun(Payload0) -> + %% Map keys are unique so this node in the prefix + %% tree must have no payload. + ?assertEqual(?NO_PAYLOAD, Payload0), + Payload + end, Path, Tree) + end, empty(), Map). + +-spec fold_prefixes_of(Fun, Acc, Path, Tree) -> Ret when + Fun :: fun((Payload, Acc) -> Acc1), + Acc :: term(), + Acc1 :: term(), + Path :: khepri_path:native_path(), + Tree :: khepri_prefix_tree:tree(Payload), + Payload :: term(), + Ret :: Acc1. +%% @doc Folds over all nodes in the tree which are prefixed by the given `Path' +%% building an accumulated value with the given fold function and initial +%% accumulator. + +fold_prefixes_of(Fun, Acc, Path, Tree) when is_function(Fun, 2) -> + fold_prefixes_of1(Fun, Acc, Path, Tree). + +fold_prefixes_of1( + Fun, Acc, [], #prefix_tree{payload = Payload, child_nodes = ChildNodes}) -> + Acc1 = case Payload of + ?NO_PAYLOAD -> + Acc; + _ -> + Fun(Payload, Acc) + end, + maps:fold( + fun(_Component, Subtree, Acc2) -> + fold_prefixes_of1(Fun, Acc2, [], Subtree) + end, Acc1, ChildNodes); +fold_prefixes_of1( + Fun, Acc, [Component | Rest], #prefix_tree{child_nodes = ChildNodes}) -> + case maps:find(Component, ChildNodes) of + {ok, Subtree} -> + fold_prefixes_of1(Fun, Acc, Rest, Subtree); + error -> + Acc + end. + +-spec find_path(Path, Tree) -> Ret when + Path :: khepri_path:native_path(), + Tree :: khepri_prefix_tree:tree(Payload), + Payload :: term(), + Ret :: {ok, Payload} | error. +%% @doc Returns the payload associated with a path in the tree. +%% +%% @returns `{ok, Payload}' where `Payload' is associated with the given path +%% or `error' if the path is not associated with a payload in the given tree. + +find_path(Path, Tree) -> + find_path1(Path, Tree). + +find_path1([], #prefix_tree{payload = Payload}) -> + case Payload of + ?NO_PAYLOAD -> + error; + _ -> + {ok, Payload} + end; +find_path1([Component | Rest], #prefix_tree{child_nodes = ChildNodes}) -> + case maps:find(Component, ChildNodes) of + {ok, Subtree} -> + find_path1(Rest, Subtree); + error -> + error + end. + +-spec update(Fun, Path, Tree) -> Ret when + Fun :: fun((Payload | ?NO_PAYLOAD) -> Payload | ?NO_PAYLOAD), + Path :: khepri_path:native_path(), + Tree :: khepri_prefix_tree:tree(Payload), + Payload :: term(), + Ret :: khepri_prefix_tree:tree(Payload). +%% @doc Updates a given path in the tree. +%% +%% This function can be used to create, update or delete tree nodes. If the +%% tree node does not exist for the given path, the update function is passed +%% `?NO_PAYLOAD'. If the update function returns `?NO_PAYLOAD' then the tree +%% node and all of its ancestors which do not have a payload or children are +%% removed. +%% +%% The update function is also be passed `?NO_PAYLOAD' if a tree node exists +%% but does not have a payload: being passed `?NO_PAYLOAD' is not a reliable +%% sign that a tree node did not exist prior to an update. + +update(Fun, Path, Tree) -> + update1(Fun, Path, Tree). + +update1(Fun, [], #prefix_tree{payload = Payload} = Tree) -> + Tree#prefix_tree{payload = Fun(Payload)}; +update1( + Fun, [Component | Rest], #prefix_tree{child_nodes = ChildNodes} = Tree) -> + Subtree = maps:get(Component, ChildNodes, khepri_prefix_tree:empty()), + ChildNodes1 = case update1(Fun, Rest, Subtree) of + #prefix_tree{payload = ?NO_PAYLOAD, child_nodes = C} + when C =:= #{} -> + %% Drop unused branches. + maps:remove(Component, ChildNodes); + Subtree1 -> + maps:put(Component, Subtree1, ChildNodes) + end, + Tree#prefix_tree{child_nodes = ChildNodes1}. diff --git a/test/prefix_tree.erl b/test/prefix_tree.erl new file mode 100644 index 00000000..c8bccf59 --- /dev/null +++ b/test/prefix_tree.erl @@ -0,0 +1,109 @@ +%% This Source Code Form is subject to the terms of the Mozilla Public +%% License, v. 2.0. If a copy of the MPL was not distributed with this +%% file, You can obtain one at https://mozilla.org/MPL/2.0/. +%% +%% Copyright © 2024 Broadcom. All Rights Reserved. The term "Broadcom" refers +%% to Broadcom Inc. and/or its subsidiaries. +%% + +-module(prefix_tree). + +-include_lib("eunit/include/eunit.hrl"). +-include("src/khepri_payload.hrl"). + +update_passes_current_payload_test() -> + Path = [stock, wood, <<"oak">>], + Payload = 100, + Tree0 = khepri_prefix_tree:empty(), + Tree1 = khepri_prefix_tree:update( + fun(Payload0) -> + ?assertEqual(Payload0, ?NO_PAYLOAD), + Payload + end, Path, Tree0), + _ = khepri_prefix_tree:update( + fun(Payload1) -> + ?assertEqual(Payload, Payload1), + ?NO_PAYLOAD + end, Path, Tree1), + ok. + +deletion_prunes_empty_branches_test() -> + %% When deleting a node in the tree, the branch leading to that node is + %% removed while non-empty. + Path = [stock, wood, <<"oak">>], + Payload = 100, + Tree0 = khepri_prefix_tree:from_map(#{Path => Payload}), + + %% This branch has a child (`<<"oak">>') so it will not be pruned. + Tree1 = khepri_prefix_tree:update( + fun(Payload0) -> + ?assertEqual(Payload0, ?NO_PAYLOAD), + ?NO_PAYLOAD + end, [stock, wood], Tree0), + ?assertNotEqual(Tree1, khepri_prefix_tree:empty()), + + Tree2 = khepri_prefix_tree:update( + fun(Payload0) -> + ?assertEqual(Payload0, Payload), + ?NO_PAYLOAD + end, Path, Tree1), + ?assertEqual(Tree2, khepri_prefix_tree:empty()), + + %% If an update doesn't store a payload then the branch is pruned. So this + %% update acts as a no-op and the tree remains empty: + Tree3 = khepri_prefix_tree:update( + fun(Payload0) -> + ?assertEqual(Payload0, ?NO_PAYLOAD), + ?NO_PAYLOAD + end, Path, Tree2), + ?assertEqual(Tree3, khepri_prefix_tree:empty()), + + ok. + +find_path_test() -> + Path1 = [stock, wood, <<"oak">>], + Path2 = [stock, wood, <<"birch">>], + Path3 = [stock, metal, <<"iron">>], + Tree = khepri_prefix_tree:from_map( + #{Path1 => 100, Path2 => 150, Path3 => 10}), + + ?assertEqual({ok, 100}, khepri_prefix_tree:find_path(Path1, Tree)), + ?assertEqual({ok, 150}, khepri_prefix_tree:find_path(Path2, Tree)), + ?assertEqual({ok, 10}, khepri_prefix_tree:find_path(Path3, Tree)), + + ?assertEqual( + error, + khepri_prefix_tree:find_path( + [stock, wood, <<"oak">>, <<"leaves">>], Tree)), + ?assertEqual(error, khepri_prefix_tree:find_path([stock, wood], Tree)), + ?assertEqual(error, khepri_prefix_tree:find_path([stock, metal], Tree)), + + ?assertEqual( + error, + khepri_prefix_tree:find_path(Path1, khepri_prefix_tree:empty())), + + ok. + +fold_prefixes_of_test() -> + Path1 = [stock, wood, <<"oak">>], + Path2 = [stock, wood, <<"birch">>], + Path3 = [stock, metal, <<"iron">>], + Tree = khepri_prefix_tree:from_map( + #{Path1 => 100, Path2 => 150, Path3 => 10}), + + ?assertEqual([100], collect_prefixes_of(Path1, Tree)), + ?assertEqual([100, 150], collect_prefixes_of([stock, wood], Tree)), + ?assertEqual([10], collect_prefixes_of([stock, metal], Tree)), + ?assertEqual([10, 100, 150], collect_prefixes_of([stock], Tree)), + ?assertEqual([10, 100, 150], collect_prefixes_of([], Tree)), + + ?assertEqual( + [], + collect_prefixes_of([stock, wood, <<"oak">>, <<"leaves">>], Tree)), + + ok. + +collect_prefixes_of(Path, Tree) -> + Payloads = khepri_prefix_tree:fold_prefixes_of( + fun(Payload, Acc) -> [Payload | Acc] end, [], Path, Tree), + lists:sort(Payloads). From 1d74ff8b02321b8d89c3099ec93403316a9c1393 Mon Sep 17 00:00:00 2001 From: Michael Davis Date: Mon, 30 Sep 2024 12:29:11 -0400 Subject: [PATCH 4/4] Replace keep-while cond reverse index with a prefix tree When deleting tree nodes in the `khepri_tree` we lookup in the reverse index to find any conditions that are associated to paths which are prefixes of the deleted path. Prior to this commit we folded over the keep-while conditions reverse index - a map - and used `lists:prefix/2` to find prefixing paths. In a store with many nodes and tracking many keep-while conditions this can become very expensive while deleting many nodes at once. The parent commit introduced a prefix tree type which allows quick lookup, given a path, of any tree nodes associated with a path which is a prefix of the given path. We set a version 2 for khepri_machine which upgrades the reverse index use to this new type. Because the reverse index is private to the khepri_tree type I have avoided introducing versioning for the `khepri_tree` module. Instead the reverse index can either be a map - as in prior versions - or a prefix tree. We act on the reverse index using the appropriate functions for the type, which we detect at runtime. --- src/khepri_machine.erl | 30 +++++++-- src/khepri_machine_v0.erl | 2 +- src/khepri_tree.erl | 132 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 151 insertions(+), 13 deletions(-) diff --git a/src/khepri_machine.erl b/src/khepri_machine.erl index 4ed3f3c9..1c8d2e36 100644 --- a/src/khepri_machine.erl +++ b/src/khepri_machine.erl @@ -42,6 +42,16 @@ %% %% %% +%% +%% 2 +%% +%%
    +%%
  • Changed the data structure for the reverse index used to track +%% keep-while conditions to be a prefix tree (see {@link khepri_prefix_tree}). +%%
    • +%%
+%% +%% %% -module(khepri_machine). @@ -167,6 +177,11 @@ -opaque state_v1() :: #khepri_machine{}. %% State of this Ra state machine, version 1. +%% +%% Note that this type is used also for machine version 2. Machine version 2 +%% changes the type of an opaque member of the {@link khepri_tree} record and +%% doesn't need any changes to the `khepri_machine' type. See the moduledoc of +%% this module for more information about version 2. -type state() :: state_v1() | khepri_machine_v0:state(). %% State of this Ra state machine. @@ -1635,17 +1650,18 @@ overview(State) -> keep_while_conds => KeepWhileConds}. -spec version() -> MacVer when - MacVer :: 1. + MacVer :: 2. %% @doc Returns the state machine version. version() -> - 1. + 2. -spec which_module(MacVer) -> Module when - MacVer :: 1 | 0, + MacVer :: 0..2, Module :: ?MODULE. %% @doc Returns the state machine module corresponding to the given version. +which_module(2) -> ?MODULE; which_module(1) -> ?MODULE; which_module(0) -> ?MODULE. @@ -2313,7 +2329,7 @@ make_virgin_state(Params) -> -endif. -spec convert_state(OldState, OldMacVer, NewMacVer) -> NewState when - OldState :: khepri_machine_v0:state(), + OldState :: khepri_machine:state(), OldMacVer :: ra_machine:version(), NewMacVer :: ra_machine:version(), NewState :: khepri_machine:state(). @@ -2339,7 +2355,11 @@ convert_state1(State, 0, 1) -> Fields1 = Fields0 ++ [#{}], State1 = list_to_tuple(Fields1), ?assert(is_state(State1)), - State1. + State1; +convert_state1(State, 1, 2) -> + Tree = get_tree(State), + Tree1 = khepri_tree:convert_tree(Tree, 1, 2), + set_tree(State, Tree1). -spec update_projections(OldState, NewState) -> ok when OldState :: khepri_machine:state(), diff --git a/src/khepri_machine_v0.erl b/src/khepri_machine_v0.erl index 7e32f68d..0f6b0903 100644 --- a/src/khepri_machine_v0.erl +++ b/src/khepri_machine_v0.erl @@ -29,7 +29,7 @@ -record(khepri_machine, {config = #config{} :: khepri_machine:machine_config(), - tree = khepri_tree:new() :: khepri_tree:tree(), + tree = khepri_tree:new() :: khepri_tree:tree_v0(), triggers = #{} :: #{khepri:trigger_id() => #{sproc := khepri_path:native_path(), diff --git a/src/khepri_tree.erl b/src/khepri_tree.erl index af46374f..3b813526 100644 --- a/src/khepri_tree.erl +++ b/src/khepri_tree.erl @@ -31,22 +31,42 @@ delete_matching_nodes/4, insert_or_update_node/5, does_path_match/3, - walk_down_the_tree/5]). + walk_down_the_tree/5, + + convert_tree/3]). -type tree_node() :: #node{}. %% A node in the tree structure. --type tree() :: #tree{}. +-type tree_v0() :: #tree{keep_while_conds_revidx :: + khepri_tree:keep_while_conds_revidx_v0()}. +-type tree_v1() :: #tree{keep_while_conds_revidx :: + khepri_tree:keep_while_conds_revidx_v1()}. + +-type tree() :: tree_v0() | tree_v1(). -type keep_while_conds_map() :: #{khepri_path:native_path() => khepri_condition:native_keep_while()}. %% Per-node `keep_while' conditions. --type keep_while_conds_revidx() :: #{khepri_path:native_path() => - #{khepri_path:native_path() => ok}}. -%% Internal reverse index of the keep_while conditions. If node A depends on a -%% condition on node B, then this reverse index will have a "node B => node A" -%% entry. +-type keep_while_conds_revidx_v0() :: #{khepri_path:native_path() => + #{khepri_path:native_path() => ok}}. + +-type keep_while_conds_revidx_v1() :: khepri_prefix_tree:tree( + #{khepri_path:native_path() => ok}). + +-type keep_while_conds_revidx() :: keep_while_conds_revidx_v0() | + keep_while_conds_revidx_v1(). +%% Internal reverse index of the keep_while conditions. +%% +%% If node A depends on a condition on node B, then this reverse index will +%% have a "node B => node A" association. The version 0 of this type used a map +%% and folded over the entries in the map using `lists:prefix/2' to find +%% matching conditions. In version 1 this type was replaced with a prefix tree +%% which improves lookup time when the reverse index contains many entries. +%% +%% This type should be treated as opaque. It is not marked as such because of +%% limitations in the dialyzer. -type applied_changes() :: #{khepri_path:native_path() => khepri:node_props() | delete}. @@ -67,8 +87,12 @@ -type ok(Type1, Type2, Type3) :: {ok, Type1, Type2, Type3}. -export_type([tree_node/0, + tree_v0/0, + tree_v1/0, tree/0, keep_while_conds_map/0, + keep_while_conds_revidx_v0/0, + keep_while_conds_revidx_v1/0, keep_while_conds_revidx/0, applied_changes/0]). @@ -314,6 +338,19 @@ update_keep_while_conds(Tree, Watcher, KeepWhile) -> KeepWhile :: khepri_condition:native_keep_while(). update_keep_while_conds_revidx( + #tree{keep_while_conds_revidx = KeepWhileCondsRevIdx} = Tree, + Watcher, KeepWhile) -> + case is_v1_keep_while_conds_revidx(KeepWhileCondsRevIdx) of + true -> + update_keep_while_conds_revidx_v1(Tree, Watcher, KeepWhile); + false -> + update_keep_while_conds_revidx_v0(Tree, Watcher, KeepWhile) + end. + +is_v1_keep_while_conds_revidx(KeepWhileCondsRevIdx) -> + khepri_prefix_tree:is_prefix_tree(KeepWhileCondsRevIdx). + +update_keep_while_conds_revidx_v0( #tree{keep_while_conds = KeepWhileConds, keep_while_conds_revidx = KeepWhileCondsRevIdx} = Tree, Watcher, KeepWhile) -> @@ -338,6 +375,37 @@ update_keep_while_conds_revidx( end, KeepWhileCondsRevIdx1, KeepWhile), Tree#tree{keep_while_conds_revidx = KeepWhileCondsRevIdx2}. +update_keep_while_conds_revidx_v1( + #tree{keep_while_conds = KeepWhileConds, + keep_while_conds_revidx = KeepWhileCondsRevIdx} = Tree, + Watcher, KeepWhile) -> + %% First, clean up reversed index where a watched path isn't watched + %% anymore in the new keep_while. + OldWatcheds = maps:get(Watcher, KeepWhileConds, #{}), + KeepWhileCondsRevIdx1 = maps:fold( + fun(Watched, _, KWRevIdx) -> + khepri_prefix_tree:update( + fun(Watchers) -> + Watchers1 = maps:remove( + Watcher, Watchers), + case maps:size(Watchers1) of + 0 -> ?NO_PAYLOAD; + _ -> Watchers1 + end + end, Watched, KWRevIdx) + end, KeepWhileCondsRevIdx, OldWatcheds), + %% Then, record the watched paths. + KeepWhileCondsRevIdx2 = maps:fold( + fun(Watched, _, KWRevIdx) -> + khepri_prefix_tree:update( + fun (?NO_PAYLOAD) -> + #{Watcher => ok}; + (Watchers) -> + Watchers#{Watcher => ok} + end, Watched, KWRevIdx) + end, KeepWhileCondsRevIdx1, KeepWhile), + Tree#tree{keep_while_conds_revidx = KeepWhileCondsRevIdx2}. + %% ------------------------------------------------------------------- %% Find matching nodes. %% ------------------------------------------------------------------- @@ -1294,6 +1362,16 @@ eval_keep_while_conditions( %% %% Those modified in AppliedChanges must be evaluated again to decide %% if they should be removed. + case is_v1_keep_while_conds_revidx(KeepWhileCondsRevIdx) of + true -> + eval_keep_while_conditions_v1(Tree, AppliedChanges); + false -> + eval_keep_while_conditions_v0(Tree, AppliedChanges) + end. + +eval_keep_while_conditions_v0( + #tree{keep_while_conds_revidx = KeepWhileCondsRevIdx} = Tree, + AppliedChanges) -> maps:fold( fun (RemovedPath, delete, ToDelete) -> @@ -1317,6 +1395,29 @@ eval_keep_while_conditions( end end, #{}, AppliedChanges). +eval_keep_while_conditions_v1( + #tree{keep_while_conds_revidx = KeepWhileCondsRevIdx} = Tree, + AppliedChanges) -> + maps:fold( + fun + (RemovedPath, delete, ToDelete) -> + khepri_prefix_tree:fold_prefixes_of( + fun(Watchers, ToDelete1) -> + eval_keep_while_conditions_after_removal( + Tree, Watchers, ToDelete1) + end, ToDelete, RemovedPath, KeepWhileCondsRevIdx); + (UpdatedPath, NodeProps, ToDelete) -> + Result = khepri_prefix_tree:find_path( + UpdatedPath, KeepWhileCondsRevIdx), + case Result of + {ok, Watchers} -> + eval_keep_while_conditions_after_update( + Tree, UpdatedPath, NodeProps, Watchers, ToDelete); + error -> + ToDelete + end + end, #{}, AppliedChanges). + eval_keep_while_conditions_after_update( #tree{keep_while_conds = KeepWhileConds} = Tree, UpdatedPath, NodeProps, Watchers, ToDelete) -> @@ -1399,3 +1500,20 @@ remove_expired_nodes( applied_changes = AppliedChanges2}, remove_expired_nodes(Rest, Walk1) end. + +%% ------------------------------------------------------------------- +%% Conversion between tree versions. +%% ------------------------------------------------------------------- + +convert_tree(Tree, MacVer, MacVer) -> + Tree; +convert_tree(Tree, 0, 1) -> + Tree; +convert_tree(Tree, 1, 2) -> + %% In version 2 the reverse index for keep while conditions was converted + %% into a prefix tree. See the `keep_while_conds_revidx_v0()' and + %% `keep_while_conds_revidx_v1()` types. + #tree{keep_while_conds_revidx = KeepWhileCondsRevIdxV0} = Tree, + KeepWhileCondsRevIdxV1 = khepri_prefix_tree:from_map( + KeepWhileCondsRevIdxV0), + Tree#tree{keep_while_conds_revidx = KeepWhileCondsRevIdxV1}.