diff --git a/Makefile b/Makefile index 4b53bd5..84e37ae 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,6 @@ REBAR = $(shell which rebar3) ## Common variables -CONFIG ?= test/test.config DEFAULT_PATH = ./_build/default DEFAULT_BUILD_PATH = $(DEFAULT_PATH)/lib/*/ebin @@ -9,7 +8,7 @@ DEFAULT_BUILD_PATH = $(DEFAULT_PATH)/lib/*/ebin CT_PATH = ./_build/test CT_BUILD_PATH = $(CT_PATH)/lib/*/ebin CT_SUITES = task_SUITE local_SUITE dist_SUITE -CT_OPTS = -cover test/cover.spec -erl_args -config ${CONFIG} +CT_OPTS = -cover test/cover.spec .PHONY: all check_rebar compile clean distclean dialyze tests shell doc @@ -46,8 +45,8 @@ tests: check_rebar ct_run -dir test -suite $(CT_SUITES) -pa $(CT_BUILD_PATH) -logdir $(CT_PATH)/logs $(CT_OPTS) rm -rf test/*.beam -shell: compile - erl -pa $(DEFAULT_BUILD_PATH) -s shards -config ${CONFIG} +shell: + $(REBAR) shell edoc: $(REBAR) edoc diff --git a/README.md b/README.md index 1d0bf30..dc42bd4 100644 --- a/README.md +++ b/README.md @@ -58,6 +58,9 @@ the options. With `shards` there are additional options: * `{restart_strategy, one_for_one | one_for_all}`: allows to configure the restart strategy for `shards_owner_sup`. Default is `one_for_one`. + * `{auto_eject_nodes, boolean()}`: A boolean value that controls if node should be ejected + when it fails. – Default is `true`. + * `{pick_shard_fun, pick_shard_fun()}`: Function to pick the **shard** on which the `key` will be handled locally – used by `shards_local`. See the spec [HERE](https://github.com/cabol/shards/blob/master/src/shards_local.erl#L145). @@ -78,10 +81,10 @@ Besides, the `shards:new/2` function returns a tuple of two elements: ``` The first element is the name of the created table; `mytab1`. And the second one is the -[State](./src/shards_local.erl#L159): `{4, #Fun, set}`. +[State](./src/shards_local.erl#L189-L205): `{4, #Fun, set}`. We'll talk about the **State** later, and see how it can be used. -> **NOTE:** For more information about `shards:new/2` go [HERE](./src/shards_local.erl#L796). +> **NOTE:** For more information about `shards:new/2` see [shards](./src/shards.erl). Let's continue: @@ -166,13 +169,13 @@ Extremely simple isn't? The module `shards` is a wrapper on top of two main modules: - * `shards_local`: Implements Sharding on top of ETS tables, but locally (on a single Erlang node). - * `shards_dist`: Implements Sharding but across multiple distributed Erlang nodes, which must + * [shards_local](./src/shards_local.erl): Implements Sharding on top of ETS tables, but locally (on a single Erlang node). + * [shards_dist](./src/shards_dist.erl): Implements Sharding but across multiple distributed Erlang nodes, which must run `shards` locally, since `shards_dist` uses `shards_local` internally. We'll cover the distributed part later. When you use `shards` on top of `shards_local`, a call to the control ETS table owned by `shards_owner_sup` -must be done, in order to recover the [State](./src/shards_local.erl#L159), mentioned previously. +must be done, in order to recover the [State](./src/shards_local.erl#L189-L205), mentioned previously. Most of the `shards_local` functions receives the **State** as parameter, so it must be fetched before to call it. You can check how `shards` module is implemented [HERE](./src/shards.erl). @@ -234,7 +237,8 @@ $ erl -sname c@localhost -pa _build/default/lib/*/ebin -s shards % when a tables is created with {scope, g}, the module shards_dist is used % internally by shards > shards:new(mytab, [{n_shards, 4}, {scope, g}]). -{mytab,{4,#Fun,set}} +{mytab,{{4,#Fun,set}, + {#Fun,true}}} ``` **3.** Setup the `shards` cluster. diff --git a/rebar.config b/rebar.config index b7d3987..937c1b4 100644 --- a/rebar.config +++ b/rebar.config @@ -42,3 +42,7 @@ {"(linux|darwin|solaris)", clean, "make -C c_src clean"}, {"(freebsd)", clean, "gmake -C c_src clean"} ]}. + +%% == Shell == + +{shell, [{apps, [shards]}]}. diff --git a/src/shards.app.src b/src/shards.app.src index e9e8429..d5c0541 100644 --- a/src/shards.app.src +++ b/src/shards.app.src @@ -1,6 +1,6 @@ {application, shards, [ {description, "ETS with Sharding support."}, - {vsn, "0.1.0"}, + {vsn, "0.2.0"}, {registered, []}, {mod, {shards, []}}, {applications, [kernel, stdlib]}, @@ -10,6 +10,7 @@ {licenses, ["MIT"]}, {links, [ {"GitHub", "https://github.com/cabol/shards"}, - {"Docs", "http://cabol.github.io/posts/2016/04/14/sharding-support-for-ets.html"} + {"Doc", "http://cabol.github.io/shards"}, + {"Blog Post", "http://cabol.github.io/posts/2016/04/14/sharding-support-for-ets.html"} ]} ]}. diff --git a/src/shards_dist.erl b/src/shards_dist.erl index 4af0299..a1a109d 100644 --- a/src/shards_dist.erl +++ b/src/shards_dist.erl @@ -52,18 +52,19 @@ -type pick_node_fun() :: shards_local:pick_node_fun(). %% @type state() = { -%% PickNode :: pick_node_fun(), -%% AutoEject :: boolean() +%% PickNode :: pick_node_fun(), +%% AutoEjectNodes :: boolean() %% }. %% %% Defines the `shards' distributed state: %%
    %%
  • `PickNode': Function callback to pick/compute the node.
  • -%%
  • `TableType': Table type.
  • +%%
  • `AutoEjectNodes': A boolean value that controls if node should be +%% ejected when it fails.
  • %%
-type state() :: { - PickNode :: pick_node_fun(), - AutoEject :: boolean() + PickNode :: pick_node_fun(), + AutoEjectNodes :: boolean() }. -export_type([ @@ -125,7 +126,7 @@ pick_node(_, Key, Nodes) -> -spec delete(Tab :: atom()) -> true. delete(Tab) -> - mapred(Tab, {?SHARDS, delete, [Tab]}, nil, nil, delete), + mapred(Tab, {?SHARDS, delete, [Tab]}, nil, state(Tab), delete), true. -spec delete(Tab, Key, State) -> true when @@ -163,10 +164,10 @@ insert(Tab, ObjOrObjL, State) when is_list(ObjOrObjL) -> lists:foreach(fun(Object) -> true = insert(Tab, Object, State) end, ObjOrObjL), true; -insert(Tab, ObjOrObjL, {Local, {PickNode, _}}) when is_tuple(ObjOrObjL) -> +insert(Tab, ObjOrObjL, {Local, {PickNode, AutoEject}}) when is_tuple(ObjOrObjL) -> [Key | _] = tuple_to_list(ObjOrObjL), Node = PickNode(write, Key, get_nodes(Tab)), - rpc_call(Node, ?SHARDS, insert, [Tab, ObjOrObjL, Local]). + rpc_call(Node, {?SHARDS, insert, [Tab, ObjOrObjL, Local]}, Tab, AutoEject). -spec insert_new(Tab, ObjOrObjL, State) -> Result when Tab :: atom(), @@ -177,7 +178,7 @@ insert_new(Tab, ObjOrObjL, State) when is_list(ObjOrObjL) -> lists:foldr(fun(Object, Acc) -> [insert_new(Tab, Object, State) | Acc] end, [], ObjOrObjL); -insert_new(Tab, ObjOrObjL, {Local, {PickNode, _} = Dist}) when is_tuple(ObjOrObjL) -> +insert_new(Tab, ObjOrObjL, {Local, {PickNode, AutoEject} = Dist}) when is_tuple(ObjOrObjL) -> [Key | _] = tuple_to_list(ObjOrObjL), Nodes = get_nodes(Tab), case PickNode(read, Key, Nodes) of @@ -187,13 +188,13 @@ insert_new(Tab, ObjOrObjL, {Local, {PickNode, _} = Dist}) when is_tuple(ObjOrObj case mapred(Tab, Map, Reduce, Dist, read) of [] -> Node = PickNode(write, Key, Nodes), - rpc_call(Node, ?SHARDS, insert_new, [Tab, ObjOrObjL, Local]); + rpc_call(Node, {?SHARDS, insert_new, [Tab, ObjOrObjL, Local]}, Tab, AutoEject); _ -> false end; _ -> Node = PickNode(write, Key, Nodes), - rpc_call(Node, ?SHARDS, insert_new, [Tab, ObjOrObjL, Local]) + rpc_call(Node, {?SHARDS, insert_new, [Tab, ObjOrObjL, Local]}, Tab, AutoEject) end. -spec lookup(Tab, Key, State) -> Result when @@ -232,7 +233,7 @@ lookup_element(Tab, Key, Pos, {Local, {PickNode, _} = Dist}) -> -spec match(Tab, Pattern, State) -> [Match] when Tab :: atom(), Pattern :: ets:match_pattern(), - State :: state(), + State :: shards:state(), Match :: [term()]. match(Tab, Pattern, {Local, Dist}) -> Map = {?SHARDS, match, [Tab, Pattern, Local]}, @@ -242,7 +243,7 @@ match(Tab, Pattern, {Local, Dist}) -> -spec match_delete(Tab, Pattern, State) -> true when Tab :: atom(), Pattern :: ets:match_pattern(), - State :: state(). + State :: shards:state(). match_delete(Tab, Pattern, {Local, Dist}) -> Map = {?SHARDS, match_delete, [Tab, Pattern, Local]}, Reduce = {fun(Res, Acc) -> Acc and Res end, true}, @@ -251,7 +252,7 @@ match_delete(Tab, Pattern, {Local, Dist}) -> -spec match_object(Tab, Pattern, State) -> [Object] when Tab :: atom(), Pattern :: ets:match_pattern(), - State :: state(), + State :: shards:state(), Object :: tuple(). match_object(Tab, Pattern, {Local, Dist}) -> Map = {?SHARDS, match_object, [Tab, Pattern, Local]}, @@ -282,7 +283,7 @@ new(Name, Options) -> -spec select(Tab, MatchSpec, State) -> [Match] when Tab :: atom(), MatchSpec :: ets:match_spec(), - State :: state(), + State :: shards:state(), Match :: term(). select(Tab, MatchSpec, {Local, Dist}) -> Map = {?SHARDS, select, [Tab, MatchSpec, Local]}, @@ -292,7 +293,7 @@ select(Tab, MatchSpec, {Local, Dist}) -> -spec select_count(Tab, MatchSpec, State) -> NumMatched when Tab :: atom(), MatchSpec :: ets:match_spec(), - State :: state(), + State :: shards:state(), NumMatched :: non_neg_integer(). select_count(Tab, MatchSpec, {Local, Dist}) -> Map = {?SHARDS, select_count, [Tab, MatchSpec, Local]}, @@ -302,7 +303,7 @@ select_count(Tab, MatchSpec, {Local, Dist}) -> -spec select_delete(Tab, MatchSpec, State) -> NumDeleted when Tab :: atom(), MatchSpec :: ets:match_spec(), - State :: state(), + State :: shards:state(), NumDeleted :: non_neg_integer(). select_delete(Tab, MatchSpec, {Local, Dist}) -> Map = {?SHARDS, select_delete, [Tab, MatchSpec, Local]}, @@ -312,7 +313,7 @@ select_delete(Tab, MatchSpec, {Local, Dist}) -> -spec select_reverse(Tab, MatchSpec, State) -> [Match] when Tab :: atom(), MatchSpec :: ets:match_spec(), - State :: state(), + State :: shards:state(), Match :: term(). select_reverse(Tab, MatchSpec, {Local, Dist}) -> Map = {?SHARDS, select_reverse, [Tab, MatchSpec, Local]}, @@ -351,12 +352,23 @@ state(TabName) -> %%%=================================================================== %% @private -rpc_call(Node, Module, Function, Args) -> +rpc_call(Node, {Module, Function, Args}, Tab, AutoEject) -> case rpc:call(Node, Module, Function, Args) of - {badrpc, _} -> throw(unexpected_error); % @TODO: call GC to remove this node - Response -> Response + {badrpc, _} -> + % unexpected to get here + maybe_eject_node(Node, Tab, AutoEject), + throw({unexpected_error, {badrpc, Node}}); + Response -> + Response end. +%% @private +maybe_eject_node(Node, Tab, true) -> + leave(Tab, [Node]), + ok; +maybe_eject_node(_, _, _) -> + ok. + %% @private mapred(Tab, Map, Reduce, State, Op) -> mapred(Tab, nil, Map, Reduce, State, Op). @@ -366,12 +378,12 @@ mapred(Tab, Key, Map, nil, State, Op) -> mapred(Tab, Key, Map, fun(E, Acc) -> [E | Acc] end, State, Op); mapred(Tab, nil, Map, Reduce, _, _) -> p_mapred(Tab, Map, Reduce); -mapred(Tab, Key, {MapMod, MapFun, MapArgs} = Map, Reduce, {PickNode, _}, Op) -> +mapred(Tab, Key, Map, Reduce, {PickNode, AutoEject}, Op) -> case PickNode(Op, Key, get_nodes(Tab)) of any -> p_mapred(Tab, Map, Reduce); Node -> - rpc_call(Node, MapMod, MapFun, MapArgs) + rpc_call(Node, Map, Tab, AutoEject) end. %% @private diff --git a/src/shards_local.erl b/src/shards_local.erl index dfb6a43..dbf8fd3 100644 --- a/src/shards_local.erl +++ b/src/shards_local.erl @@ -149,6 +149,43 @@ %% Defines spec function to pick or compute the node. -type pick_node_fun() :: fun((operation_t(), key(), [node()]) -> node()) | any. +%% @type tweaks() = {write_concurrency, boolean()} +%% | {read_concurrency, boolean()} +%% | compressed. +%% +%% ETS tweaks option +-type tweaks() :: {write_concurrency, boolean()} + | {read_concurrency, boolean()} + | compressed. + +%% @type shards_opt() = {scope, l | g} +%% | {n_shards, pos_integer()} +%% | {pick_shard_fun, pick_shard_fun()} +%% | {pick_node_fun, pick_node_fun()} +%% | {restart_strategy, one_for_one | one_for_all} +%% | {eject_nodes_on_failure, boolean()}. +%% +%% Shards extended options. +-type shards_opt() :: {scope, l | g} + | {n_shards, pos_integer()} + | {pick_shard_fun, pick_shard_fun()} + | {pick_node_fun, pick_node_fun()} + | {restart_strategy, one_for_one | one_for_all} + | {auto_eject_nodes, boolean()}. + +%% @type option() = ets:type() | ets:access() | named_table +%% | {keypos, pos_integer()} +%% | {heir, pid(), HeirData :: term()} +%% | {heir, none} | tweaks() +%% | shards_opt(). +%% +%% Create table options – used by `new/2'. +-type option() :: ets:type() | ets:access() | named_table + | {keypos, pos_integer()} + | {heir, pid(), HeirData :: term()} + | {heir, none} | tweaks() + | shards_opt(). + %% @type state() = { %% NumShards :: pos_integer(), %% PickShard :: pick_shard_fun(), @@ -191,41 +228,6 @@ Continuation :: ets:continuation() }. -%% @type tweaks() = {write_concurrency, boolean()} -%% | {read_concurrency, boolean()} -%% | compressed. -%% -%% ETS tweaks option --type tweaks() :: {write_concurrency, boolean()} - | {read_concurrency, boolean()} - | compressed. - -%% @type shards_opt() = {scope, l | g} -%% | {n_shards, pos_integer()} -%% | {pick_shard_fun, pick_shard_fun()} -%% | {pick_node_fun, pick_node_fun()} -%% | {restart_strategy, one_for_one | one_for_all}. -%% -%% Shards extended options. --type shards_opt() :: {scope, l | g} - | {n_shards, pos_integer()} - | {pick_shard_fun, pick_shard_fun()} - | {pick_node_fun, pick_node_fun()} - | {restart_strategy, one_for_one | one_for_all}. - -%% @type option() = ets:type() | ets:access() | named_table -%% | {keypos, pos_integer()} -%% | {heir, pid(), HeirData :: term()} -%% | {heir, none} | tweaks() -%% | shards_opt(). -%% -%% Create table options – used by `new/2'. --type option() :: ets:type() | ets:access() | named_table - | {keypos, pos_integer()} - | {heir, pid(), HeirData :: term()} - | {heir, none} | tweaks() - | shards_opt(). - %% Exported types -export_type([ operation_t/0, @@ -309,7 +311,7 @@ file2tab(Filenames) -> Options :: [Option], Option :: {verify, boolean()}, Reason :: term(), - Response :: [{Tab, state()} | {error, Reason}]. + Response :: {ok, Tab} | {error, Reason}. file2tab(Filenames, Options) -> try ShardTabs = [{First, _} | _] = [begin @@ -322,7 +324,11 @@ file2tab(Filenames, Options) -> end end || FN <- Filenames], Tab = name_from_shard(First), - new(Tab, [{restore, ShardTabs, Options}, {n_shards, length(Filenames)}]) + {Tab, _} = new(Tab, [ + {restore, ShardTabs, Options}, + {n_shards, length(Filenames)} + ]), + {ok, Tab} catch _:Error -> Error end. diff --git a/src/shards_owner_sup.erl b/src/shards_owner_sup.erl index f6e72a3..bd89e1d 100644 --- a/src/shards_owner_sup.erl +++ b/src/shards_owner_sup.erl @@ -118,7 +118,7 @@ parse_opts(Opts) -> type => set, pick_shard_fun => fun shards_local:pick_shard/3, pick_node_fun => fun shards_dist:pick_node/3, - autoeject_nodes => true, + auto_eject_nodes => true, restart_strategy => one_for_one, opts => [] }, @@ -137,8 +137,8 @@ parse_opts([{pick_shard_fun, PickShard} | Opts], Acc) when is_function(PickShard parse_opts(Opts, Acc#{pick_shard_fun := PickShard}); parse_opts([{pick_node_fun, PickNode} | Opts], Acc) when is_function(PickNode) -> parse_opts(Opts, Acc#{pick_node_fun := PickNode}); -parse_opts([{autoeject_nodes, AutoEject} | Opts], Acc) when is_boolean(AutoEject) -> - parse_opts(Opts, Acc#{autoeject_nodes := AutoEject}); +parse_opts([{auto_eject_nodes, Flag} | Opts], Acc) when is_boolean(Flag) -> + parse_opts(Opts, Acc#{auto_eject_nodes := Flag}); parse_opts([{restart_strategy, Strategy} | Opts], Acc) when ?is_restart_strategy(Strategy) -> parse_opts(Opts, Acc#{restart_strategy := Strategy}); parse_opts([Opt | Opts], #{opts := NOpts} = Acc) when ?is_ets_type(Opt) -> @@ -155,9 +155,9 @@ local_state(Opts) -> %% @private dist_state(Opts) -> - #{pick_node_fun := PickNode, - autoeject_nodes := AutoEject} = Opts, - {PickNode, AutoEject}. + #{pick_node_fun := PickNode, + auto_eject_nodes := AutoEjectNodes} = Opts, + {PickNode, AutoEjectNodes}. %% @private init_shards_dist(Tab, shards_dist) -> diff --git a/test/dist_SUITE.erl b/test/dist_SUITE.erl index 093991e..781b5b3 100644 --- a/test/dist_SUITE.erl +++ b/test/dist_SUITE.erl @@ -25,6 +25,7 @@ %% Tests Cases -export([ t_join_leave_ops/1, + t_eject_node_on_failure/1, t_delete_tabs/1 ]). @@ -45,6 +46,7 @@ groups() -> t_basic_ops, t_match_ops, t_select_ops, + t_eject_node_on_failure, t_delete_tabs ]}]. @@ -77,13 +79,11 @@ t_join_leave_ops(Config) -> setup_tabs(Config), timer:sleep(500), - % join - AllNodes = shards:join(?DUPLICATE_BAG, AllNodes), - AllNodes = shards:join(?SET, AllNodes), - - % check nodes - 7 = length(shards:get_nodes(?SET)), - 7 = length(shards:get_nodes(?DUPLICATE_BAG)), + % join and check nodes + lists:foreach(fun(Tab) -> + AllNodes = shards:join(Tab, AllNodes), + 7 = length(shards:get_nodes(Tab)) + end, ?SHARDS_TABS), % check no duplicate members Members = pg2:get_members(?SET), @@ -126,6 +126,39 @@ t_join_leave_ops(Config) -> ct:print("\e[1;1m t_join_leave_ops: \e[0m\e[32m[OK] \e[0m"), ok. +t_eject_node_on_failure(Config) -> + ok = cleanup_tabs(Config), + + UpNodes = shards:get_nodes(?SET), + 6 = length(UpNodes), + + % add new node + NewNodes = [Z] = start_slaves([z]), + ok = rpc:call(Z, test_helper, init_shards, [g]), + UpNodes1 = shards:join(?SET, NewNodes), + UpNodes1 = shards:get_nodes(?SET), + + % insert some data on that node + Z = lists:last(UpNodes1), + Z = shards_dist:pick_node(read, 2, UpNodes1), + true = shards:insert(?SET, {2, 2}), + + % cause an error + ok = rpc:call(Z, shards, stop, []), + timer:sleep(500), + 7 = length(UpNodes1), + + % new node should be ejected on failure + UpNodes2 = lists:usort(UpNodes1 -- NewNodes), + UpNodes2 = shards:get_nodes(?SET), + 6 = length(UpNodes2), + + % stop failure node + stop_slaves([z]), + + ct:print("\e[1;1m t_eject_node_on_failure: \e[0m\e[32m[OK] \e[0m"), + ok. + t_delete_tabs(Config) -> ok = cleanup_tabs(Config), diff --git a/test/test.config b/test/test.config deleted file mode 100644 index ff31c60..0000000 --- a/test/test.config +++ /dev/null @@ -1,3 +0,0 @@ -[{shards, [ - {module, shards_local} -]}]. diff --git a/test/test_helper.erl b/test/test_helper.erl index 1b43698..613d770 100644 --- a/test/test_helper.erl +++ b/test/test_helper.erl @@ -332,9 +332,10 @@ t_info_ops(_Config) -> R0 = ets:i(), % test info/1,2 - 2 = length(shards:info(?SET)), + DefaultShards = ?N_SHARDS, + DefaultShards = length(shards:info(?SET)), 5 = length(shards:info(?DUPLICATE_BAG)), - L1 = lists:duplicate(2, public), + L1 = lists:duplicate(DefaultShards, public), L1 = shards:info(?SET, protection), L2 = lists:duplicate(5, public), L2 = shards:info(?DUPLICATE_BAG, protection), @@ -380,17 +381,23 @@ t_tab2list_tab2file_file2tab(_Config) -> 4 = length(R1), % save tab to files - [ok, ok] = shards:tab2file(?SET, ["myfile0", "myfile1"]), + DefaultShards = ?N_SHARDS, + L = lists:duplicate(DefaultShards, ok), + FileL = [ + "myfile0", "myfile1", "myfile2", "myfile3", + "myfile4", "myfile5", "myfile6", "myfile7" + ], + L = shards:tab2file(?SET, FileL), % delete table shards:delete(?SET), % restore table from files {error, _} = shards:file2tab(["myfile0", "wrong_file"]), - {?SET, _} = shards:file2tab(["myfile0", "myfile1"]), + {ok, ?SET} = shards:file2tab(FileL), % check - [_, _] = shards:info(?SET), + [_, _, _, _, _, _, _, _] = shards:info(?SET), KVPairs = lookup_keys(shards, ?SET, [k1, k2, k3, k4]), ct:print("\e[1;1m t_tab2list_tab2file_file2tab: \e[0m\e[32m[OK] \e[0m"), @@ -476,7 +483,10 @@ init_shards(Scope) -> %% @private init_shards_new(g) -> DefaultShards = ?N_SHARDS, - {_, {{2, _, set}, _}} = shards:new(?SET, [{n_shards, 2}, {scope, g}, set]), + {_, {{DefaultShards, _, set}, _}} = shards:new(?SET, [ + {scope, g}, + {auto_eject_nodes, false} + ]), {_, {{5, _, duplicate_bag}, _}} = shards:new(?DUPLICATE_BAG, [{n_shards, 5}, {scope, g}, duplicate_bag]), {_, {{DefaultShards, _, ordered_set}, _}} = @@ -490,7 +500,7 @@ init_shards_new(g) -> ]); init_shards_new(Scope) -> DefaultShards = ?N_SHARDS, - {_, {2, _, set}} = shards:new(?SET, [{n_shards, 2}, {scope, Scope}, set]), + {_, {DefaultShards, _, set}} = shards:new(?SET, [{scope, Scope}]), {_, {5, _, duplicate_bag}} = shards:new(?DUPLICATE_BAG, [{n_shards, 5}, {scope, Scope}, duplicate_bag]), {_, {DefaultShards, _, ordered_set}} =