From 190424d209f72eb200fa02ae3dc059bf51429280 Mon Sep 17 00:00:00 2001 From: cabol Date: Sun, 19 Mar 2017 18:24:01 -0500 Subject: [PATCH] =?UTF-8?q?[#34]=20=E2=80=93=C2=A0Allows=20to=20register?= =?UTF-8?q?=20`shards=5Fsup`=20with=20custom=20name=20(given=20as=20parame?= =?UTF-8?q?ter)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 96 +++++++++++++++++++++------------------- src/shards_local.erl | 16 +++++-- src/shards_owner_sup.erl | 8 ++-- src/shards_state.erl | 40 ++++++++++++++--- src/shards_sup.erl | 45 ++++++++++++------- test/local_SUITE.erl | 14 +++++- test/state_SUITE.erl | 34 ++++++++++---- test/test_helper.erl | 2 + 8 files changed, 173 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index 3ad080c..6ac0724 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ Option | Description | Default `{pick_shard_fun, pick_fun()}` | Function to pick the **shard** on which the `key` will be handled locally – used by `shards_local`. See [shards_state](./src/shards_state.erl). | `shards_local:pick/3` `{pick_node_fun, pick_fun()}` | Function to pick the **node** on which the `key` will be handled globally/distributed – used by `shards_dist`. See [shards_state](./src/shards_state.erl). | `shards_local:pick/3` `{nodes, [node()]}` | Allows to set a list of nodes to auto setup a distributed table – the table is created in all given nodes and then all nodes are joined. This option only has effect if the option `{scope, g}` has been set. | `[]` +`{sup_name, atom()}` | Allows to define a custom name for `shards_sup`. | `shards_sup` > **NOTE:** By default `shards` uses a built-in functions to pick the **shard** (local scope) and the **node** (distributed scope) on which the key will be handled. BUT you can override @@ -98,12 +99,12 @@ You can get the **State** at any time you want: ```erlang > shards_state:get(mytab1). -{state,shards_local,4,#Fun, +{state,shards_local,shards_sup,4,#Fun, #Fun} % this is a wrapper on top of shards_state:get/1 > shards:state(mytab1). -{state,shards_local,4,#Fun, +{state,shards_local,shards_sup,4,#Fun, #Fun} ``` @@ -198,6 +199,7 @@ The `shards` state is defined as: ```erlang -record(state, { module = shards_local :: module(), + sup_name = shards_sup :: atom(), n_shards = ?N_SHARDS :: pos_integer(), pick_shard_fun = fun shards_local:pick/3 :: pick_fun(), pick_node_fun = fun shards_local:pick/3 :: pick_fun() @@ -224,54 +226,58 @@ Most of the `shards_local` functions receives the **State** as parameter, so it to call it. You can check how `shards` module is implemented [HERE](./src/shards.erl). If any microsecond matters to you, you can skip the call to the control ETS table by calling -`shards_local` directly. We have two options: +`shards_local` directly – there are two options to do it. - 1. The first option is getting the `state`, and passing it as argument. Now the question is: - how to get the **State**? Well, it's extremely easy, you can get the `state` when you call - `shards:new/2` by first time, or you can call `shards:state/1`, `shards_state:get/1` or - `shards_state:new/0,1,2,3,4` at any time you want, and then it might be stored within - the calling process, or wherever you want. E.g.: +### Option 1 - ```erlang - % take a look at the 2nd element of the returned tuple, that is the state - > shards:new(mytab, [{n_shards, 4}]). - mytab +The first option is getting the `state`, and passing it as argument. Now the question is: +how to get the **State**? Well, it's extremely easy, you can get the `state` when you call +`shards:new/2` by first time, or you can call `shards:state/1`, `shards_state:get/1` or +`shards_state:new/0,1,2,3,4` at any time you want, and then it might be stored within +the calling process, or wherever you want. E.g.: - % remember you can get the state at any time you want - > State = shards:state(mytab). - {state,shards_local,4,#Fun, - #Fun} - - % now you can call shards_local directly - > shards_local:insert(mytab, {1, 1}, State). - true - > shards_local:lookup(mytab, 1, State). - [{1,1}] - - % in this case, only the n_shards is different from default, so you - % can do this: - > shards_local:lookup(mytab, 1, shards_state:new(4)). - [{1,1}] - ``` +```erlang +% take a look at the 2nd element of the returned tuple, that is the state +> shards:new(mytab, [{n_shards, 4}]). +mytab - 2. The 2nd option is to call `shards_local` directly without the `state`, but this is only - possible if you have created a table with default `shards` options – such as `n_shards`, - `pick_shard_fun` and `pick_node_fun`. If you can take this option it might be significantly - better, since in this case no additional calls are needed, not even to recover the `state` - (like in the previous option), because a new `state` is created with default values. - Therefore, the call is mapped directly to an **ETS** function. E.g.: +% remember you can get the state at any time you want +> State = shards:state(mytab). +{state,shards_local,shards_sup,4,#Fun, + #Fun} - ```erlang - % create a table without set n_shards, pick_shard_fun or pick_node_fun - > shards:new(mytab, []). - mytab +% now you can call shards_local directly +> shards_local:insert(mytab, {1, 1}, State). +true +> shards_local:lookup(mytab, 1, State). +[{1,1}] - % call shards_local without the state - > shards_local:insert(mytab, {1, 1}). - true - > shards_local:lookup(mytab, 1). - [{1,1}] - ``` +% in this case, only the n_shards is different from default, so you +% can do this: +> shards_local:lookup(mytab, 1, shards_state:new(4)). +[{1,1}] +``` + +### Option 2 + +The second option is to call `shards_local` directly without the `state`, but this is only +possible if you have created a table with default `shards` options – such as `n_shards`, +`pick_shard_fun` and `pick_node_fun`. If you can take this option it might be significantly +better, since in this case no additional calls are needed, not even to recover the `state` +(like in the previous option), because a new `state` is created with default values. +Therefore, the call is mapped directly to an **ETS** function. E.g.: + +```erlang +% create a table without set n_shards, pick_shard_fun or pick_node_fun +> shards:new(mytab, []). +mytab + +% call shards_local without the state +> shards_local:insert(mytab, {1, 1}). +true +> shards_local:lookup(mytab, 1). +[{1,1}] +``` Most of the cases this is not necessary, `shards` wrapper is more than enough, it adds only a few microseconds of latency. In conclusion, **Shards** gives you the flexibility to do it, @@ -406,7 +412,7 @@ And again, let's check it out from any node: ## Examples and/or Projects using Shards -* [ExShards](https://github.com/cabol/ex_shards) is an **Elixir** wrapper for `shards` – with extra and nicer functions. +* [ExShards](https://github.com/cabol/ex_shards) – **Elixir** wrapper for `shards` with extra and nicer functions. * [KVX](https://github.com/cabol/kvx) – Simple/basic **Elixir** in-memory Key/Value Store using `shards` (default adapter). diff --git a/src/shards_local.erl b/src/shards_local.erl index 4ce44d6..953e16b 100644 --- a/src/shards_local.erl +++ b/src/shards_local.erl @@ -220,7 +220,8 @@ all() -> %% @end -spec delete(Tab :: atom()) -> true. delete(Tab) -> - ok = shards_sup:terminate_child(shards_sup, get_pid(Tab)), + SupName = shards_state:sup_name(Tab), + ok = shards_sup:terminate_child(SupName, Tab), true. %% @equiv delete(Tab, Key, shards_state:new()) @@ -850,7 +851,16 @@ member(Tab, Key, State) -> Name :: atom(), Options :: [option()]. new(Name, Options) -> - case shards_sup:start_child([Name, Options]) of + case lists:keyfind(sup_name, 1, Options) of + {sup_name, SupName} -> + do_new(SupName, Name, Options); + false -> + do_new(shards_sup, Name, Options) + end. + +%% @private +do_new(SupName, Name, Options) -> + case shards_sup:start_child(SupName, Name, Options) of {ok, Pid} -> true = register(Name, Pid), Name; @@ -881,7 +891,7 @@ next(Tab, Key1, State) -> PickShardFun = shards_state:pick_shard_fun(State), case PickShardFun(Key1, N, r) of any -> - throw(bad_pick_fun_ret); + error(bad_pick_fun_ret); Shard -> ShardName = shard_name(Tab, Shard), next_(Tab, ets:next(ShardName, Key1), Shard) diff --git a/src/shards_owner_sup.erl b/src/shards_owner_sup.erl index 1b49f23..dd5103c 100644 --- a/src/shards_owner_sup.erl +++ b/src/shards_owner_sup.erl @@ -31,14 +31,14 @@ Options :: [term()], Response :: supervisor:startlink_ret(). start_link(Name, Options) -> - supervisor:start_link(?MODULE, [Name, Options]). + supervisor:start_link(?MODULE, {Name, Options}). %%%=================================================================== %%% Supervisor callbacks %%%=================================================================== %% @hidden -init([Name, Options]) -> +init({Name, Options}) -> % ETS table to store state info. Name = ets:new(Name, [ set, @@ -101,7 +101,7 @@ assert_unique_ids([]) -> ok; assert_unique_ids([Id | Rest]) -> case lists:member(Id, Rest) of - true -> throw({badarg, duplicated_id}); + true -> error(badarg); _ -> assert_unique_ids(Rest) end. @@ -126,6 +126,8 @@ parse_opts([{scope, l} | Opts], Acc) -> parse_opts(Opts, Acc#{module := shards_local}); parse_opts([{scope, g} | Opts], Acc) -> parse_opts(Opts, Acc#{module := shards_dist}); +parse_opts([{sup_name, SupName} | Opts], Acc) when is_atom(SupName) -> + parse_opts(Opts, Acc#{sup_name := SupName}); parse_opts([{n_shards, N} | Opts], Acc) when is_integer(N), N > 0 -> parse_opts(Opts, Acc#{n_shards := N}); parse_opts([{pick_shard_fun, Val} | Opts], Acc) when is_function(Val) -> diff --git a/src/shards_state.erl b/src/shards_state.erl index 8fd1fe7..c120b13 100644 --- a/src/shards_state.erl +++ b/src/shards_state.erl @@ -14,6 +14,7 @@ new/2, new/3, new/4, + new/5, to_map/1, from_map/1 ]). @@ -22,6 +23,8 @@ -export([ module/1, module/2, + sup_name/1, + sup_name/2, n_shards/1, n_shards/2, pick_shard_fun/1, @@ -69,6 +72,7 @@ %% State definition -record(state, { module = shards_local :: module(), + sup_name = shards_sup :: atom(), n_shards = ?N_SHARDS :: pos_integer(), pick_shard_fun = fun shards_local:pick/3 :: pick_fun(), pick_node_fun = fun shards_local:pick/3 :: pick_fun() @@ -81,6 +85,7 @@ %% @type state_map() = #{ %% module => module(), +%% sup_name => atom(), %% n_shards => pos_integer(), %% pick_shard_fun => pick_fun(), %% pick_node_fun => pick_fun() @@ -90,12 +95,14 @@ %%
    %%
  • `module': Module to be used depending on the `scope': %% `shards_local' or `shards_dist'.
    • +%%
  • `sup_name': Registered name for `shards_sup'.
    • %%
  • `n_shards': Number of ETS shards/fragments.
    • %%
  • `pick_shard_fun': Function callback to pick/compute the shard.
    • %%
  • `pick_node_fun': Function callback to pick/compute the node.
    • %%
-type state_map() :: #{ module => module(), + sup_name => atom(), n_shards => pos_integer(), pick_shard_fun => pick_fun(), pick_node_fun => pick_fun() @@ -128,15 +135,24 @@ new(Shards) -> new(Shards, Module) -> #state{n_shards = Shards, module = Module}. --spec new(pos_integer(), module(), pick_fun()) -> state(). -new(Shards, Module, PickShardFun) -> - #state{n_shards = Shards, module = Module, pick_shard_fun = PickShardFun}. +-spec new(pos_integer(), module(), atom()) -> state(). +new(Shards, Module, SupName) -> + #state{n_shards = Shards, module = Module, sup_name = SupName}. --spec new(pos_integer(), module(), pick_fun(), pick_fun()) -> state(). -new(Shards, Module, PickShardFun, PickNodeFun) -> +-spec new(pos_integer(), module(), atom(), pick_fun()) -> state(). +new(Shards, Module, SupName, PickShardFun) -> #state{ n_shards = Shards, module = Module, + sup_name = SupName, + pick_shard_fun = PickShardFun}. + +-spec new(pos_integer(), module(), atom(), pick_fun(), pick_fun()) -> state(). +new(Shards, Module, SupName, PickShardFun, PickNodeFun) -> + #state{ + n_shards = Shards, + module = Module, + sup_name = SupName, pick_shard_fun = PickShardFun, pick_node_fun = PickNodeFun}. @@ -146,6 +162,7 @@ new(Shards, Module, PickShardFun, PickNodeFun) -> -spec to_map(state()) -> state_map(). to_map(State) -> #{module => State#state.module, + sup_name => State#state.sup_name, n_shards => State#state.n_shards, pick_shard_fun => State#state.pick_shard_fun, pick_node_fun => State#state.pick_node_fun}. @@ -157,6 +174,7 @@ to_map(State) -> from_map(Map) -> #state{ module = maps:get(module, Map, shards_local), + sup_name = maps:get(sup_name, Map, shards_sup), n_shards = maps:get(n_shards, Map, ?N_SHARDS), pick_shard_fun = maps:get(pick_shard_fun, Map, fun shards_local:pick/3), pick_node_fun = maps:get(pick_node_fun, Map, fun shards_local:pick/3)}. @@ -168,7 +186,7 @@ from_map(Map) -> get(Tab) when is_atom(Tab) -> case ets:lookup(Tab, state) of [State] -> State; - _ -> throw({badarg, Tab}) + _ -> error(badarg) end. %%%=================================================================== @@ -185,6 +203,16 @@ module(Tab) when is_atom(Tab) -> module(Module, #state{} = State) when is_atom(Module) -> State#state{module = Module}. +-spec sup_name(state() | atom()) -> atom(). +sup_name(#state{sup_name = SupName}) -> + SupName; +sup_name(Tab) when is_atom(Tab) -> + sup_name(?MODULE:get(Tab)). + +-spec sup_name(atom(), state()) -> state(). +sup_name(SupName, #state{} = State) when is_atom(SupName) -> + State#state{sup_name = SupName}. + -spec n_shards(state() | atom()) -> pos_integer(). n_shards(#state{n_shards = Shards}) -> Shards; diff --git a/src/shards_sup.erl b/src/shards_sup.erl index e58a4b9..8debb15 100644 --- a/src/shards_sup.erl +++ b/src/shards_sup.erl @@ -10,7 +10,8 @@ %% API -export([ start_link/0, - start_child/1, + start_link/1, + start_child/3, terminate_child/2 ]). @@ -21,30 +22,44 @@ %%% API %%%=================================================================== --spec start_link() -> supervisor:startlink_ret(). +%% @equiv start_link(?MODULE) start_link() -> - supervisor:start_link({local, ?MODULE}, ?MODULE, []). + start_link(?MODULE). --spec start_child([term()]) -> supervisor:startchild_ret(). -start_child(Args) -> - supervisor:start_child(?MODULE, Args). +-spec start_link(Name :: atom()) -> supervisor:startlink_ret(). +start_link(Name) -> + supervisor:start_link({local, Name}, ?MODULE, {Name}). --spec terminate_child(SupRef, Id) -> Response when - SupRef :: supervisor:sup_ref(), - Id :: pid() | supervisor:child_id(), - Error :: not_found | simple_one_for_one, - Response :: ok | {error, Error}. -terminate_child(SupRef, Id) -> - supervisor:terminate_child(SupRef, Id). +%%%=================================================================== +%%% shards_supervisor callbacks +%%%=================================================================== + +-spec start_child(SupName, TabName, Options) -> Return when + SupName :: atom(), + TabName :: atom(), + Options :: [shards_local:option()], + Return :: supervisor:startchild_ret(). +start_child(SupName, TabName, Options) -> + supervisor:start_child(SupName, [TabName, Options]). + +-spec terminate_child(SupName, Tab) -> Return when + SupName :: atom(), + Tab :: pid() | atom(), + Error :: not_found | simple_one_for_one, + Return :: ok | {error, Error}. +terminate_child(SupName, Tab) when is_atom(Tab) -> + terminate_child(SupName, shards_local:get_pid(Tab)); +terminate_child(SupName, Tab) when is_pid(Tab) -> + supervisor:terminate_child(SupName, Tab). %%%=================================================================== %%% Supervisor callbacks %%%=================================================================== %% @hidden -init([]) -> +init({Name}) -> ChildSpec = { - ?MODULE, + Name, {shards_owner_sup, start_link, []}, permanent, infinity, diff --git a/test/local_SUITE.erl b/test/local_SUITE.erl index 8b0311e..569db41 100644 --- a/test/local_SUITE.erl +++ b/test/local_SUITE.erl @@ -32,7 +32,8 @@ %% Test Cases -export([ - t_shard_restarted_when_down/1 + t_shard_restarted_when_down/1, + t_custom_supervisor/1 ]). -include("test_helper.hrl"). @@ -118,6 +119,17 @@ t_shard_restarted_when_down(_Config) -> ok. +t_custom_supervisor(_Config) -> + {ok, _Pid} = shards_sup:start_link(my_sup), + + test = shards:new(test, [{sup_name, my_sup}]), + true = shards:insert(test, [{1, 1}, {2, 2}, {3, 3}]), + assert_values(test, [1, 2, 3], [1, 2, 3]), + + true = shards:delete(test), + + ok. + %%%=================================================================== %%% Internal functions %%%=================================================================== diff --git a/test/state_SUITE.erl b/test/state_SUITE.erl index 4de9982..c3e959c 100644 --- a/test/state_SUITE.erl +++ b/test/state_SUITE.erl @@ -49,6 +49,7 @@ t_create_state(_Config) -> % create state with all default attr values State0 = shards_state:new(), shards_local = shards_state:module(State0), + shards_sup = shards_state:sup_name(State0), true = ?N_SHARDS == shards_state:n_shards(State0), true = fun shards_local:pick/3 == shards_state:pick_shard_fun(State0), true = fun shards_local:pick/3 == shards_state:pick_node_fun(State0), @@ -56,6 +57,7 @@ t_create_state(_Config) -> % create state using shards_state:new/1 State1 = shards_state:new(4), shards_local = shards_state:module(State1), + shards_sup = shards_state:sup_name(State0), true = 4 == shards_state:n_shards(State1), true = fun shards_local:pick/3 == shards_state:pick_shard_fun(State1), true = fun shards_local:pick/3 == shards_state:pick_node_fun(State1), @@ -63,24 +65,35 @@ t_create_state(_Config) -> % create state using shards_state:new/2 State2 = shards_state:new(2, shards_dist), shards_dist = shards_state:module(State2), + shards_sup = shards_state:sup_name(State2), true = 2 == shards_state:n_shards(State2), true = fun shards_local:pick/3 == shards_state:pick_shard_fun(State2), true = fun shards_local:pick/3 == shards_state:pick_node_fun(State2), % create state using shards_state:new/3 - Fun = fun(X, Y, Z) -> (X + Y + Z) rem Y end, - State3 = shards_state:new(4, shards_dist, Fun), + State3 = shards_state:new(2, shards_dist, my_shards_sup), shards_dist = shards_state:module(State3), - true = 4 == shards_state:n_shards(State3), - true = Fun == shards_state:pick_shard_fun(State3), + my_shards_sup = shards_state:sup_name(State3), + true = 2 == shards_state:n_shards(State3), + true = fun shards_local:pick/3 == shards_state:pick_shard_fun(State3), true = fun shards_local:pick/3 == shards_state:pick_node_fun(State3), - % create state using shards_state:new/4 - State4 = shards_state:new(4, shards_dist, Fun, Fun), + % create state using shards_state:new/3 + Fun = fun(X, Y, Z) -> (X + Y + Z) rem Y end, + State4 = shards_state:new(4, shards_dist, my_shards_sup, Fun), shards_dist = shards_state:module(State4), + my_shards_sup = shards_state:sup_name(State4), true = 4 == shards_state:n_shards(State4), true = Fun == shards_state:pick_shard_fun(State4), - true = Fun == shards_state:pick_node_fun(State4), + true = fun shards_local:pick/3 == shards_state:pick_node_fun(State4), + + % create state using shards_state:new/4 + State5 = shards_state:new(4, shards_dist, my_shards_sup, Fun, Fun), + shards_dist = shards_state:module(State5), + my_shards_sup = shards_state:sup_name(State5), + true = 4 == shards_state:n_shards(State5), + true = Fun == shards_state:pick_shard_fun(State5), + true = Fun == shards_state:pick_node_fun(State5), ok. @@ -94,6 +107,7 @@ t_state_ops(_Config) -> Mod = shards_state:module(test_set), true = Mod == shards_local orelse Mod == shards_dist, + shards_sup = shards_state:sup_name(test_set), DefaultShards = shards_state:n_shards(test_set), Fun1 = fun shards_local:pick/3, Fun1 = shards_state:pick_shard_fun(test_set), @@ -106,17 +120,19 @@ t_state_ops(_Config) -> Fun = fun(X, Y, Z) -> (X + Y + Z) rem Y end, State3 = shards_state:pick_shard_fun(Fun, State2), State4 = shards_state:pick_node_fun(Fun, State3), + State5 = shards_state:sup_name(my_sup, State4), #{module := shards_dist, + sup_name := my_sup, n_shards := 100, pick_shard_fun := Fun, pick_node_fun := Fun - } = shards_state:to_map(State4), + } = shards_state:to_map(State5), ok. t_get_state_badarg_error(_Config) -> wrong_tab = ets:new(wrong_tab, [public, named_table]), _ = try shards_state:get(wrong_tab) - catch _:{badarg, wrong_tab} -> ok + catch _:badarg -> ok end, ok. \ No newline at end of file diff --git a/test/test_helper.erl b/test/test_helper.erl index d083807..06c1b2d 100644 --- a/test/test_helper.erl +++ b/test/test_helper.erl @@ -627,6 +627,8 @@ init_shards_new(Scope) -> DefaultShards = ?N_SHARDS, ?SET = shards:new(?SET, [ + named_table, + public, {scope, Scope}, {pick_node_fun, fun ?MODULE:pick_node/3} ]),