Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Rebase per-key actor epoch (kv679) on 2.0 [JIRA: RIAK-1331] #1070

Closed
wants to merge 33 commits into from
Closed

Rebase per-key actor epoch (kv679) on 2.0 [JIRA: RIAK-1331] #1070

Show file tree
Hide file tree
Changes from 25 commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
4d0beb0
WIP move vnode persistent state into new module/process
russelldb Oct 1, 2014
d1347a3
WIP kv679 fixes. Adds a per vnode monotonic counter
russelldb Oct 8, 2014
f203d8f
WIP: don't allow the counter to get above 32bit int size
russelldb Oct 14, 2014
dbac830
WIP fix xref errors
russelldb Oct 14, 2014
794685d
WIP fix dialyzer
russelldb Oct 14, 2014
6390a2b
WIP really fix dialyzer
russelldb Oct 14, 2014
ecad7ad
WIP only increment vnode counter on new per key epoch
russelldb Nov 11, 2014
2c3c1c6
WIP fix new dialyzer errors
russelldb Nov 11, 2014
857180a
WIP update the vnode status command as per fix on 2.0 branch
russelldb Nov 11, 2014
116aac7
Don't force a new epoch on old keys
russelldb Nov 12, 2014
3a3d7dd
Fix some doco/spec errors, fix an error getting the timeout for leases
russelldb Nov 13, 2014
f3f1292
Shhhh, loud test.
russelldb Nov 13, 2014
905ec88
Add some docs and specs and fix the transposed args that dialyzer found
russelldb Nov 14, 2014
5c33539
fix filename spec typo
russelldb Nov 14, 2014
3994c7a
Add a `status` command for testing purposes
russelldb Nov 18, 2014
de433ce
WIP for help on test
russelldb Nov 19, 2014
16c9258
WIP eqc test
russelldb Nov 20, 2014
0a1514f
Address review comments
russelldb Nov 24, 2014
ffbc005
Make sure the vnode resets the counter when a new vnodeid is assigned
russelldb Dec 17, 2014
d5cfab6
Clean up in the set up for vnode counter tests
russelldb Dec 17, 2014
24a6671
IFDEF out the fysnc when testing.
russelldb Dec 22, 2014
b54780c
Fix tests to account for the `ok` match in file saving of vnode status
russelldb Dec 22, 2014
c023d5a
Don't match on delete return for setup/cleanup of status tests
russelldb Dec 23, 2014
a80a086
Don't match on ok on delete for test teardown
russelldb Dec 29, 2014
20d2bd0
Again, don't match on ok for teardown
russelldb Dec 29, 2014
5d5f425
First draft of test to exercise riak_kv_vnode_status_mgr
kellymclaughlin Dec 9, 2014
89a80a3
Add random killing of the status manager process
kellymclaughlin Dec 10, 2014
fa6157a
Move state records into an include file for vnode_status_mgr testing
kellymclaughlin Dec 16, 2014
6ee6fd9
add Kelly's test and a runner for it
russelldb Jan 7, 2015
2de8f4b
Address review comments and remove kelly's test for now
russelldb Jan 14, 2015
90e5c76
Remove vnode_driver added in error
russelldb Jan 14, 2015
d77f65f
use the normalised lease value in all the clauses
russelldb Jan 14, 2015
ded9489
Remove errant test file
russelldb Mar 17, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
310 changes: 139 additions & 171 deletions src/riak_kv_backend.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,22 +73,12 @@ callback_after(Time, Ref, Msg) when is_integer(Time), is_reference(Ref) ->
-ifdef(TEST).

standard_test(BackendMod, Config) ->
{spawn,
[
{setup,
local,
fun() -> setup({BackendMod, Config}) end,
fun cleanup/1,
fun(X) ->
[basic_store_and_fetch(X),
fold_buckets(X),
fold_keys(X),
delete_object(X),
fold_objects(X),
empty_check(X)
]
end
}]}.
{"Basic Backend",
fun() ->
{Mod, State} = setup({BackendMod, Config}),
State2 = basic_store_and_fetch(Mod, State),
cleanup({Mod, State2})
end}.

make_test_bucket(Backend, Suffix) ->
try
Expand All @@ -110,183 +100,161 @@ make_bs_and_ks(Backend) ->
make_test_key(Backend, 1),
make_test_key(Backend, 2)}.

basic_store_and_fetch({Backend, State}) ->
basic_store_and_fetch(Backend, State) ->
{B1, B2, K1, K2} = make_bs_and_ks(Backend),
{"basic store and fetch test",
fun() ->
?assertMatch({ok, _},
Backend:put(B1, K1, [], <<"v1">>, State)),
?assertMatch({ok, _},
Backend:put(B2, K2, [], <<"v2">>, State)),
?assertMatch({ok,<<"v2">>, _},
Backend:get(B2, K2, State)),
?assertMatch({error, not_found, _},
Backend:get(B1, <<"k3">>, State))
end
}.
{ok, State2} = Backend:put(B1, K1, [], <<"v1">>, State),
{ok, State3} = Backend:put(B2, K2, [], <<"v2">>, State2),
{ok,<<"v2">>, State4} = Backend:get(B2, K2, State3),
{error, not_found, State5} = Backend:get(B1, <<"k3">>, State4),
fold_buckets(Backend, State5).

fold_buckets({Backend, State}) ->
fold_buckets(Backend, State) ->
{B1, B2, _K1, _K2} = make_bs_and_ks(Backend),
{"bucket folding test",
fun() ->
FoldBucketsFun =
fun(Bucket, Acc) ->
[Bucket | Acc]
end,
FoldBucketsFun =
fun(Bucket, Acc) ->
[Bucket | Acc]
end,

?assertEqual([B1, B2],
begin
{ok, Buckets1} =
Backend:fold_buckets(FoldBucketsFun,
[],
[],
State),
lists:sort(Buckets1)
end)
end
}.
?assertEqual([B1, B2],
begin
{ok, Buckets1} =
Backend:fold_buckets(FoldBucketsFun,
[],
[],
State),
lists:sort(Buckets1)
end),
fold_keys(Backend, State).

fold_keys({Backend, State}) ->
fold_keys(Backend, State) ->
{B1, B2, K1, K2} = make_bs_and_ks(Backend),
{"key folding test",
fun() ->
FoldKeysFun =
fun(Bucket, Key, Acc) ->
[{Bucket, Key} | Acc]
end,
FoldKeysFun1 =
fun(_Bucket, Key, Acc) ->
[Key | Acc]
end,
FoldKeysFun2 =
fun(Bucket, Key, Acc) ->
case Bucket =:= B1 of
true ->
[Key | Acc];
false ->
Acc
end
end,
FoldKeysFun3 =
fun(Bucket, Key, Acc) ->
case Bucket =:= B1 of
true ->
Acc;
false ->
[Key | Acc]
end
end,

?assertEqual([{B1, K1}, {B2, K2}],
begin
{ok, Keys1} =
Backend:fold_keys(FoldKeysFun,
[],
[],
State),
lists:sort(Keys1)
end),
?assertEqual({ok, [K1]},
Backend:fold_keys(FoldKeysFun1,
[],
[{bucket, B1}],
State)),
?assertEqual({ok, [K2]},
Backend:fold_keys(FoldKeysFun1,
[],
[{bucket, B2}],
State)),
?assertEqual({ok, [K1]},
Backend:fold_keys(FoldKeysFun2, [], [], State)),
?assertEqual({ok, [K1]},
Backend:fold_keys(FoldKeysFun2,
[],
[{bucket, B1}],
State)),
?assertEqual({ok, [K2]},
Backend:fold_keys(FoldKeysFun3, [], [], State)),
?assertEqual({ok, []},
Backend:fold_keys(FoldKeysFun3,
[],
[{bucket, B1}],
State))
end
}.
FoldKeysFun =
fun(Bucket, Key, Acc) ->
[{Bucket, Key} | Acc]
end,
FoldKeysFun1 =
fun(_Bucket, Key, Acc) ->
[Key | Acc]
end,
FoldKeysFun2 =
fun(Bucket, Key, Acc) ->
case Bucket =:= B1 of
true ->
[Key | Acc];
false ->
Acc
end
end,
FoldKeysFun3 =
fun(Bucket, Key, Acc) ->
case Bucket =:= B1 of
true ->
Acc;
false ->
[Key | Acc]
end
end,

delete_object({Backend, State}) ->
?assertEqual([{B1, K1}, {B2, K2}],
begin
{ok, Keys1} =
Backend:fold_keys(FoldKeysFun,
[],
[],
State),
lists:sort(Keys1)
end),
?assertEqual({ok, [K1]},
Backend:fold_keys(FoldKeysFun1,
[],
[{bucket, B1}],
State)),
?assertEqual({ok, [K2]},
Backend:fold_keys(FoldKeysFun1,
[],
[{bucket, B2}],
State)),
?assertEqual({ok, [K1]},
Backend:fold_keys(FoldKeysFun2, [], [], State)),
?assertEqual({ok, [K1]},
Backend:fold_keys(FoldKeysFun2,
[],
[{bucket, B1}],
State)),
?assertEqual({ok, [K2]},
Backend:fold_keys(FoldKeysFun3, [], [], State)),
?assertEqual({ok, []},
Backend:fold_keys(FoldKeysFun3,
[],
[{bucket, B1}],
State)),
delete_object(Backend, State).

delete_object(Backend, State) ->
{_B1, B2, _K1, K2} = make_bs_and_ks(Backend),
{"object deletion test",
fun() ->
?assertMatch({ok, _}, Backend:delete(B2, K2, [], State)),
?assertMatch({error, not_found, _},
Backend:get(B2, K2, State))
end
}.
{ok, State2} = Backend:delete(B2, K2, [], State),
?assertMatch({error, not_found, _},
Backend:get(B2, K2, State2)),
fold_objects(Backend, State2).

fold_objects({Backend, State}) ->
fold_objects(Backend, State) ->
{B1, _B2, K1, _K2} = make_bs_and_ks(Backend),
B3 = make_test_bucket(Backend, 3),
K3 = make_test_key(Backend, 3),
ObjFilter = fun(Os) -> [if is_binary(X) -> {BK, X};
true -> {BK, riak_object:get_value(X)}
end || {BK, X} <- Os]
end,
{"object folding test",
fun() ->
FoldKeysFun =
fun(Bucket, Key, Acc) ->
[{Bucket, Key} | Acc]
end,
FoldObjectsFun =
fun(Bucket, Key, Value, Acc) ->
[{{Bucket, Key}, Value} | Acc]
end,
?assertEqual([{B1, K1}],
begin
{ok, Keys} =
Backend:fold_keys(FoldKeysFun,
[],
[],
State),
lists:sort(Keys)
end),
FoldKeysFun =
fun(Bucket, Key, Acc) ->
[{Bucket, Key} | Acc]
end,
FoldObjectsFun =
fun(Bucket, Key, Value, Acc) ->
[{{Bucket, Key}, Value} | Acc]
end,
?assertEqual([{B1, K1}],
begin
{ok, Keys} =
Backend:fold_keys(FoldKeysFun,
[],
[],
State),
lists:sort(Keys)
end),

?assertEqual([{{B1,K1}, <<"v1">>}],
begin
{ok, Objects1} =
Backend:fold_objects(FoldObjectsFun,
[],
[],
State),
lists:sort(ObjFilter(Objects1))
end),
?assertMatch({ok, _},
Backend:put(B3, K3, [], <<"v3">>, State)),
?assertEqual([{{B1,K1},<<"v1">>},
{{B3,K3},<<"v3">>}],
begin
{ok, Objects} =
Backend:fold_objects(FoldObjectsFun,
[],
[],
State),
lists:sort(ObjFilter(Objects))
end)
end
}.
?assertEqual([{{B1,K1}, <<"v1">>}],
begin
{ok, Objects1} =
Backend:fold_objects(FoldObjectsFun,
[],
[],
State),
lists:sort(ObjFilter(Objects1))
end),
{ok, State2} = Backend:put(B3, K3, [], <<"v3">>, State),
?assertEqual([{{B1,K1},<<"v1">>},
{{B3,K3},<<"v3">>}],
begin
{ok, Objects} =
Backend:fold_objects(FoldObjectsFun,
[],
[],
State2),
lists:sort(ObjFilter(Objects))
end),
empty_check(Backend, State).

empty_check({Backend, State}) ->
empty_check(Backend, State) ->
{B1, _B2, K1, _K2} = make_bs_and_ks(Backend),
B3 = make_test_bucket(Backend, 3),
K3 = make_test_key(Backend, 3),
{"is_empty test",
fun() ->
?assertEqual(false, Backend:is_empty(State)),
?assertMatch({ok, _}, Backend:delete(B1,K1, [], State)),
?assertMatch({ok, _}, Backend:delete(B3,K3, [], State)),
?assertEqual(true, Backend:is_empty(State))
end
}.
?assertEqual(false, Backend:is_empty(State)),
{ok, State2} = Backend:delete(B1,K1, [], State),
{ok, State3} = Backend:delete(B3,K3, [], State2),
?assertEqual(true, Backend:is_empty(State3)),
State3.

setup({BackendMod, Config}) ->
%% Start the backend
Expand Down
Loading