Develop 3.0 -> 3.1 merge

.gitignore

@@ -17,3 +17,7 @@ _build
 rebar.lock
 .DS_Store
 nonode@nohost
+data/*
+test/*.beam
+log/*
+core_vnode_eqc.log

.travis.yml

@@ -1,6 +1,8 @@
 language: erlang
 otp_release:
   - 20.3.8
+  - 21.3
+  - 22.3
 script:
   - chmod u+x rebar3
   - ./rebar3 do upgrade, compile, dialyzer, xref, eunit

Makefile

@@ -1,4 +1,4 @@
-.PHONY: compile rel cover test dialyzer
+.PHONY: compile rel cover test dialyzer eqc
 REBAR=./rebar3
 
 compile:
@@ -7,16 +7,21 @@ compile:
 clean:
 	$(REBAR) clean
 
-cover: test
+cover: 
+	$(REBAR) eunit --cover
 	$(REBAR) cover
 
 test: compile
-	$(REBAR) as test do eunit
+	$(REBAR) eunit
 
 dialyzer:
 	$(REBAR) dialyzer
 
 xref:
 	$(REBAR) xref
 
+eqc:
+	$(REBAR) as test eqc --testing_budget 120
+	$(REBAR) as eqc eunit
+
 check: test dialyzer xref

eqc/bg_manager_eqc.erl

@@ -19,7 +19,6 @@
 
 -module(bg_manager_eqc).
 
--ifdef(TEST).
 -ifdef(EQC).
 
 -include("include/riak_core_bg_manager.hrl").
@@ -858,4 +857,3 @@ prop_bgmgr_parallel() ->
                          end))).
 
 -endif.
--endif.

eqc/worker_pool_pulse.erl

@@ -59,7 +59,7 @@ handle_work(Work, _From, State) ->
 
 %% @doc Any amount of work should complete through any size pool.
 prop_any_pool() ->
-    ?SETUP(fun setup_and_teardown/1,
+    ?SETUP(fun setup_and_teardown/0,
            ?FORALL({Seed, ExtraWork, WorkList},
                    {pulse:seed(),
                     frequency([{10,true},{1,false}]),

rebar.config

@@ -1,36 +1,40 @@
 {erl_first_files, ["src/gen_nb_server.erl", "src/riak_core_gen_server.erl",
 		   "src/riak_core_stat_xform"]}.
+
 {cover_enabled, true}.
+
 {erl_opts, [warnings_as_errors,
             {parse_transform, lager_transform},
             debug_info,
             {platform_define, "^[0-9]+", namespaced_types},
             {platform_define, "^R15", "old_hash"},
-            {platform_define, "^[2-9][1-9][0-9]*(.?[0-9]*)", deprecated_21}]}.
+            {platform_define, "^[2-9][1-9][0-9]*(.?[0-9]*)", deprecated_21},
+            {platform_define, "^[2-9][2-9][0-9]*(.?[0-9]*)", deprecated_22}]}.
+
 {edoc_opts, [{preprocess, true}]}.
+
 {eunit_opts, [verbose]}.
-{xref_checks,[undefined_function_calls,undefined_functions,locals_not_used,
-    deprecated_function_calls, deprecated_functions]}.
 
-{plugins, [{rebar3_eqc, {git, "https://github.com/Vagabond/rebar3-eqc-plugin", {branch, "master"}}}]}.
+{xref_checks,[undefined_function_calls,undefined_functions,locals_not_used]}.
+
+{plugins, [{eqc_rebar, {git, "https://github.com/Quviq/eqc-rebar", {branch, "master"}}}]}.
 
 {deps, [
   {lager, {git, "git://github.com/erlang-lager/lager.git", {tag, "3.8.0"}}},
   {poolboy, {git, "https://github.com/basho/poolboy.git", {tag, "riak_kv-3.0.0"}}},
-  {basho_stats, {git, "git://github.com/basho/basho_stats.git", {tag, "1.1.0"}}},
   {riak_sysmon, {git, "git://github.com/basho/riak_sysmon.git", {tag, "2.2.0"}}},
   {clique, {git, "git://github.com/basho/clique.git", {tag, "0.3.11"}}},
-  gen_fsm_compat,
   {eleveldb, {git, "git://github.com/basho/eleveldb.git", {tag, "riak_kv-3.0.0"}}},
   {riak_ensemble, {git, "https://github.com/basho/riak_ensemble", {tag, "riak_kv-3.0.0"}}},
   {pbkdf2, {git, "git://github.com/basho/erlang-pbkdf2.git", {tag, "2.1.0"}}},
   {cluster_info, {git, "git://github.com/basho/cluster_info.git", {tag, "2.1.0"}}},
-  {exometer_core, {git, "git://github.com/Feuerlabs/exometer_core.git", {tag, "v1.5.7"}}}
+  {exometer_core, {git, "git://github.com/Feuerlabs/exometer_core.git", {tag, "v1.5.7"}}},
+  {basho_stats, {git, "git://github.com/basho/basho_stats.git", {tag, "1.1.0"}}}
 ]}.
 
 {dialyzer, [{plt_apps, all_deps}]}.
 
 {profiles, [
     {test, [{deps, [meck]}, {erl_opts, [nowarn_export_all]}]},
-    {eqc, [{deps, [meck]}, {erl_opts, [nowarn_export_all]}]}
+    {eqc, [{deps, [meck]}, {erl_opts, [{d, 'EQC'}, nowarn_export_all]}]}
 ]}.

src/hashtree.erl

@@ -154,16 +154,14 @@
          run_multiple/2,
          run_remote/0,
          run_remote/1]).
--endif. % TEST
 
 -ifdef(EQC).
 -export([prop_correct/0, prop_sha/0, prop_est/0]).
 -include_lib("eqc/include/eqc.hrl").
 -endif.
 
--ifdef(TEST).
 -include_lib("eunit/include/eunit.hrl").
--endif.
+-endif. %% TEST
 
 -define(NUM_SEGMENTS, (1024*1024)).
 -define(WIDTH, 1024).
@@ -1167,7 +1165,7 @@ compare(Level, Bucket, Tree, Remote, AccFun, KeyAcc) ->
     Inter = ordsets:intersection(ordsets:from_list(HL1),
                                  ordsets:from_list(HL2)),
     Diff = ordsets:subtract(Union, Inter),
-    lager:debug("Tree ~p level ~p bucket ~p\nL=~p\nR=~p\nD=\n",
+    lager:debug("Tree ~p level ~p bucket ~p\nL=~p\nR=~p\nD=~p\n",
         [Tree, Level, Bucket, HL1, HL2, Diff]),
     KeyAcc3 =
         lists:foldl(fun({Bucket2, _}, KeyAcc2) ->
@@ -1584,6 +1582,7 @@ opened_closed_test() ->
 %%% EQC
 %%%===================================================================
 
+-ifdef(TEST).
 -ifdef(EQC).
 prop_sha() ->
     %% NOTE: Generating 1MB (1024 * 1024) size binaries is incredibly slow
@@ -1692,3 +1691,4 @@ prop_est() ->
                 true
             end).
 -endif.
+-endif.

src/riak_core.app.src

@@ -16,14 +16,13 @@
                   tools,
                   riak_sysmon,
                   os_mon,
-                  basho_stats,
                   eleveldb,
                   pbkdf2,
                   poolboy,
                   exometer_core,
                   clique,
                   cluster_info,
-                  gen_fsm_compat
+                  basho_stats
                  ]},
   {mod, {riak_core_app, []}},
   {env, [

src/riak_core_claim.erl

@@ -70,7 +70,7 @@
 -compile(export_all).
 -ifdef(EQC).
 -export([prop_claim_ensures_unique_nodes/1, prop_wants/0, prop_wants_counts/0,eqc_check/2,
-         prop_claim_ensures_unique_nodes_v2/0, prop_claim_ensures_unique_nodes_v3/0,
+         prop_claim_ensures_unique_nodes_v2/0, % prop_claim_ensures_unique_nodes_v3/0,
          prop_take_idxs/0]).
 -include_lib("eqc/include/eqc.hrl").
 -endif.
@@ -1364,15 +1364,27 @@ claim_ensures_unique_nodes_adding_singly_v2_test_() ->
     {timeout, 120, fun() -> ?assert(eqc:quickcheck(Prop)) end}.
 
 %% Run few tests in eunit and more if needed by calling "./rebar3 eqc"
-claim_ensures_unique_nodes_v3_test_() ->
-    Prop = eqc:numtests(5, ?QC_OUT(prop_claim_ensures_unique_nodes_old(choose_claim_v3))),
-    {timeout, 240, fun() -> ?assert(eqc:quickcheck(Prop)) end}.
+% claim_ensures_unique_nodes_v3_test_() ->
+%     Prop = eqc:numtests(5, ?QC_OUT(prop_claim_ensures_unique_nodes_old(choose_claim_v3))),
+%     {timeout, 240, fun() -> ?assert(eqc:quickcheck(Prop)) end}.
 
 prop_claim_ensures_unique_nodes_v2() ->
     prop_claim_ensures_unique_nodes(choose_claim_v2).
 
-prop_claim_ensures_unique_nodes_v3() ->
-    prop_claim_ensures_unique_nodes(choose_claim_v3).
+%% No longer test properties of claim_v3.
+%% Although claim_v3 continues to exist as a hidden configuration option, it
+%% is known to fail to meet the required properties, and claim_v2 should be
+%% used in all known circumstances.
+%%
+%% TODO : Remove claim_v3 from the code base
+%%
+%% This TODO is currently deferred due to the difficulty of understanding
+%% how to test the full possibility of cluster change scenarios.  Perhaps
+%% there may be circumstances where a probabilistic approach to planning
+%% cluster changes may still be beneficial
+%%
+% prop_claim_ensures_unique_nodes_v3() ->
+%    prop_claim_ensures_unique_nodes(choose_claim_v3).
 
 %% NOTE: this is a less than adequate test that has been re-instated
 %% so that we don't leave the code worse than we found it. Work that

src/riak_core_claim_sim.erl

@@ -554,7 +554,8 @@ run_test() ->
                           {analysis, [{failures, 2},{n_val, 3}]},
                           {print,Fh},
                           {return_ring, false}])),
-    file:close(Fh).
+    file:close(Fh),
+    file:delete("sim.out").
 
 
 %% Decided not to run by default, perhaps better as an

src/riak_core_coverage_fsm.erl

@@ -64,7 +64,11 @@
 
 -include("riak_core_vnode.hrl").
 
--behaviour(gen_fsm_compat).
+-behaviour(gen_fsm).
+
+-compile({nowarn_deprecated_function, 
+            [{gen_fsm, start_link, 3},
+                {gen_fsm, start_timer, 2}]}).
 
 -export([start_link/3]).
 
@@ -74,7 +78,7 @@
 -export([test_link/5]).
 -endif.
 
-%% gen_fsm_compat callbacks
+%% gen_fsm callbacks
 -export([init/1,
          initialize/2,
          waiting_results/2,
@@ -142,7 +146,7 @@
 -spec start_link(module(), from(), [term()]) ->
                         {ok, pid()} | ignore | {error, term()}.
 start_link(Mod, From, RequestArgs) ->
-    gen_fsm_compat:start_link(?MODULE, [Mod, From, RequestArgs], []).
+    gen_fsm:start_link(?MODULE, [Mod, From, RequestArgs], []).
 
 %% ===================================================================
 %% Test API
@@ -153,7 +157,7 @@ start_link(Mod, From, RequestArgs) ->
 %% Create a coverage FSM for testing.
 test_link(Mod, From, RequestArgs, _Options, StateProps) ->
     Timeout = 60000,
-    gen_fsm_compat:start_link(?MODULE,
+    gen_fsm:start_link(?MODULE,
                        {test,
                         [Mod,
                          From,
@@ -165,7 +169,7 @@ test_link(Mod, From, RequestArgs, _Options, StateProps) ->
 -endif.
 
 %% ====================================================================
-%% gen_fsm_compat callbacks
+%% gen_fsm callbacks
 %% ====================================================================
 
 %% @private
@@ -214,7 +218,7 @@ maybe_start_timeout_timer(infinity) ->
 maybe_start_timeout_timer(Bad) when not is_integer(Bad) ->
     maybe_start_timeout_timer(?DEFAULT_TIMEOUT);
 maybe_start_timeout_timer(Timeout) ->
-    gen_fsm_compat:start_timer(Timeout, {timer_expired, Timeout}),
+    gen_fsm:start_timer(Timeout, {timer_expired, Timeout}),
     ok.
 
 

src/riak_core_gen_server.erl

@@ -182,7 +182,7 @@
 %%%  Preprocessor
 %%%=========================================================================
 
--ifdef(deprecated_21).
+-ifdef(deprecated_22).
 get_log(Debug) ->
     sys:get_log(Debug).
 -else.

src/riak_core_handoff_receiver.erl

@@ -23,6 +23,10 @@
 -module(riak_core_handoff_receiver).
 -include("riak_core_handoff.hrl").
 -behaviour(riak_core_gen_server).
+
+-compile({nowarn_deprecated_function, 
+            [{gen_fsm, sync_send_all_state_event, 3}]}).
+
 -export([start_link/0,                          % Don't use SSL
          start_link/1,                          % SSL options list, empty=no SSL
          set_socket/2,
@@ -141,7 +145,7 @@ process_message(?PT_MSG_BATCH, MsgData, State) ->
 process_message(?PT_MSG_OBJ, MsgData, State=#state{vnode=VNode, count=Count,
                                                    vnode_timeout_len=VNodeTimeout}) ->
     Msg = {handoff_data, MsgData},
-    try gen_fsm_compat:sync_send_all_state_event(VNode, Msg, VNodeTimeout) of
+    try gen_fsm:sync_send_all_state_event(VNode, Msg, VNodeTimeout) of
         ok ->
             State#state{count=Count+1};
         E={error, _} ->

src/riak_core_handoff_sender.erl

@@ -22,6 +22,10 @@
 
 -module(riak_core_handoff_sender).
 -export([start_link/4, get_handoff_ssl_options/0]).
+
+-compile({nowarn_deprecated_function, 
+            [{gen_fsm, send_event, 2}]}).
+
 -include("riak_core_vnode.hrl").
 -include("riak_core_handoff.hrl").
 -include("stacktrace.hrl").
@@ -252,9 +256,9 @@ start_fold(TargetNode, Module, {Type, Opts}, ParentPid, SslOpts) ->
                              riak_core_format:human_size_fmt("~.2f", ThroughputBytes)]),
                  case Type of
                      repair -> ok;
-                     resize -> gen_fsm_compat:send_event(ParentPid, {resize_transfer_complete,
+                     resize -> gen_fsm:send_event(ParentPid, {resize_transfer_complete,
                                                                        NotSentAcc});
-                     _ -> gen_fsm_compat:send_event(ParentPid, handoff_complete)
+                     _ -> gen_fsm:send_event(ParentPid, handoff_complete)
                  end;
              {error, ErrReason} ->
                  if ErrReason == timeout ->
@@ -274,15 +278,15 @@ start_fold(TargetNode, Module, {Type, Opts}, ParentPid, SslOpts) ->
              exit({shutdown, timeout});
          exit:{shutdown, {error, Reason}} ->
              ?log_fail("because of ~p", [Reason]),
-             gen_fsm_compat:send_event(ParentPid, {handoff_error,
+             gen_fsm:send_event(ParentPid, {handoff_error,
                                             fold_error, Reason}),
              exit({shutdown, {error, Reason}});
          throw:{be_quiet, Err, Reason} ->
-             gen_fsm_compat:send_event(ParentPid, {handoff_error, Err, Reason});
+             gen_fsm:send_event(ParentPid, {handoff_error, Err, Reason});
          ?_exception_(Err, Reason, StackToken) ->
              ?log_fail("because of ~p:~p ~p",
                        [Err, Reason, ?_get_stacktrace_(StackToken)]),
-             gen_fsm_compat:send_event(ParentPid, {handoff_error, Err, Reason})
+             gen_fsm:send_event(ParentPid, {handoff_error, Err, Reason})
      end.
 
 start_visit_item_timer() ->