Skip to content
This repository has been archived by the owner on Jun 23, 2022. It is now read-only.

Commit

Permalink
fix: restrict the replica count while creating app
Browse files Browse the repository at this point in the history
  • Loading branch information
@empiredan
empiredan committed Nov 25, 2021
1 parent 71664b8 commit e4113ef
Show file tree
Hide file tree
Showing 2 changed files with 81 additions and 34 deletions.
4 changes: 2 additions & 2 deletions src/meta/server_state.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1085,7 +1085,7 @@ void server_state::create_app(dsn::message_ex *msg)

auto level = _meta_svc->get_function_level();
if (level <= meta_function_level::fl_freezed) {
dwarn_f("current meta function level is freezed since there are too few alive nodes");
derror_f("current meta function level is freezed since there are too few alive nodes");
response.err = ERR_STATE_FREEZED;
will_create_app = false;
} else if (request.options.partition_count <= 0 ||
Expand Down Expand Up @@ -2925,7 +2925,7 @@ bool server_state::validate_target_max_replica_count(int32_t max_replica_count)
bool is_valid = validate_target_max_replica_count_internal(
max_replica_count, alive_node_count, hint_message);
if (!is_valid) {
dwarn_f("target max replica count is invalid: message={}", hint_message);
derror_f("target max replica count is invalid: message={}", hint_message);
}

return is_valid;
Expand Down
111 changes: 79 additions & 32 deletions src/meta/test/meta_app_operation_test.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
// under the License.

#include <gtest/gtest.h>
#include <dsn/dist/fmt_logging.h>
#include <dsn/service_api_c.h>

#include "meta_service_test_app.h"
Expand Down Expand Up @@ -107,10 +108,38 @@ class meta_app_operation_test : public meta_test_base

TEST_F(meta_app_operation_test, create_app)
{
// Test cases:
// Test cases: (assert min_allowed_replica_count <= max_allowed_replica_count)
// - wrong partition_count (< 0)
// - wrong partition_count (= 0)
// - wrong replica_count (< 0)
// - wrong replica_count (= 0)
// - wrong replica_count (> max_allowed_replica_count > alive_node_count)
// - wrong replica_count (> alive_node_count > max_allowed_replica_count)
// - wrong replica_count (> alive_node_count = max_allowed_replica_count)
// - wrong replica_count (= max_allowed_replica_count, and > alive_node_count)
// - wrong replica_count (< max_allowed_replica_count, and > alive_node_count)
// - wrong replica_count (= alive_node_count, and > max_allowed_replica_count)
// - wrong replica_count (< alive_node_count, and > max_allowed_replica_count)
// - valid replica_count (= max_allowed_replica_count, and = alive_node_count)
// - valid replica_count (= max_allowed_replica_count, and < alive_node_count)
// - valid replica_count (< max_allowed_replica_count, and = alive_node_count)
// - valid replica_count (< max_allowed_replica_count < alive_node_count)
// - valid replica_count (< alive_node_count < max_allowed_replica_count)
// - valid replica_count (< alive_node_count = max_allowed_replica_count)
// - wrong replica_count (< min_allowed_replica_count < alive_node_count)
// - wrong replica_count (< alive_node_count < min_allowed_replica_count)
// - wrong replica_count (< min_allowed_replica_count = alive_node_count)
// - wrong replica_count (< min_allowed_replica_count, and > alive_node_count)
// - wrong replica_count (< min_allowed_replica_count, and = alive_node_count)
// - wrong replica_count (= min_allowed_replica_count, and > alive_node_count)
// - valid replica_count (= min_allowed_replica_count, and < alive_node_count)
// - cluster freezed (alive_node_count = 0)
// - cluster freezed (alive_node_count = 1 < min_live_node_count_for_unfreeze)
// - cluster freezed (alive_node_count = 2 < min_live_node_count_for_unfreeze)
// - cluster not freezed (alive_node_count = min_live_node_count_for_unfreeze)
// - create succeed with single-replica
// - create succeed with double-replica
// - create app succeed
// - wrong partition_count
// - wrong replica_count
// - create failed with table existed
// - wrong app_status creating
// - wrong app_status recalling
Expand All @@ -123,52 +152,70 @@ TEST_F(meta_app_operation_test, create_app)
int32_t partition_count;
int32_t replica_count;
uint64_t min_live_node_count_for_unfreeze;
int32_t unalive_node_count;
int alive_node_count;
int32_t min_allowed_replica_count;
bool success_if_exist;
app_status::type before_status;
error_code expected_err;
} tests[] = {{APP_NAME, -1, 3, 2, 0, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 0, 3, 2, 0, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, -1, 1, 0, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 0, 1, 0, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 16, 2, 0, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 15, 2, 0, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 1, 1, 3, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED},
{APP_NAME, 4, 1, 1, 2, 3, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME + "_1", 4, 1, 1, 2, 1, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME, 4, 2, 1, 3, 2, false, app_status::AS_INVALID, ERR_STATE_FREEZED},
{APP_NAME, 4, 2, 1, 2, 2, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 2, 1, 1, 3, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME + "_2", 4, 2, 1, 1, 2, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME, 4, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED},
{APP_NAME, 4, 3, 2, 2, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED},
{APP_NAME, 4, 3, 2, 1, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 3, 2, 0, 5, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME + "_3", 4, 3, 2, 0, 3, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME, 4, 3, 2, 0, 1, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME, 4, 3, 2, 0, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 3, 2, 0, 1, false, app_status::AS_CREATING, ERR_BUSY_CREATING},
{APP_NAME, 4, 3, 2, 0, 1, false, app_status::AS_RECALLING, ERR_BUSY_CREATING},
{APP_NAME, 4, 3, 2, 0, 1, false, app_status::AS_DROPPING, ERR_BUSY_DROPPING},
{APP_NAME, 4, 3, 2, 0, 1, false, app_status::AS_DROPPED, ERR_OK},
{APP_NAME, 4, 3, 2, 0, 1, true, app_status::AS_INVALID, ERR_OK}};
} tests[] = {{APP_NAME, -1, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 0, 3, 2, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, -1, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 0, 1, 3, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 16, 2, 14, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 17, 2, 16, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 16, 2, 15, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 15, 2, 14, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 14, 2, 13, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 16, 2, 16, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 16, 2, 17, 1, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME + "_1", 4, 15, 2, 15, 1, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME + "_2", 4, 15, 2, 16, 1, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME + "_3", 4, 14, 2, 14, 1, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME + "_4", 4, 14, 2, 16, 1, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME + "_5", 4, 13, 2, 14, 1, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME + "_6", 4, 14, 2, 15, 1, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME, 4, 3, 2, 5, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 3, 2, 4, 5, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 3, 2, 4, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 3, 2, 2, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 3, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 4, 2, 3, 4, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME + "_7", 4, 3, 2, 4, 3, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME, 4, 1, 1, 0, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED},
{APP_NAME, 4, 2, 2, 1, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED},
{APP_NAME, 4, 3, 3, 2, 1, false, app_status::AS_INVALID, ERR_STATE_FREEZED},
{APP_NAME + "_8", 4, 3, 3, 3, 1, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME + "_9", 4, 1, 1, 1, 1, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME + "_10", 4, 2, 1, 2, 2, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_OK},
{APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_INVALID, ERR_INVALID_PARAMETERS},
{APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_CREATING, ERR_BUSY_CREATING},
{APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_RECALLING, ERR_BUSY_CREATING},
{APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPING, ERR_BUSY_DROPPING},
{APP_NAME, 4, 3, 2, 3, 3, false, app_status::AS_DROPPED, ERR_OK},
{APP_NAME, 4, 3, 2, 3, 3, true, app_status::AS_INVALID, ERR_OK}};

clear_nodes();
std::vector<rpc_address> nodes = ensure_enough_alive_nodes(3);

const int total_node_count = 20;
std::vector<rpc_address> nodes = ensure_enough_alive_nodes(total_node_count);

// the meta function level will become freezed once
// alive_nodes * 100 < total_nodes * node_live_percentage_threshold_for_update
// even if alive_nodes >= min_live_node_count_for_unfreeze
set_node_live_percentage_threshold_for_update(30);
set_node_live_percentage_threshold_for_update(0);

auto reserved_min_allowed_replica_count = FLAGS_min_allowed_replica_count;

for (auto test : tests) {
FLAGS_min_allowed_replica_count = test.min_allowed_replica_count;
set_min_live_node_count_for_unfreeze(test.min_live_node_count_for_unfreeze);

for (int32_t i = 0; i < test.unalive_node_count; i++) {
dassert_f(total_node_count >= test.alive_node_count,
"total_node_count({}) should be >= alive_node_count({})",
total_node_count,
test.alive_node_count);
for (int i = 0; i < total_node_count - test.alive_node_count; i++) {
_ms->set_node_state({nodes[i]}, false);
}

Expand Down

0 comments on commit e4113ef

Please sign in to comment.