Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(collector): add statistics for estimate key number of partition #435

Merged
merged 12 commits into from
Dec 3, 2019
10 changes: 10 additions & 0 deletions src/server/pegasus_server_impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -338,6 +338,10 @@ pegasus_server_impl::pegasus_server_impl(dsn::replication::replica *r)
snprintf(name, 255, "rdb.memtable.memory_usage@%s", str_gpid.c_str());
_pfc_rdb_memtable_mem_usage.init_app_counter(
"app.pegasus", name, COUNTER_TYPE_NUMBER, "statistic the memory usage of rocksdb memtable");

snprintf(name, 255, "rdb.estimate.key_number@%s", str_gpid.c_str());
neverchanje marked this conversation as resolved.
Show resolved Hide resolved
foreverneverer marked this conversation as resolved.
Show resolved Hide resolved
_pfc_rdb_estimate_key_number.init_app_counter(
"app.pegasus", name, COUNTER_TYPE_NUMBER, "statistic the key number of rocksdb all data");
}

void pegasus_server_impl::parse_checkpoints()
Expand Down Expand Up @@ -2242,6 +2246,12 @@ void pegasus_server_impl::update_replica_rocksdb_statistics()
_pfc_rdb_memtable_mem_usage->set(val);
dinfo_replica("_pfc_rdb_memtable_mem_usage: {} bytes", val);
}

if (_db->GetProperty(rocksdb::DB::Properties::kEstimateNumKeys, &str_val) &&
dsn::buf2uint64(str_val, val)) {
_pfc_rdb_estimate_key_number->set(val);
dinfo_replica("_pfc_rdb_estimate_key_number: {}", val);
}
}

void pegasus_server_impl::update_server_rocksdb_statistics()
Expand Down
1 change: 1 addition & 0 deletions src/server/pegasus_server_impl.h
Original file line number Diff line number Diff line change
Expand Up @@ -364,6 +364,7 @@ class pegasus_server_impl : public ::dsn::apps::rrdb_service
::dsn::perf_counter_wrapper _pfc_rdb_block_cache_total_count;
::dsn::perf_counter_wrapper _pfc_rdb_index_and_filter_blocks_mem_usage;
::dsn::perf_counter_wrapper _pfc_rdb_memtable_mem_usage;
::dsn::perf_counter_wrapper _pfc_rdb_estimate_key_number;
};

} // namespace server
Expand Down
37 changes: 36 additions & 1 deletion src/server/test/pegasus_server_impl_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,26 @@
// can be found in the LICENSE file in the root directory of this source tree.

#include <base/pegasus_key_schema.h>
#include <dsn/utility/defer.h>
#include "server/pegasus_server_write.h"
#include "server/pegasus_write_service_impl.h"
#include "pegasus_server_test_base.h"
#include "message_utils.h"

namespace pegasus {
namespace server {

class pegasus_server_impl_test : public pegasus_server_test_base
{

std::unique_ptr<pegasus_server_write> _server_write;

public:
pegasus_server_impl_test() : pegasus_server_test_base() { start(); }
pegasus_server_impl_test() : pegasus_server_test_base()
{
start();
_server_write = dsn::make_unique<pegasus_server_write>(_server.get(), true);
}

void test_table_level_slow_query()
{
Expand Down Expand Up @@ -55,9 +66,33 @@ class pegasus_server_impl_test : public pegasus_server_test_base
ASSERT_EQ(before_count + test.expect_perf_counter_incr, after_count);
}
}

void test_table_property()
neverchanje marked this conversation as resolved.
Show resolved Hide resolved
{
std::unique_ptr<pegasus_server_write> _server_write;
foreverneverer marked this conversation as resolved.
Show resolved Hide resolved
_server_write = dsn::make_unique<pegasus_server_write>(_server.get(), true);
dsn::blob key;
pegasus_generate_key(key, std::string("hash"), std::string("sort"));
dsn::apps::update_request req;
req.key = key;
req.value.assign("value", 0, 5);
foreverneverer marked this conversation as resolved.
Show resolved Hide resolved

int put_rpc_cnt = 5;
foreverneverer marked this conversation as resolved.
Show resolved Hide resolved
auto writes = new dsn::message_ex *[put_rpc_cnt];
foreverneverer marked this conversation as resolved.
Show resolved Hide resolved
for (int i = 0; i < put_rpc_cnt; i++) {
writes[i] = pegasus::create_put_request(req);
}
_server_write->on_batched_write_requests(writes, put_rpc_cnt, 0, 0);

std::string str_val;
_server->_db->GetProperty(rocksdb::DB::Properties::kEstimateNumKeys, &str_val);
ASSERT_EQ(str_val, std::to_string(put_rpc_cnt));
neverchanje marked this conversation as resolved.
Show resolved Hide resolved
foreverneverer marked this conversation as resolved.
Show resolved Hide resolved
}
};

TEST_F(pegasus_server_impl_test, test_table_level_slow_query) { test_table_level_slow_query(); }

TEST_F(pegasus_server_impl_test, test_table_property) { test_table_property(); }
neverchanje marked this conversation as resolved.
Show resolved Hide resolved

} // namespace server
} // namespace pegasus