From eae740b4fc360b09e646526596091082261350c2 Mon Sep 17 00:00:00 2001 From: levy Date: Mon, 31 May 2021 10:34:25 +0800 Subject: [PATCH 1/5] feat: add hashkey pattern rule --- rdsn | 2 +- src/server/compaction_filter_rule.cpp | 60 +++++++++++++++++++ src/server/compaction_filter_rule.h | 31 ++++++++++ .../test/compaction_filter_rule_test.cpp | 60 +++++++++++++++++++ 4 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 src/server/compaction_filter_rule.cpp create mode 100644 src/server/test/compaction_filter_rule_test.cpp diff --git a/rdsn b/rdsn index e52e9a0d94..5b8e4f6a6a 160000 --- a/rdsn +++ b/rdsn @@ -1 +1 @@ -Subproject commit e52e9a0d94471fad442c6d3f3d46ebe1f7ffa23f +Subproject commit 5b8e4f6a6a2eaf23ec2a54eebf2e72c11a82e86e diff --git a/src/server/compaction_filter_rule.cpp b/src/server/compaction_filter_rule.cpp new file mode 100644 index 0000000000..b356091f36 --- /dev/null +++ b/src/server/compaction_filter_rule.cpp @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include "compaction_filter_rule.h" + +#include +#include +#include + +namespace pegasus { +namespace server { +bool string_pattern_match(const std::string &value, + string_match_type type, + const std::string &filter_pattern) +{ + if (filter_pattern.length() == 0) + return false; + if (value.length() < filter_pattern.length()) + return false; + + switch (type) { + case string_match_type::SMT_MATCH_ANYWHERE: + return dsn::string_view(value).find(filter_pattern) != dsn::string_view::npos; + case string_match_type::SMT_MATCH_PREFIX: + return memcmp(value.data(), filter_pattern.data(), filter_pattern.length()) == 0; + case string_match_type::SMT_MATCH_POSTFIX: + return memcmp(value.data() + value.length() - filter_pattern.length(), + filter_pattern.data(), + filter_pattern.length()) == 0; + default: + derror_f("invalid match type {}", type); + return false; + } +} + +bool hashkey_pattern_rule::match(const std::string &hash_key, + const std::string &sort_key, + const rocksdb::Slice &existing_value) const +{ + return string_pattern_match(hash_key, match_type, pattern); +} + +} // namespace server +} // namespace pegasus diff --git a/src/server/compaction_filter_rule.h b/src/server/compaction_filter_rule.h index 2c18209379..ad5c2ffa2b 100644 --- a/src/server/compaction_filter_rule.h +++ b/src/server/compaction_filter_rule.h @@ -20,6 +20,8 @@ #pragma once #include +#include +#include namespace pegasus { namespace server { @@ -36,5 +38,34 @@ class compaction_filter_rule const std::string &sort_key, const rocksdb::Slice &existing_value) const = 0; }; + +enum string_match_type +{ + SMT_MATCH_ANYWHERE, + SMT_MATCH_PREFIX, + SMT_MATCH_POSTFIX, + SMT_INVALID, +}; +ENUM_BEGIN(string_match_type, SMT_INVALID) +ENUM_REG(SMT_MATCH_ANYWHERE) +ENUM_REG(SMT_MATCH_PREFIX) +ENUM_REG(SMT_MATCH_POSTFIX) +ENUM_END(string_match_type) + +class hashkey_pattern_rule : public compaction_filter_rule +{ +public: + explicit hashkey_pattern_rule(uint32_t pegasus_data_version) = default; + + bool match(const std::string &hash_key, + const std::string &sort_key, + const rocksdb::Slice &existing_value) const; + +private: + std::string pattern; + string_match_type match_type; + + FRIEND_TEST(hashkey_pattern_rule_test, match); +}; } // namespace server } // namespace pegasus diff --git a/src/server/test/compaction_filter_rule_test.cpp b/src/server/test/compaction_filter_rule_test.cpp new file mode 100644 index 0000000000..b9f7e52af7 --- /dev/null +++ b/src/server/test/compaction_filter_rule_test.cpp @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include "server/compaction_filter_rule.h" + +namespace pegasus { +namespace server { + +TEST(hashkey_pattern_rule_test, match) +{ + struct test_case + { + std::string hashkey; + std::string pattern; + string_match_type match_type; + bool match; + } tests[] = { + {"sortkey", "", SMT_MATCH_ANYWHERE, false}, + {"hashkey", "hashkey", SMT_MATCH_ANYWHERE, true}, + {"hashkey", "shke", SMT_MATCH_ANYWHERE, true}, + {"hashkey", "hash", SMT_MATCH_ANYWHERE, true}, + {"hashkey", "key", SMT_MATCH_ANYWHERE, true}, + {"hashkey", "sortkey", SMT_MATCH_ANYWHERE, false}, + {"hashkey", "hashkey", SMT_MATCH_PREFIX, true}, + {"hashkey", "hash", SMT_MATCH_PREFIX, true}, + {"hashkey", "key", SMT_MATCH_PREFIX, false}, + {"hashkey", "sortkey", SMT_MATCH_PREFIX, false}, + {"hashkey", "hashkey", SMT_MATCH_POSTFIX, true}, + {"hashkey", "hash", SMT_MATCH_POSTFIX, false}, + {"hashkey", "key", SMT_MATCH_POSTFIX, true}, + {"hashkey", "sortkey", SMT_MATCH_POSTFIX, false}, + }; + + rocksdb::Slice slice; + hashkey_pattern_rule rule(1); + for (const auto &test : tests) { + rule.match_type = test.match_type; + rule.pattern = test.pattern; + ASSERT_EQ(rule.match(test.hashkey, "", slice), test.match); + } +} +} // namespace server +} // namespace pegasus From c9c1600798006c38d27772b8868bbd2f5d1f7906 Mon Sep 17 00:00:00 2001 From: levy Date: Mon, 31 May 2021 11:10:18 +0800 Subject: [PATCH 2/5] fix --- src/server/compaction_filter_rule.cpp | 2 ++ src/server/compaction_filter_rule.h | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/server/compaction_filter_rule.cpp b/src/server/compaction_filter_rule.cpp index b356091f36..42c0161b49 100644 --- a/src/server/compaction_filter_rule.cpp +++ b/src/server/compaction_filter_rule.cpp @@ -49,6 +49,8 @@ bool string_pattern_match(const std::string &value, } } +hashkey_pattern_rule::hashkey_pattern_rule(uint32_t pegasus_data_version) {} + bool hashkey_pattern_rule::match(const std::string &hash_key, const std::string &sort_key, const rocksdb::Slice &existing_value) const diff --git a/src/server/compaction_filter_rule.h b/src/server/compaction_filter_rule.h index ad5c2ffa2b..556717d727 100644 --- a/src/server/compaction_filter_rule.h +++ b/src/server/compaction_filter_rule.h @@ -55,7 +55,7 @@ ENUM_END(string_match_type) class hashkey_pattern_rule : public compaction_filter_rule { public: - explicit hashkey_pattern_rule(uint32_t pegasus_data_version) = default; + explicit hashkey_pattern_rule(uint32_t pegasus_data_version); bool match(const std::string &hash_key, const std::string &sort_key, From 8320edabec75a0c594ff8c87437db06ee2f24f65 Mon Sep 17 00:00:00 2001 From: levy Date: Mon, 31 May 2021 11:16:57 +0800 Subject: [PATCH 3/5] fix --- src/server/test/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/server/test/CMakeLists.txt b/src/server/test/CMakeLists.txt index 80dc4d6403..1beddd4172 100644 --- a/src/server/test/CMakeLists.txt +++ b/src/server/test/CMakeLists.txt @@ -29,7 +29,8 @@ set(MY_PROJ_SRC "../pegasus_server_impl.cpp" "../meta_store.cpp" "../hotkey_collector.cpp" "../rocksdb_wrapper.cpp" -) + "../compaction_filter_rule.cpp" + ) set(MY_SRC_SEARCH_MODE "GLOB") From fdc5593bf4bf494605bedeb1a13265e91d0d14a7 Mon Sep 17 00:00:00 2001 From: levy Date: Mon, 31 May 2021 17:07:37 +0800 Subject: [PATCH 4/5] fix --- src/server/compaction_filter_rule.cpp | 4 +--- src/server/compaction_filter_rule.h | 2 +- src/server/test/compaction_filter_rule_test.cpp | 4 +++- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/server/compaction_filter_rule.cpp b/src/server/compaction_filter_rule.cpp index 42c0161b49..7e64e9fdac 100644 --- a/src/server/compaction_filter_rule.cpp +++ b/src/server/compaction_filter_rule.cpp @@ -29,7 +29,7 @@ bool string_pattern_match(const std::string &value, string_match_type type, const std::string &filter_pattern) { - if (filter_pattern.length() == 0) + if (filter_pattern.empty()) return false; if (value.length() < filter_pattern.length()) return false; @@ -49,8 +49,6 @@ bool string_pattern_match(const std::string &value, } } -hashkey_pattern_rule::hashkey_pattern_rule(uint32_t pegasus_data_version) {} - bool hashkey_pattern_rule::match(const std::string &hash_key, const std::string &sort_key, const rocksdb::Slice &existing_value) const diff --git a/src/server/compaction_filter_rule.h b/src/server/compaction_filter_rule.h index 556717d727..435aeb5cb8 100644 --- a/src/server/compaction_filter_rule.h +++ b/src/server/compaction_filter_rule.h @@ -55,7 +55,7 @@ ENUM_END(string_match_type) class hashkey_pattern_rule : public compaction_filter_rule { public: - explicit hashkey_pattern_rule(uint32_t pegasus_data_version); + explicit hashkey_pattern_rule() = default; bool match(const std::string &hash_key, const std::string &sort_key, diff --git a/src/server/test/compaction_filter_rule_test.cpp b/src/server/test/compaction_filter_rule_test.cpp index b9f7e52af7..0d9d3fde8c 100644 --- a/src/server/test/compaction_filter_rule_test.cpp +++ b/src/server/test/compaction_filter_rule_test.cpp @@ -46,10 +46,12 @@ TEST(hashkey_pattern_rule_test, match) {"hashkey", "hash", SMT_MATCH_POSTFIX, false}, {"hashkey", "key", SMT_MATCH_POSTFIX, true}, {"hashkey", "sortkey", SMT_MATCH_POSTFIX, false}, + {"hash", "hashkey", SMT_MATCH_POSTFIX, false}, + {"hashkey", "hashkey", SMT_INVALID, false}, }; rocksdb::Slice slice; - hashkey_pattern_rule rule(1); + hashkey_pattern_rule rule; for (const auto &test : tests) { rule.match_type = test.match_type; rule.pattern = test.pattern; From 3158a0ccb2bf8ecd9a181503e3b530d5b988ab67 Mon Sep 17 00:00:00 2001 From: levy Date: Mon, 31 May 2021 17:09:10 +0800 Subject: [PATCH 5/5] fix --- rdsn | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rdsn b/rdsn index 5b8e4f6a6a..e52e9a0d94 160000 --- a/rdsn +++ b/rdsn @@ -1 +1 @@ -Subproject commit 5b8e4f6a6a2eaf23ec2a54eebf2e72c11a82e86e +Subproject commit e52e9a0d94471fad442c6d3f3d46ebe1f7ffa23f