-
Notifications
You must be signed in to change notification settings - Fork 312
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat(rocksdb): Adapt prefix bloom filter to speedup scans by hashkey (#…
…438)
- Loading branch information
Showing
8 changed files
with
228 additions
and
20 deletions.
There are no files selected for viewing
Submodule rdsn
updated
4 files
| +15 −0 | src/dist/http/http_server.cpp | |
| +78 −0 | src/dist/http/test/uri_decoder_test.cpp | |
| +69 −0 | src/dist/http/uri_decoder.cpp | |
| +18 −0 | src/dist/http/uri_decoder.h |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,51 @@ | ||
| // Copyright (c) 2017, Xiaomi, Inc. All rights reserved. | ||
| // This source code is licensed under the Apache License Version 2.0, which | ||
| // can be found in the LICENSE file in the root directory of this source tree. | ||
|
|
||
| #pragma once | ||
|
|
||
| #include <rocksdb/slice_transform.h> | ||
| #include <rocksdb/slice.h> | ||
|
|
||
| #include <dsn/c/api_utilities.h> | ||
| #include <dsn/utility/blob.h> | ||
|
|
||
| namespace pegasus { | ||
| namespace server { | ||
|
|
||
| class HashkeyTransform : public rocksdb::SliceTransform | ||
| { | ||
| public: | ||
| HashkeyTransform() = default; | ||
|
|
||
| // NOTE: You must change the name if Transform() algorithm changed. | ||
| const char *Name() const override { return "pegasus.HashkeyTransform"; } | ||
|
|
||
| rocksdb::Slice Transform(const rocksdb::Slice &src) const override | ||
| { | ||
| // TODO(yingchun): There is a bug in rocksdb 5.9.2, it has been fixed by | ||
| // cca141ecf8634a42b5eb548cb0ac3a6b77d783c1, we can remove this judgement after upgrading | ||
| // rocksdb. | ||
| if (src.size() < 2) { | ||
| return src; | ||
| } | ||
|
|
||
| // hash_key_len is in big endian | ||
| uint16_t hash_key_len = be16toh(*(int16_t *)(src.data())); | ||
| dassert(src.size() >= 2 + hash_key_len, | ||
| "key length must be no less than (2 + hash_key_len)"); | ||
| return rocksdb::Slice(src.data(), 2 + hash_key_len); | ||
| } | ||
|
|
||
| bool InDomain(const rocksdb::Slice &src) const override | ||
| { | ||
| // Empty put keys are not in domain. | ||
| return src.size() >= 2; | ||
| } | ||
|
|
||
| bool InRange(const rocksdb::Slice &dst) const override { return true; } | ||
|
|
||
| bool SameResultWhenAppended(const rocksdb::Slice &prefix) const override { return false; } | ||
| }; | ||
| } // namespace server | ||
| } // namespace pegasus |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,64 @@ | ||
| // Copyright (c) 2017, Xiaomi, Inc. All rights reserved. | ||
| // This source code is licensed under the Apache License Version 2.0, which | ||
| // can be found in the LICENSE file in the root directory of this source tree. | ||
|
|
||
| #include "server/hashkey_transform.h" | ||
|
|
||
| #include <gtest/gtest.h> | ||
| #include <rocksdb/comparator.h> | ||
|
|
||
| #include "base/pegasus_key_schema.h" | ||
|
|
||
| // User define SliceTransform must obey the 4 rules of ColumnFamilyOptions.prefix_extractor: | ||
| // 1) key.starts_with(prefix(key)) | ||
| // 2) Compare(prefix(key), key) <= 0. | ||
| // 3) If Compare(k1, k2) <= 0, then Compare(prefix(k1), prefix(k2)) <= 0 | ||
| // 4) prefix(prefix(key)) == prefix(key) | ||
| TEST(HashkeyTransformTest, Basic) | ||
| { | ||
| pegasus::server::HashkeyTransform prefix_extractor; | ||
| const rocksdb::Comparator *comp = rocksdb::BytewiseComparator(); | ||
|
|
||
| dsn::blob bkey1, bkey2, bkey3, bkey4; | ||
| pegasus::pegasus_generate_key(bkey1, std::string("h1"), std::string("s1")); | ||
| pegasus::pegasus_generate_key(bkey2, std::string("h2"), std::string("s1")); | ||
| pegasus::pegasus_generate_key(bkey3, std::string("h1"), std::string("s2")); | ||
| pegasus::pegasus_generate_key(bkey4, std::string("h1"), std::string("")); | ||
| rocksdb::Slice skey1(bkey1.data(), bkey1.size()); | ||
| rocksdb::Slice skey2(bkey2.data(), bkey2.size()); | ||
| rocksdb::Slice skey3(bkey3.data(), bkey3.size()); | ||
| rocksdb::Slice skey4(bkey4.data(), bkey4.size()); | ||
|
|
||
| // 1) key.starts_with(prefix(key)) | ||
| ASSERT_TRUE(skey1.starts_with(prefix_extractor.Transform(skey1))); | ||
| ASSERT_TRUE(skey2.starts_with(prefix_extractor.Transform(skey2))); | ||
| ASSERT_TRUE(skey3.starts_with(prefix_extractor.Transform(skey3))); | ||
| ASSERT_TRUE(skey4.starts_with(prefix_extractor.Transform(skey4))); | ||
|
|
||
| // 2) Compare(prefix(key), key) <= 0. | ||
| ASSERT_LT(comp->Compare(prefix_extractor.Transform(skey1), skey1), 0); // h1 < h1s1 | ||
| ASSERT_LT(comp->Compare(prefix_extractor.Transform(skey2), skey2), 0); // h2 < h2s1 | ||
| ASSERT_LT(comp->Compare(prefix_extractor.Transform(skey3), skey3), 0); // h1 < h1s2 | ||
| ASSERT_EQ(comp->Compare(prefix_extractor.Transform(skey4), skey4), 0); // h1 == h1 | ||
|
|
||
| // 3) If Compare(k1, k2) <= 0, then Compare(prefix(k1), prefix(k2)) <= 0 | ||
| ASSERT_LT(comp->Compare(skey1, skey2), 0); // h1s1 < h2s1 | ||
| ASSERT_LT(comp->Compare(prefix_extractor.Transform(skey1), prefix_extractor.Transform(skey2)), | ||
| 0); // h1 < h2 | ||
| ASSERT_LT(comp->Compare(skey1, skey3), 0); // h1s1 < h1s2 | ||
| ASSERT_EQ(comp->Compare(prefix_extractor.Transform(skey1), prefix_extractor.Transform(skey3)), | ||
| 0); // h1 == h1 | ||
| ASSERT_GT(comp->Compare(skey1, skey4), 0); // h1s1 > h1 | ||
| ASSERT_EQ(comp->Compare(prefix_extractor.Transform(skey1), prefix_extractor.Transform(skey4)), | ||
| 0); // h1 == h1 | ||
|
|
||
| // 4) prefix(prefix(key)) == prefix(key) | ||
| ASSERT_EQ(prefix_extractor.Transform(prefix_extractor.Transform(skey1)), | ||
| prefix_extractor.Transform(skey1)); | ||
| ASSERT_EQ(prefix_extractor.Transform(prefix_extractor.Transform(skey2)), | ||
| prefix_extractor.Transform(skey2)); | ||
| ASSERT_EQ(prefix_extractor.Transform(prefix_extractor.Transform(skey3)), | ||
| prefix_extractor.Transform(skey3)); | ||
| ASSERT_EQ(prefix_extractor.Transform(prefix_extractor.Transform(skey4)), | ||
| prefix_extractor.Transform(skey4)); | ||
| } |
Oops, something went wrong.