Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(split): scan support validate_partition_hash #702

Merged
merged 7 commits into from
Mar 31, 2021
Merged

feat(split): scan support validate_partition_hash #702

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
0c54b49
feat(split): scan support validate_partition_hash
hycdong Mar 23, 2021
d3646fc
Merge branch 'pegasus' into scan_validate_hash
hycdong Mar 24, 2021
bb362a9
update by cr
hycdong Mar 24, 2021
633fbb2
Merge branch 'master' into scan_validate_hash
hycdong Mar 26, 2021
8c85d55
format
hycdong Mar 26, 2021
5480637
Merge branch 'pegasus' into scan_validate_hash
hycdong Mar 26, 2021
976ec69
Merge branch 'master' into scan_validate_hash
hycdong Mar 31, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion rdsn
Open in desktop
Submodule rdsn updated 12 files
+1 −0 include/dsn/dist/replication/replication.codes.h
+27 −0 src/common/backup.thrift
+19 −6 src/meta/backup_engine.cpp
+8 −2 src/meta/backup_engine.h
+45 −10 src/meta/meta_backup_service.cpp
+6 −11 src/meta/meta_backup_service.h
+9 −0 src/meta/meta_rpc_types.h
+15 −0 src/meta/meta_service.cpp
+3 −0 src/meta/meta_service.h
+2 −0 src/meta/server_state.h
+1 −1 src/meta/test/clear.sh
+248 −0 src/meta/test/meta_backup_test.cpp
48 changes: 29 additions & 19 deletions src/base/rrdb_types.cpp
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions src/client_lib/pegasus_client_impl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1179,7 +1179,7 @@ int pegasus_client_impl::get_scanner(const std::string &hash_key,
if (c < 0 || (c == 0 && o.start_inclusive && o.stop_inclusive)) {
v.push_back(pegasus_key_hash(start));
}
scanner = new pegasus_scanner_impl(_client, std::move(v), o, start, stop);
scanner = new pegasus_scanner_impl(_client, std::move(v), o, start, stop, false);

return PERR_OK;
}
Expand Down Expand Up @@ -1223,7 +1223,7 @@ void pegasus_client_impl::async_get_unordered_scanners(
std::vector<uint64_t> hash(s);
for (int j = 0; j < s; j++)
hash[j] = --count;
scanners[i] = new pegasus_scanner_impl(_client, std::move(hash), options);
scanners[i] = new pegasus_scanner_impl(_client, std::move(hash), options, true);
}
}
}
Expand Down
7 changes: 5 additions & 2 deletions src/client_lib/pegasus_client_impl.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,12 +265,14 @@ class pegasus_client_impl : public pegasus_client

pegasus_scanner_impl(::dsn::apps::rrdb_client *client,
std::vector<uint64_t> &&hash,
const scan_options &options);
const scan_options &options,
bool validate_partition_hash);
levy5307 marked this conversation as resolved.
Show resolved Hide resolved
pegasus_scanner_impl(::dsn::apps::rrdb_client *client,
std::vector<uint64_t> &&hash,
const scan_options &options,
const ::dsn::blob &start_key,
const ::dsn::blob &stop_key);
const ::dsn::blob &stop_key,
bool validate_partition_hash);

private:
::dsn::apps::rrdb_client *_client;
Expand All @@ -288,6 +290,7 @@ class pegasus_client_impl : public pegasus_client
mutable ::dsn::zlock _lock;
std::list<async_scan_next_callback_t> _queue;
volatile bool _rpc_started;
bool _validate_partition_hash;

void _async_next_internal();
void _start_scan();
Expand Down
12 changes: 8 additions & 4 deletions src/client_lib/pegasus_scanner_impl.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,9 @@ namespace client {

pegasus_client_impl::pegasus_scanner_impl::pegasus_scanner_impl(::dsn::apps::rrdb_client *client,
std::vector<uint64_t> &&hash,
const scan_options &options)
: pegasus_scanner_impl(client, std::move(hash), options, _min, _max)
const scan_options &options,
bool validate_partition_hash)
: pegasus_scanner_impl(client, std::move(hash), options, _min, _max, validate_partition_hash)
{
_options.start_inclusive = true;
_options.stop_inclusive = false;
Expand All @@ -39,15 +40,17 @@ pegasus_client_impl::pegasus_scanner_impl::pegasus_scanner_impl(::dsn::apps::rrd
std::vector<uint64_t> &&hash,
const scan_options &options,
const ::dsn::blob &start_key,
const ::dsn::blob &stop_key)
const ::dsn::blob &stop_key,
bool validate_partition_hash)
: _client(client),
_start_key(start_key),
_stop_key(stop_key),
_options(options),
_splits_hash(std::move(hash)),
_p(-1),
_context(SCAN_CONTEXT_ID_COMPLETED),
_rpc_started(false)
_rpc_started(false),
_validate_partition_hash(validate_partition_hash)
{
}

Expand Down Expand Up @@ -211,6 +214,7 @@ void pegasus_client_impl::pegasus_scanner_impl::_start_scan()
req.sort_key_filter_pattern = ::dsn::blob(
_options.sort_key_filter_pattern.data(), 0, _options.sort_key_filter_pattern.size());
req.no_value = _options.no_value;
req.__set_validate_partition_hash(_validate_partition_hash);

dassert(!_rpc_started, "");
_rpc_started = true;
Expand Down
1 change: 1 addition & 0 deletions src/idl/rrdb.thrift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -252,6 +252,7 @@ struct get_scanner_request
8:dsn.blob hash_key_filter_pattern;
9:filter_type sort_key_filter_type;
10:dsn.blob sort_key_filter_pattern;
11:optional bool validate_partition_hash;
}

struct scan_request
Expand Down
34 changes: 13 additions & 21 deletions src/include/rrdb/rrdb_types.h
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

7 changes: 5 additions & 2 deletions src/server/pegasus_scan_context.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ struct pegasus_scan_context
::dsn::apps::filter_type::type sort_key_filter_type_,
const std::string &&sort_key_filter_pattern_,
int32_t batch_size_,
bool no_value_)
bool no_value_,
bool validate_partition_hash_)
: _stop_holder(std::move(stop_)),
_hash_key_filter_pattern_holder(std::move(hash_key_filter_pattern_)),
_sort_key_filter_pattern_holder(std::move(sort_key_filter_pattern_)),
Expand All @@ -55,7 +56,8 @@ struct pegasus_scan_context
sort_key_filter_pattern(
_sort_key_filter_pattern_holder.data(), 0, _sort_key_filter_pattern_holder.length()),
batch_size(batch_size_),
no_value(no_value_)
no_value(no_value_),
validate_partition_hash(validate_partition_hash_)
{
}

Expand All @@ -74,6 +76,7 @@ struct pegasus_scan_context
dsn::blob sort_key_filter_pattern;
int32_t batch_size;
bool no_value;
bool validate_partition_hash;
};

class pegasus_context_cache
Expand Down
Loading