-
Notifications
You must be signed in to change notification settings - Fork 6.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Track each SST's timestamp information as user properties #9093
Changes from 5 commits
873c7ad
5991279
57bfcd6
886c942
35f9154
a7dfe15
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -6,12 +6,14 @@ | |
// This file defines a collection of statistics collectors. | ||
#pragma once | ||
|
||
#include "rocksdb/table_properties.h" | ||
|
||
#include <memory> | ||
#include <string> | ||
#include <vector> | ||
|
||
#include "db/dbformat.h" | ||
#include "rocksdb/comparator.h" | ||
#include "rocksdb/table_properties.h" | ||
|
||
namespace ROCKSDB_NAMESPACE { | ||
|
||
// Base class for internal table properties collector. | ||
|
@@ -108,4 +110,65 @@ class UserKeyTablePropertiesCollectorFactory | |
std::shared_ptr<TablePropertiesCollectorFactory> user_collector_factory_; | ||
}; | ||
|
||
// When rocksdb creates a newtable, it will encode all "user keys" into | ||
// "internal keys". This class collects min/max timestamp from the encoded | ||
// internal key when Add() is invoked. | ||
// | ||
// @param cmp the Comparator to compare timestamp of key by the | ||
// Comparator::CompareTimestamp function. | ||
class TimestampTablePropertiesCollector : public IntTblPropCollector { | ||
sunlike-Lipo marked this conversation as resolved.
Show resolved
Hide resolved
|
||
public: | ||
explicit TimestampTablePropertiesCollector(const Comparator* cmp) | ||
: cmp_(cmp) {} | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Maybe here we can explicitly initialize There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed it. |
||
|
||
Status InternalAdd(const Slice& key, const Slice& /* value */, | ||
uint64_t /* file_size */) override { | ||
auto user_key = ExtractUserKey(key); | ||
assert(cmp_ && cmp_->timestamp_size() > 0); | ||
if (user_key.size() < cmp_->timestamp_size()) { | ||
return Status::Corruption( | ||
"User key size mismatch when comparing to timestamp size."); | ||
} | ||
auto timestamp_in_key = | ||
ExtractTimestampFromUserKey(user_key, cmp_->timestamp_size()); | ||
if (max_timestamp_ == kDisableUserTimestamp || | ||
cmp_->CompareTimestamp(timestamp_in_key, max_timestamp_) > 0) { | ||
max_timestamp_ = timestamp_in_key.ToString(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. If we do
will it save us some string creation and copy caused by calling There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed it. |
||
} | ||
if (min_timestamp_ == kDisableUserTimestamp || | ||
cmp_->CompareTimestamp(min_timestamp_, timestamp_in_key) > 0) { | ||
min_timestamp_ = timestamp_in_key.ToString(); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Ditto There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed it. |
||
} | ||
return Status::OK(); | ||
} | ||
|
||
void BlockAdd(uint64_t /* block_raw_bytes */, | ||
uint64_t /* block_compressed_bytes_fast */, | ||
uint64_t /* block_compressed_bytes_slow */) override { | ||
return; | ||
} | ||
|
||
Status Finish(UserCollectedProperties* properties) override { | ||
assert(min_timestamp_.size() == max_timestamp_.size() && | ||
max_timestamp_.size() == cmp_->timestamp_size()); | ||
properties->insert({"rocksdb.min_timestamp", min_timestamp_}); | ||
properties->insert({"rocksdb.max_timestamp", max_timestamp_}); | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I used
This makes me wonder whether we can instead use rocksdb.timestamp_min and rocksdb.timestamp_max so that they can be next to each other. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed it. |
||
return Status::OK(); | ||
} | ||
|
||
const char* Name() const override { | ||
return "TimestampTablePropertiesCollector"; | ||
} | ||
|
||
UserCollectedProperties GetReadableProperties() const override { | ||
return {{"rocksdb.min_timestamp", Slice(min_timestamp_).ToString(true)}, | ||
{"rocksdb.max_timestamp", Slice(max_timestamp_).ToString(true)}}; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Same as previous one. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed it. |
||
} | ||
|
||
protected: | ||
const Comparator* const cmp_; | ||
std::string max_timestamp_; | ||
std::string min_timestamp_; | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Should these two be explicitly initialized to There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fixed it. |
||
}; | ||
|
||
} // namespace ROCKSDB_NAMESPACE |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Not sure, but the indentation here seems weird.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
fixed