Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Group rocksdb.sst.read.micros stat by IOActivity flush and compaction #11288

Closed
wants to merge 17 commits into from
Closed
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Explicitly check ReadOptions::io_activity in user-facing APIs
hx235 committed Apr 20, 2023
commit ab58576577f007cea996ca28cf565776672468bf
5 changes: 5 additions & 0 deletions db/db_impl/compacted_db_impl.cc
Original file line number Diff line number Diff line change
@@ -46,6 +46,11 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*,
const Slice& key, PinnableSlice* value,
std::string* timestamp) {
if (options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call Get with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}
assert(user_comparator_);
if (options.timestamp) {
const Status s = FailIfTsMismatchCf(
41 changes: 41 additions & 0 deletions db/db_impl/db_impl.cc
Original file line number Diff line number Diff line change
@@ -1943,6 +1943,12 @@ Status DBImpl::GetEntity(const ReadOptions& read_options,
"Cannot call GetEntity without a PinnableWideColumns object");
}

if (read_options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call GetEntity with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}

columns->Reset();

GetImplOptions get_impl_options;
@@ -1988,6 +1994,12 @@ Status DBImpl::GetImpl(const ReadOptions& read_options, const Slice& key,

assert(get_impl_options.column_family);

if (read_options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call Get with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}

if (read_options.timestamp) {
const Status s = FailIfTsMismatchCf(get_impl_options.column_family,
*(read_options.timestamp),
@@ -2938,6 +2950,11 @@ Status DBImpl::MultiGetImpl(
autovector<KeyContext*, MultiGetContext::MAX_BATCH_SIZE>* sorted_keys,
SuperVersion* super_version, SequenceNumber snapshot,
ReadCallback* callback) {
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call MultiGet with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}
PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock);
StopWatch sw(immutable_db_options_.clock, stats_, DB_MULTIGET);

@@ -3336,6 +3353,8 @@ bool DBImpl::KeyMayExist(const ReadOptions& read_options,
std::string* value, std::string* timestamp,
bool* value_found) {
assert(value != nullptr);
assert(read_options.io_activity == Env::IOActivity::kUnknown);

if (value_found != nullptr) {
// falsify later if key-may-exist but can't fetch value
*value_found = true;
@@ -3368,6 +3387,11 @@ Iterator* DBImpl::NewIterator(const ReadOptions& read_options,
return NewErrorIterator(Status::NotSupported(
"ReadTier::kPersistedData is not yet supported in iterators."));
}
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return NewErrorIterator(Status::InvalidArgument(
"Cannot call NewIterator with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`"));
}

assert(column_family);

@@ -3503,6 +3527,11 @@ Status DBImpl::NewIterators(
return Status::NotSupported(
"ReadTier::kPersistedData is not yet supported in iterators.");
}
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call NewIterators with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}

if (read_options.timestamp) {
for (auto* cf : column_families) {
@@ -5672,6 +5701,12 @@ Status DBImpl::VerifyChecksumInternal(const ReadOptions& read_options,

Status s;

if (read_options.io_activity != Env::IOActivity::kUnknown) {
s = Status::InvalidArgument(
"Cannot verify file checksum with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
return s;
}
if (use_file_checksum) {
FileChecksumGenFactory* const file_checksum_gen_factory =
immutable_db_options_.file_checksum_gen_factory.get();
@@ -5785,6 +5820,12 @@ Status DBImpl::VerifyFullFileChecksum(const std::string& file_checksum_expected,
const std::string& func_name_expected,
const std::string& fname,
const ReadOptions& read_options) {
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call VerifyChecksum with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}

Status s;
if (file_checksum_expected == kUnknownFileChecksum) {
return s;
10 changes: 10 additions & 0 deletions db/db_impl/db_impl_readonly.cc
Original file line number Diff line number Diff line change
@@ -40,6 +40,11 @@ Status DBImplReadOnly::Get(const ReadOptions& read_options,
ColumnFamilyHandle* column_family, const Slice& key,
PinnableSlice* pinnable_val,
std::string* timestamp) {
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call Get with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}
assert(pinnable_val != nullptr);
// TODO: stopwatch DB_GET needed?, perf timer needed?
PERF_TIMER_GUARD(get_snapshot_time);
@@ -112,6 +117,11 @@ Status DBImplReadOnly::Get(const ReadOptions& read_options,

Iterator* DBImplReadOnly::NewIterator(const ReadOptions& read_options,
ColumnFamilyHandle* column_family) {
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return NewErrorIterator(Status::InvalidArgument(
"Cannot call NewIterator with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`"));
}
assert(column_family);
if (read_options.timestamp) {
const Status s = FailIfTsMismatchCf(
15 changes: 15 additions & 0 deletions db/db_impl/db_impl_secondary.cc
Original file line number Diff line number Diff line change
@@ -345,6 +345,11 @@ Status DBImplSecondary::GetImpl(const ReadOptions& read_options,
ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* pinnable_val,
std::string* timestamp) {
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call Get with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}
assert(pinnable_val != nullptr);
PERF_CPU_TIMER_GUARD(get_cpu_nanos, immutable_db_options_.clock);
StopWatch sw(immutable_db_options_.clock, stats_, DB_GET);
@@ -445,6 +450,11 @@ Iterator* DBImplSecondary::NewIterator(const ReadOptions& read_options,
return NewErrorIterator(Status::NotSupported(
"ReadTier::kPersistedData is not yet supported in iterators."));
}
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return NewErrorIterator(Status::InvalidArgument(
"Cannot call NewIterator with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`"));
}

assert(column_family);
if (read_options.timestamp) {
@@ -511,6 +521,11 @@ Status DBImplSecondary::NewIterators(
return Status::NotSupported(
"ReadTier::kPersistedData is not yet supported in iterators.");
}
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call NewIterators with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}
ReadCallback* read_callback = nullptr; // No read callback provided.
if (iterators == nullptr) {
return Status::InvalidArgument("iterators not allowed to be nullptr");
1 change: 0 additions & 1 deletion include/rocksdb/utilities/transaction.h
Original file line number Diff line number Diff line change
@@ -681,4 +681,3 @@ class Transaction {
};

} // namespace ROCKSDB_NAMESPACE

3 changes: 2 additions & 1 deletion table/sst_file_reader.cc
Original file line number Diff line number Diff line change
@@ -66,6 +66,7 @@ Status SstFileReader::Open(const std::string& file_path) {
}

Iterator* SstFileReader::NewIterator(const ReadOptions& roptions) {
assert(roptions.io_activity == Env::IOActivity::kUnknown);
auto r = rep_.get();
auto sequence = roptions.snapshot != nullptr
? roptions.snapshot->GetSequenceNumber()
@@ -91,9 +92,9 @@ std::shared_ptr<const TableProperties> SstFileReader::GetTableProperties()
}

Status SstFileReader::VerifyChecksum(const ReadOptions& read_options) {
assert(read_options.io_activity == Env::IOActivity::kUnknown);
return rep_->table_reader->VerifyChecksum(read_options,
TableReaderCaller::kSSTFileReader);
}

} // namespace ROCKSDB_NAMESPACE

10 changes: 10 additions & 0 deletions utilities/blob_db/blob_db_impl.cc
Original file line number Diff line number Diff line change
@@ -1631,6 +1631,11 @@ Status BlobDBImpl::GetImpl(const ReadOptions& read_options,
return Status::NotSupported(
"Blob DB doesn't support non-default column family.");
}
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call Get with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}
// Get a snapshot to avoid blob file get deleted between we
// fetch and index entry and reading from the file.
// TODO(yiwu): For Get() retry if file not found would be a simpler strategy.
@@ -2036,6 +2041,11 @@ void BlobDBImpl::CopyBlobFiles(
}

Iterator* BlobDBImpl::NewIterator(const ReadOptions& read_options) {
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return NewErrorIterator(Status::InvalidArgument(
"Cannot call NewIterator with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`"));
}
auto* cfd =
static_cast_with_check<ColumnFamilyHandleImpl>(DefaultColumnFamily())
->cfd();
6 changes: 5 additions & 1 deletion utilities/transactions/pessimistic_transaction.cc
Original file line number Diff line number Diff line change
@@ -166,6 +166,11 @@ template <typename TValue>
inline Status WriteCommittedTxn::GetForUpdateImpl(
const ReadOptions& read_options, ColumnFamilyHandle* column_family,
const Slice& key, TValue* value, bool exclusive, const bool do_validate) {
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call GetForUpdate with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}
column_family =
column_family ? column_family : db_impl_->DefaultColumnFamily();
assert(column_family);
@@ -1170,4 +1175,3 @@ Status PessimisticTransaction::SetName(const TransactionName& name) {
}

} // namespace ROCKSDB_NAMESPACE

30 changes: 29 additions & 1 deletion utilities/transactions/transaction_base.cc
Original file line number Diff line number Diff line change
@@ -235,6 +235,11 @@ Status TransactionBaseImpl::PopSavePoint() {
Status TransactionBaseImpl::Get(const ReadOptions& read_options,
ColumnFamilyHandle* column_family,
const Slice& key, std::string* value) {
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call Get with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}
assert(value != nullptr);
PinnableSlice pinnable_val(value);
assert(!pinnable_val.IsPinned());
@@ -262,6 +267,11 @@ Status TransactionBaseImpl::GetForUpdate(const ReadOptions& read_options,
"If do_validate is false then GetForUpdate with snapshot is not "
"defined.");
}
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call GetForUpdate with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}
Status s =
TryLock(column_family, key, true /* read_only */, exclusive, do_validate);

@@ -288,6 +298,11 @@ Status TransactionBaseImpl::GetForUpdate(const ReadOptions& read_options,
"If do_validate is false then GetForUpdate with snapshot is not "
"defined.");
}
if (read_options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call GetForUpdate with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}
Status s =
TryLock(column_family, key, true /* read_only */, exclusive, do_validate);

@@ -302,6 +317,13 @@ std::vector<Status> TransactionBaseImpl::MultiGet(
const std::vector<ColumnFamilyHandle*>& column_family,
const std::vector<Slice>& keys, std::vector<std::string>* values) {
size_t num_keys = keys.size();
if (read_options.io_activity != Env::IOActivity::kUnknown) {
Status s = Status::InvalidArgument(
"Cannot call MultiGet with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
return std::vector<Status>(num_keys, s);
}

values->resize(num_keys);

std::vector<Status> stat_list(num_keys);
@@ -317,6 +339,7 @@ void TransactionBaseImpl::MultiGet(const ReadOptions& read_options,
const size_t num_keys, const Slice* keys,
PinnableSlice* values, Status* statuses,
const bool sorted_input) {
assert(read_options.io_activity == Env::IOActivity::kUnknown);
write_batch_.MultiGetFromBatchAndDB(db_, read_options, column_family,
num_keys, keys, values, statuses,
sorted_input);
@@ -328,6 +351,12 @@ std::vector<Status> TransactionBaseImpl::MultiGetForUpdate(
const std::vector<Slice>& keys, std::vector<std::string>* values) {
// Regardless of whether the MultiGet succeeded, track these keys.
size_t num_keys = keys.size();
if (read_options.io_activity != Env::IOActivity::kUnknown) {
Status s = Status::InvalidArgument(
"Cannot call MultiGetForUpdate with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
return std::vector<Status>(num_keys, s);
}
values->resize(num_keys);

// Lock all keys
@@ -726,4 +755,3 @@ WriteBatch* TransactionBaseImpl::GetCommitTimeWriteBatch() {
return &commit_time_batch_;
}
} // namespace ROCKSDB_NAMESPACE

7 changes: 6 additions & 1 deletion utilities/transactions/write_prepared_txn.cc
Original file line number Diff line number Diff line change
@@ -44,6 +44,7 @@ void WritePreparedTxn::MultiGet(const ReadOptions& options,
const size_t num_keys, const Slice* keys,
PinnableSlice* values, Status* statuses,
const bool sorted_input) {
assert(options.io_activity == Env::IOActivity::kUnknown);
SequenceNumber min_uncommitted, snap_seq;
const SnapshotBackup backed_by_snapshot =
wpt_db_->AssignMinMaxSeqs(options.snapshot, &min_uncommitted, &snap_seq);
@@ -64,6 +65,11 @@ void WritePreparedTxn::MultiGet(const ReadOptions& options,
Status WritePreparedTxn::Get(const ReadOptions& options,
ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* pinnable_val) {
if (options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call Get with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}
SequenceNumber min_uncommitted, snap_seq;
const SnapshotBackup backed_by_snapshot =
wpt_db_->AssignMinMaxSeqs(options.snapshot, &min_uncommitted, &snap_seq);
@@ -507,4 +513,3 @@ Status WritePreparedTxn::RebuildFromWriteBatch(WriteBatch* src_batch) {
}

} // namespace ROCKSDB_NAMESPACE

10 changes: 10 additions & 0 deletions utilities/transactions/write_prepared_txn_db.cc
Original file line number Diff line number Diff line change
@@ -250,6 +250,11 @@ Status WritePreparedTxnDB::WriteInternal(const WriteOptions& write_options_orig,
Status WritePreparedTxnDB::Get(const ReadOptions& options,
ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* value) {
if (options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call Get with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}
SequenceNumber min_uncommitted, snap_seq;
const SnapshotBackup backed_by_snapshot =
AssignMinMaxSeqs(options.snapshot, &min_uncommitted, &snap_seq);
@@ -343,6 +348,11 @@ static void CleanupWritePreparedTxnDBIterator(void* arg1, void* /*arg2*/) {

Iterator* WritePreparedTxnDB::NewIterator(const ReadOptions& options,
ColumnFamilyHandle* column_family) {
if (options.io_activity != Env::IOActivity::kUnknown) {
return NewErrorIterator(Status::InvalidArgument(
"Cannot call NewIterator with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`"));
}
constexpr bool expose_blob_index = false;
constexpr bool allow_refresh = false;
std::shared_ptr<ManagedSnapshot> own_snapshot = nullptr;
7 changes: 6 additions & 1 deletion utilities/transactions/write_unprepared_txn.cc
Original file line number Diff line number Diff line change
@@ -948,6 +948,7 @@ void WriteUnpreparedTxn::MultiGet(const ReadOptions& options,
const size_t num_keys, const Slice* keys,
PinnableSlice* values, Status* statuses,
const bool sorted_input) {
assert(options.io_activity == Env::IOActivity::kUnknown);
SequenceNumber min_uncommitted, snap_seq;
const SnapshotBackup backed_by_snapshot =
wupt_db_->AssignMinMaxSeqs(options.snapshot, &min_uncommitted, &snap_seq);
@@ -968,6 +969,11 @@ void WriteUnpreparedTxn::MultiGet(const ReadOptions& options,
Status WriteUnpreparedTxn::Get(const ReadOptions& options,
ColumnFamilyHandle* column_family,
const Slice& key, PinnableSlice* value) {
if (options.io_activity != Env::IOActivity::kUnknown) {
return Status::InvalidArgument(
"Cannot call Get with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`");
}
SequenceNumber min_uncommitted, snap_seq;
const SnapshotBackup backed_by_snapshot =
wupt_db_->AssignMinMaxSeqs(options.snapshot, &min_uncommitted, &snap_seq);
@@ -1048,4 +1054,3 @@ WriteUnpreparedTxn::GetUnpreparedSequenceNumbers() {
}

} // namespace ROCKSDB_NAMESPACE

5 changes: 5 additions & 0 deletions utilities/transactions/write_unprepared_txn_db.cc
Original file line number Diff line number Diff line change
@@ -388,6 +388,11 @@ static void CleanupWriteUnpreparedTxnDBIterator(void* arg1, void* /*arg2*/) {
Iterator* WriteUnpreparedTxnDB::NewIterator(const ReadOptions& options,
ColumnFamilyHandle* column_family,
WriteUnpreparedTxn* txn) {
if (options.io_activity != Env::IOActivity::kUnknown) {
return NewErrorIterator(Status::InvalidArgument(
"Cannot call NewIterator with `ReadOptions::io_activity` != "
"`Env::IOActivity::kUnknown`"));
}
// TODO(lth): Refactor so that this logic is shared with WritePrepared.
constexpr bool expose_blob_index = false;
constexpr bool allow_refresh = false;
Loading