From 9cd8875937070ac30cbbf9a1c072a9ed1ab30b1a Mon Sep 17 00:00:00 2001 From: Huachao Huang Date: Wed, 8 Aug 2018 15:05:29 +0800 Subject: [PATCH] titandb: add check commands to Makefile and add some comments (#27) --- Makefile | 23 ++++++++++++++++++++ utilities/titandb/blob_file_cache.h | 2 +- utilities/titandb/db_test.cc | 1 + utilities/titandb/options.h | 12 +++++++++++ utilities/titandb/version.h | 2 +- utilities/titandb/version_builder.h | 3 +++ utilities/titandb/version_set.cc | 33 ++++++++++++++++------------- utilities/titandb/version_set.h | 8 +++++++ 8 files changed, 67 insertions(+), 17 deletions(-) diff --git a/Makefile b/Makefile index 85fb232fb00..f02d1dfaac1 100644 --- a/Makefile +++ b/Makefile @@ -1517,6 +1517,13 @@ range_del_aggregator_test: db/range_del_aggregator_test.o db/db_test_util.o $(LI blob_db_test: utilities/blob_db/blob_db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) +TITANDB_TESTS = \ + titandb_blob_format_test \ + titandb_blob_file_test \ + titandb_table_builder_test \ + titandb_version_test \ + titandb_db_test + titandb_blob_format_test: utilities/titandb/blob_format_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) @@ -1532,6 +1539,22 @@ titandb_version_test: utilities/titandb/version_test.o $(LIBOBJECTS) $(TESTHARNE titandb_db_test: utilities/titandb/db_test.o $(LIBOBJECTS) $(TESTHARNESS) $(AM_LINK) +titandb_check: $(TITANDB_TESTS) + for t in $(TITANDB_TESTS); \ + do \ + echo "======== Running $$t ========"; \ + ./$$t || exit 1; \ + done; + +titandb_valgrind_check: $(TITANDB_TESTS) + for t in $(TITANDB_TESTS); do \ + $(VALGRIND_VER) $(VALGRIND_OPTS) ./$$t; \ + code=$$?; \ + if [ $$code -ne 0 ]; then \ + exit $$code; \ + fi; \ + done; + #------------------------------------------------- # make install related stuff INSTALL_PATH ?= /usr/local diff --git a/utilities/titandb/blob_file_cache.h b/utilities/titandb/blob_file_cache.h index c6008086f2e..f6ee63152ee 100644 --- a/utilities/titandb/blob_file_cache.h +++ b/utilities/titandb/blob_file_cache.h @@ -14,7 +14,7 @@ class BlobFileCache { BlobFileCache(const DBOptions& db_options, const TitanDBOptions& tdb_options); - // Gets the blob record pointed by the handle in the speficied file + // Gets the blob record pointed by the handle in the specified file // number. The corresponding file size must be exactly "file_size" // bytes. The provided buffer is used to store the record data, so // the buffer must be valid when the record is used. diff --git a/utilities/titandb/db_test.cc b/utilities/titandb/db_test.cc index f2aa063bbf5..da48f694157 100644 --- a/utilities/titandb/db_test.cc +++ b/utilities/titandb/db_test.cc @@ -23,6 +23,7 @@ class TitanDBTest : public testing::Test { void Reopen() { ASSERT_OK(db_->Close()); + delete db_; ASSERT_OK(TitanDB::Open(dbname_, options_, tdb_options_, &db_)); } diff --git a/utilities/titandb/options.h b/utilities/titandb/options.h index 725a4eec1e9..9602558b861 100644 --- a/utilities/titandb/options.h +++ b/utilities/titandb/options.h @@ -6,6 +6,9 @@ namespace rocksdb { namespace titandb { struct TitanDBOptions { + // The directory to store data specific to TitanDB alongside with + // the base DB. + // // Default: {dbname}/titandb std::string dirname; @@ -15,10 +18,19 @@ struct TitanDBOptions { // Default: 4096 uint64_t min_blob_size {4096}; + // The maximum open blob files in the blob file cache. + // + // Default: 1 << 20 uint64_t max_open_files {1 << 20}; + // The compression algorithm used to compress blob records in blob files. + // + // Default: kNoCompression CompressionType blob_file_compression {kNoCompression}; + // The desirable blob file size. This is not a hard limit but a wish. + // + // Default: 256MB uint64_t blob_file_target_size {256 << 20}; std::string ToString() const; diff --git a/utilities/titandb/version.h b/utilities/titandb/version.h index 323e7d6ca8e..e788717b1d9 100644 --- a/utilities/titandb/version.h +++ b/utilities/titandb/version.h @@ -2,7 +2,7 @@ #include "rocksdb/options.h" #include "utilities/titandb/blob_format.h" -#include "utilities/titandb/blob_file_cache.h" +#include "utilities/titandb/blob_file_reader.h" namespace rocksdb { namespace titandb { diff --git a/utilities/titandb/version_builder.h b/utilities/titandb/version_builder.h index 5eba16b1b04..82786c130be 100644 --- a/utilities/titandb/version_builder.h +++ b/utilities/titandb/version_builder.h @@ -17,7 +17,10 @@ class VersionBuilder { ~VersionBuilder(); + // Applies "*edit" on the current state. void Apply(VersionEdit* edit); + + // Saves the current state to the version "*v". void SaveTo(Version* v); private: diff --git a/utilities/titandb/version_set.cc b/utilities/titandb/version_set.cc index 30e085db222..5e06535515f 100644 --- a/utilities/titandb/version_set.cc +++ b/utilities/titandb/version_set.cc @@ -70,22 +70,25 @@ Status VersionSet::Recover() { bool has_next_file_number = false; uint64_t next_file_number = 0; - VersionBuilder builder(current_); - LogReporter reporter; - reporter.status = &s; - log::Reader reader(nullptr, std::move(file), &reporter, - true /*checksum*/, 0 /*initial_offset*/, 0); - Slice record; - std::string scratch; - while (reader.ReadRecord(&record, &scratch) && s.ok()) { - VersionEdit edit; - s = DecodeInto(record, &edit); - if (!s.ok()) return s; - builder.Apply(&edit); - if (edit.has_next_file_number_) { - next_file_number = edit.next_file_number_; - has_next_file_number = true; + // Reads edits from the manifest and applies them one by one. + VersionBuilder builder(current_); + { + LogReporter reporter; + reporter.status = &s; + log::Reader reader(nullptr, std::move(file), &reporter, + true /*checksum*/, 0 /*initial_offset*/, 0); + Slice record; + std::string scratch; + while (reader.ReadRecord(&record, &scratch) && s.ok()) { + VersionEdit edit; + s = DecodeInto(record, &edit); + if (!s.ok()) return s; + builder.Apply(&edit); + if (edit.has_next_file_number_) { + next_file_number = edit.next_file_number_; + has_next_file_number = true; + } } } diff --git a/utilities/titandb/version_set.h b/utilities/titandb/version_set.h index 8202e53d70c..96108d5b8c6 100644 --- a/utilities/titandb/version_set.h +++ b/utilities/titandb/version_set.h @@ -19,12 +19,20 @@ class VersionSet { ~VersionSet(); + // Sets up the storage specified in "tdb_options.dirname". + // If the manifest doesn't exist, it will create one. + // If the manifest exists, it will recover from the lastest one. Status Open(); + // Applies *edit on the current version to form a new version that is + // both saved to the manifest and installed as the new current version. + // REQUIRES: *mutex is held Status LogAndApply(VersionEdit* edit, port::Mutex* mutex); + // Returns the current version. Version* current() { return current_; } + // Allocates a new file number. uint64_t NewFileNumber(); private: