diff --git a/c-deps/libroach/CMakeLists.txt b/c-deps/libroach/CMakeLists.txt index 98ece39ccc67..e39fda611d41 100644 --- a/c-deps/libroach/CMakeLists.txt +++ b/c-deps/libroach/CMakeLists.txt @@ -29,8 +29,8 @@ add_library(roach db.cc encoding.cc engine.cc - env_switching.cc eventlistener.cc + file_registry.cc getter.cc godefs.cc ldb.cc @@ -49,6 +49,7 @@ add_library(roach protos/util/hlc/legacy_timestamp.pb.cc protos/util/hlc/timestamp.pb.cc protos/util/unresolved_addr.pb.cc + rocksdbutils/env_encryption.cc ) target_include_directories(roach PUBLIC ./include @@ -59,6 +60,7 @@ target_include_directories(roach add_library(roachccl ccl/crypto_utils.cc + ccl/ctr_stream.cc ccl/db.cc ccl/key_manager.cc protosccl/ccl/baseccl/encryption_options.pb.cc @@ -87,6 +89,7 @@ enable_testing() set(tests db_test.cc encoding_test.cc + file_registry_test.cc ccl/db_test.cc ccl/key_manager_test.cc ) @@ -132,7 +135,7 @@ foreach(tsrc ${tests}) ${CRYPTOPP_LIB} ) target_include_directories(${tname} - PRIVATE ../cryptopp + PRIVATE .. # CryptoPP headers are directly in the directory. Include .. to be able to include PRIVATE protosccl ) endif() diff --git a/c-deps/libroach/ccl/crypto_utils.cc b/c-deps/libroach/ccl/crypto_utils.cc index ae9a098d213c..e24fab03a436 100644 --- a/c-deps/libroach/ccl/crypto_utils.cc +++ b/c-deps/libroach/ccl/crypto_utils.cc @@ -27,3 +27,17 @@ std::string RandomBytes(size_t length) { CryptoPP::OS_GenerateRandomBlock(false /* blocking */, data, length); return std::string(reinterpret_cast(data.data()), data.size()); } + +AESCipher::~AESCipher() {} + +size_t AESCipher::BlockSize() { return CryptoPP::AES::BLOCKSIZE; } + +rocksdb::Status AESCipher::Encrypt(char* data) { + enc_.ProcessBlock((byte*)data); + return rocksdb::Status::OK(); +} + +rocksdb::Status AESCipher::Decrypt(char* data) { + enc_.ProcessBlock((byte*)data); + return rocksdb::Status::OK(); +} diff --git a/c-deps/libroach/ccl/crypto_utils.h b/c-deps/libroach/ccl/crypto_utils.h index bed00b71b072..e4941842fe11 100644 --- a/c-deps/libroach/ccl/crypto_utils.h +++ b/c-deps/libroach/ccl/crypto_utils.h @@ -8,7 +8,9 @@ #pragma once +#include #include +#include "../rocksdbutils/env_encryption.h" /* * These provide various crypto primitives. They currently use CryptoPP. @@ -23,3 +25,27 @@ std::string HexString(const std::string& s); // TODO(mberhault): it would be good to have a blocking version (/dev/random on *nix), // but to do it properly we might want to pre-read in the background. std::string RandomBytes(size_t length); + +// AES block cipher using CryptoPP. +class AESCipher : public rocksdb_utils::BlockCipher { + public: + // The key must have a valid length (16/24/32 bytes) or CryptoPP will fail. + AESCipher(std::string key) + : enc_((byte*)key.data(), key.size()), dec_((byte*)key.data(), key.size()) {} + virtual ~AESCipher(); + + // Blocksize is fixed for AES. + virtual size_t BlockSize() override; + + // Encrypt a block of data. + // Length of data is equal to BlockSize(). + virtual rocksdb::Status Encrypt(char* data) override; + + // Decrypt a block of data. + // Length of data is equal to BlockSize(). + virtual rocksdb::Status Decrypt(char* data) override; + + private: + CryptoPP::AES::Encryption enc_; + CryptoPP::AES::Decryption dec_; +}; diff --git a/c-deps/libroach/ccl/ctr_stream.cc b/c-deps/libroach/ccl/ctr_stream.cc new file mode 100644 index 000000000000..6c2143b8d183 --- /dev/null +++ b/c-deps/libroach/ccl/ctr_stream.cc @@ -0,0 +1,148 @@ +// Copyright 2018 The Cockroach Authors. +// +// Licensed as a CockroachDB Enterprise file under the Cockroach Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt +// (found in the LICENSE.Apache file in the root directory). + +#include "ctr_stream.h" +#include +#include "../fmt.h" +#include "../plaintext_stream.h" +#include "crypto_utils.h" + +using namespace cockroach; + +namespace cockroach { + +rocksdb::Status BuildCipherStream(const enginepbccl::EncryptionSettings& settings, + const enginepbccl::SecretKey* key, + std::unique_ptr* result) { + // We should not be getting called for plaintext, and we only have AES. + if (settings.encryption_type() != enginepbccl::AES128_CTR && + settings.encryption_type() != enginepbccl::AES192_CTR && + settings.encryption_type() != enginepbccl::AES256_CTR) { + return rocksdb::Status::InvalidArgument( + fmt::StringPrintf("unknown encryption type %d", settings.encryption_type())); + } + + (*result) = std::unique_ptr( + new CTRCipherStream(new AESCipher(key->key()), settings.nonce(), settings.counter())); + + return rocksdb::Status::OK(); +} + +CTRCipherStreamCreator::~CTRCipherStreamCreator() {} + +rocksdb::Status CTRCipherStreamCreator::InitSettingsAndCreateCipherStream( + std::string* settings, std::unique_ptr* result) { + auto key = key_manager_->CurrentKey(); + if (key == nullptr || key->info().encryption_type() == enginepbccl::Plaintext) { + // Plaintext: don't set "settings". + (*result) = std::unique_ptr(new PlaintextStream()); + return rocksdb::Status::OK(); + } + + // Create the settings. + enginepbccl::EncryptionSettings enc_settings; + enc_settings.set_encryption_type(key->info().encryption_type()); + enc_settings.set_key_id(key->info().key_id()); + + // Let's get 16 random bytes. 12 for the nonce, 4 for the counter. + std::string random_bytes = RandomBytes(16); + assert(random_bytes.size() == 16); + + // First 12 bytes for the nonce. + enc_settings.set_nonce(random_bytes.substr(0, 12)); + // Last 4 as an unsigned int32 for the counter. + uint32_t counter; + memcpy(&counter, random_bytes.data() + 12, 4); + enc_settings.set_counter(counter); + + // Serialize enc_settings directly into the passed settings pointer. This will be ignored + // on error. + if (!enc_settings.SerializeToString(settings)) { + return rocksdb::Status::InvalidArgument("failed to serialize encryption settings"); + } + + return BuildCipherStream(enc_settings, key.get(), result); +} + +// Create a cipher stream given encryption settings. +rocksdb::Status CTRCipherStreamCreator::CreateCipherStreamFromSettings( + const std::string& settings, std::unique_ptr* result) { + enginepbccl::EncryptionSettings enc_settings; + if (!enc_settings.ParseFromString(settings)) { + return rocksdb::Status::InvalidArgument("failed to parse encryption settings"); + } + + if (settings.size() == 0 || enc_settings.encryption_type() == enginepbccl::Plaintext) { + // Plaintext. + (*result) = std::unique_ptr(new PlaintextStream()); + return rocksdb::Status::OK(); + } + + // Get the key from the manager. + auto key = key_manager_->GetKey(enc_settings.key_id()); + if (key == nullptr) { + return rocksdb::Status::InvalidArgument(fmt::StringPrintf( + "key_manager does not have a key with ID %s", enc_settings.key_id().c_str())); + } + + return BuildCipherStream(enc_settings, key.get(), result); +} + +enginepb::EnvType CTRCipherStreamCreator::GetEnvType() { return env_type_; } + +CTRCipherStream::CTRCipherStream(rocksdb_utils::BlockCipher* c, const std::string& nonce, + uint32_t counter) + : cipher_(c), nonce_(nonce), counter_(counter) { + assert(iv_.size() == (cipher_->BlockSize() - 4)); +} + +CTRCipherStream::~CTRCipherStream() {} + +size_t CTRCipherStream::BlockSize() { return cipher_->BlockSize(); } + +void CTRCipherStream::AllocateScratch(std::string& scratch) { + auto blockSize = cipher_->BlockSize(); + scratch.reserve(blockSize); +} + +rocksdb::Status CTRCipherStream::EncryptBlock(uint64_t blockIndex, char* data, char* scratch) { + // Create IV = nonce + counter + auto blockSize = cipher_->BlockSize(); + auto nonce_size = blockSize - 4; + // Write the nonce at the beginning of the scratch space. + memcpy(scratch, nonce_.data(), nonce_size); + + // Counter value for this block, converted to network byte order. + uint32_t block_counter = htonl(counter_ + blockIndex); + // Write after the nonce. + memcpy(scratch + nonce_size, &block_counter, 4); + + // Encrypt nonce+counter + auto status = cipher_->Encrypt(scratch); + if (!status.ok()) { + return status; + } + + // XOR data with ciphertext. + // TODO(mberhault): this is not an efficient XOR. Instead, we could move + // this into the cipher and use something like CryptoPP::ProcessAndXorBlock. + for (size_t i = 0; i < blockSize; i++) { + data[i] = data[i] ^ scratch[i]; + } + return rocksdb::Status::OK(); +} + +// Decrypt a block of data at the given block index. +// Length of data is equal to BlockSize(); +rocksdb::Status CTRCipherStream::DecryptBlock(uint64_t blockIndex, char* data, char* scratch) { + // For CTR decryption & encryption are the same + return EncryptBlock(blockIndex, data, scratch); +} + +} // namespace cockroach diff --git a/c-deps/libroach/ccl/ctr_stream.h b/c-deps/libroach/ccl/ctr_stream.h new file mode 100644 index 000000000000..4207ef850edd --- /dev/null +++ b/c-deps/libroach/ccl/ctr_stream.h @@ -0,0 +1,71 @@ +// Copyright 2018 The Cockroach Authors. +// +// Licensed as a CockroachDB Enterprise file under the Cockroach Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt + +#pragma once + +#include +#include "../rocksdbutils/env_encryption.h" +#include "key_manager.h" + +namespace cockroach { + +// CTRCipherStreamCreator creates a CTRCipherStream using a KeyManager. +// Takes ownership of the KeyManager. +class CTRCipherStreamCreator final : public rocksdb_utils::CipherStreamCreator { + public: + CTRCipherStreamCreator(KeyManager* key_mgr, enginepb::EnvType env_type) + : key_manager_(key_mgr), env_type_(env_type) {} + virtual ~CTRCipherStreamCreator(); + + virtual rocksdb::Status InitSettingsAndCreateCipherStream( + std::string* settings, + std::unique_ptr* result) override; + + // Create a cipher stream given encryption settings. + virtual rocksdb::Status CreateCipherStreamFromSettings( + const std::string& settings, + std::unique_ptr* result) override; + + virtual enginepb::EnvType GetEnvType() override; + + private: + std::unique_ptr key_manager_; + enginepb::EnvType env_type_; +}; + +class CTRCipherStream final : public rocksdb_utils::BlockAccessCipherStream { + public: + // Create a CTR cipher stream given: + // - a block cipher (takes ownership) + // - nonce of size 'cipher.BlockSize - sizeof(counter)' (eg: 16-4 = 12 bytes for AES) + // - counter + CTRCipherStream(rocksdb_utils::BlockCipher* c, const std::string& nonce, uint32_t counter); + virtual ~CTRCipherStream(); + + // BlockSize returns the size of each block supported by this cipher stream. + virtual size_t BlockSize() override; + + protected: + // Allocate scratch space which is passed to EncryptBlock/DecryptBlock. + virtual void AllocateScratch(std::string&) override; + + // Encrypt a block of data at the given block index. + // Length of data is equal to BlockSize(); + virtual rocksdb::Status EncryptBlock(uint64_t blockIndex, char* data, char* scratch) override; + + // Decrypt a block of data at the given block index. + // Length of data is equal to BlockSize(); + virtual rocksdb::Status DecryptBlock(uint64_t blockIndex, char* data, char* scratch) override; + + private: + std::unique_ptr cipher_; + std::string nonce_; + uint32_t counter_; +}; + +} // namespace cockroach diff --git a/c-deps/libroach/ccl/db.cc b/c-deps/libroach/ccl/db.cc index e6b0801cbf28..23f295e28d28 100644 --- a/c-deps/libroach/ccl/db.cc +++ b/c-deps/libroach/ccl/db.cc @@ -7,7 +7,6 @@ // https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt #include "../db.h" -#include "../encoding.h" #include #include #include @@ -17,8 +16,12 @@ #include #include "../batch.h" #include "../comparator.h" +#include "../encoding.h" +#include "../env_manager.h" +#include "../rocksdbutils/env_encryption.h" #include "../status.h" #include "ccl/baseccl/encryption_options.pb.h" +#include "ctr_stream.h" #include "key_manager.h" using namespace cockroach; @@ -27,12 +30,21 @@ namespace cockroach { // DBOpenHook parses the extra_options field of DBOptions and initializes encryption objects if // needed. -rocksdb::Status DBOpenHook(const std::string& db_dir, const DBOptions db_opts) { +rocksdb::Status DBOpenHook(const std::string& db_dir, const DBOptions db_opts, + EnvManager* env_mgr) { DBSlice options = db_opts.extra_options; if (options.len == 0) { return rocksdb::Status::OK(); } + // The Go code sets the "switching_env" storage version if we specified encryption flags, + // but let's double check anyway. + if (!db_opts.use_switching_env) { + return rocksdb::Status::InvalidArgument( + "on-disk version does not support encryption, but we found encryption flags"); + } + + // Parse extra_options. cockroach::ccl::baseccl::EncryptionOptions opts; if (!opts.ParseFromArray(options.data, options.len)) { return rocksdb::Status::InvalidArgument("failed to parse extra options"); @@ -42,36 +54,57 @@ rocksdb::Status DBOpenHook(const std::string& db_dir, const DBOptions db_opts) { return rocksdb::Status::InvalidArgument("unknown encryption key source"); } - std::cout << "found encryption options:\n" - << " active key: " << opts.key_files().current_key() << "\n" - << " old key: " << opts.key_files().old_key() << "\n" - << " rotation duration: " << opts.data_key_rotation_period() << "\n"; - // Initialize store key manager. - std::shared_ptr store_key_manager(new FileKeyManager( - rocksdb::Env::Default(), opts.key_files().current_key(), opts.key_files().old_key())); + // NOTE: FileKeyManager uses the default env as the MemEnv can never have pre-populated files. + FileKeyManager* store_key_manager = new FileKeyManager( + rocksdb::Env::Default(), opts.key_files().current_key(), opts.key_files().old_key()); rocksdb::Status status = store_key_manager->LoadKeys(); if (!status.ok()) { + delete store_key_manager; return status; } - // Initialize data key manager. - std::shared_ptr data_key_manager( - new DataKeyManager(rocksdb::Env::Default(), db_dir, opts.data_key_rotation_period())); + // Create a cipher stream creator using the store_key_manager. + auto store_stream = new CTRCipherStreamCreator(store_key_manager, enginepb::Store); + + // Construct an EncryptedEnv using this stream creator and the base_env (Default or Mem). + // It takes ownership of the stream. + rocksdb::Env* store_keyed_env = + rocksdb_utils::NewEncryptedEnv(env_mgr->base_env, env_mgr->file_registry.get(), store_stream); + // Transfer ownership to the env manager. + env_mgr->TakeEnvOwnership(store_keyed_env); + + // Initialize data key manager using the stored-keyed-env. + DataKeyManager* data_key_manager = + new DataKeyManager(store_keyed_env, db_dir, opts.data_key_rotation_period()); status = data_key_manager->LoadKeys(); if (!status.ok()) { + delete data_key_manager; return status; } - // Generate a new data key if needed by giving the active store key info to the data key manager. + // Create a cipher stream creator using the data_key_manager. + auto data_stream = new CTRCipherStreamCreator(data_key_manager, enginepb::Data); + + // Construct an EncryptedEnv using this stream creator and the base_env (Default or Mem). + // It takes ownership of the stream. + rocksdb::Env* data_keyed_env = + rocksdb_utils::NewEncryptedEnv(env_mgr->base_env, env_mgr->file_registry.get(), data_stream); + // Transfer ownership to the env manager and make it as the db_env. + env_mgr->TakeEnvOwnership(data_keyed_env); + env_mgr->db_env = data_keyed_env; + + // Fetch the current store key info. std::unique_ptr store_key = store_key_manager->CurrentKeyInfo(); assert(store_key != nullptr); + + // Generate a new data key if needed by giving the active store key info to the data key manager. status = data_key_manager->SetActiveStoreKey(std::move(store_key)); if (!status.ok()) { return status; } - return rocksdb::Status::InvalidArgument("encryption is not supported"); + return rocksdb::Status::OK(); } } // namespace cockroach diff --git a/c-deps/libroach/ccl/db_test.cc b/c-deps/libroach/ccl/db_test.cc index e0d71ffde0a2..b1904eedd03b 100644 --- a/c-deps/libroach/ccl/db_test.cc +++ b/c-deps/libroach/ccl/db_test.cc @@ -13,12 +13,20 @@ using namespace cockroach; TEST(LibroachCCL, DBOpenHook) { DBOptions db_opts; + db_opts.use_switching_env = false; // Try an empty extra_options. db_opts.extra_options = ToDBSlice(""); - EXPECT_OK(DBOpenHook("", db_opts)); + EXPECT_OK(DBOpenHook("", db_opts, nullptr)); - // Try extra_options with bad data. + // Try without switching env enabled and bogus options. We should fail + // because encryption options without switching env is not allowed. db_opts.extra_options = ToDBSlice("blah"); - EXPECT_ERR(DBOpenHook("", db_opts), "failed to parse extra options"); + EXPECT_ERR(DBOpenHook("", db_opts, nullptr), + "on-disk version does not support encryption, but we found encryption flags"); + + db_opts.use_switching_env = true; + // Try with switching env but bogus encryption flags. + db_opts.extra_options = ToDBSlice("blah"); + EXPECT_ERR(DBOpenHook("", db_opts, nullptr), "failed to parse extra options"); } diff --git a/c-deps/libroach/ccl/key_manager.cc b/c-deps/libroach/ccl/key_manager.cc index 2369bf56a8d0..a31e4d2fd3f8 100644 --- a/c-deps/libroach/ccl/key_manager.cc +++ b/c-deps/libroach/ccl/key_manager.cc @@ -216,6 +216,7 @@ rocksdb::Status DataKeyManager::LoadKeysHelper(enginepbccl::DataKeysRegistry* re return rocksdb::Status::InvalidArgument( fmt::StringPrintf("failed to parse key registry %s", registry_path_.c_str())); } + return rocksdb::Status::OK(); } diff --git a/c-deps/libroach/db.cc b/c-deps/libroach/db.cc index 31a54c6fb639..4fa679247cb5 100644 --- a/c-deps/libroach/db.cc +++ b/c-deps/libroach/db.cc @@ -25,7 +25,8 @@ #include "defines.h" #include "encoding.h" #include "engine.h" -#include "env_switching.h" +#include "env_manager.h" +#include "eventlistener.h" #include "fmt.h" #include "getter.h" #include "godefs.h" @@ -40,7 +41,8 @@ using namespace cockroach; namespace cockroach { // DBOpenHook in OSS mode only verifies that no extra options are specified. -__attribute__((weak)) rocksdb::Status DBOpenHook(const std::string& db_dir, const DBOptions opts) { +__attribute__((weak)) rocksdb::Status DBOpenHook(const std::string& db_dir, const DBOptions opts, + EnvManager* env_ctx) { if (opts.extra_options.len != 0) { return rocksdb::Status::InvalidArgument( "DBOptions has extra_options, but OSS code cannot handle them"); @@ -118,30 +120,60 @@ DBStatus DBOpen(DBEngine** db, DBSlice dir, DBOptions db_opts) { const std::string db_dir = ToString(dir); + // Make the default options.env the default. It points to Env::Default which does not + // need to be deleted. + std::unique_ptr env_ctx(new cockroach::EnvManager(options.env)); + + if (dir.len == 0) { + // In-memory database: use a MemEnv as the base Env. + auto memenv = rocksdb::NewMemEnv(rocksdb::Env::Default()); + // Register it for deletion. + env_ctx->TakeEnvOwnership(memenv); + // Make it the env that all other Envs must wrap. + env_ctx->base_env = memenv; + // Make it the env for rocksdb. + env_ctx->db_env = memenv; + } + + // Create the file registry. It uses the base_env to access the registry file. + auto file_registry = std::unique_ptr(new FileRegistry(env_ctx->base_env, db_dir)); + + if (db_opts.use_switching_env) { + // We're using the switching env. + auto status = file_registry->Load(); + if (!status.ok()) { + return ToDBStatus(status); + } + + // EnvManager takes ownership of the file registry. + env_ctx->file_registry.swap(file_registry); + } else { + // Switching env format not enabled: check whether we have a registry file (we shouldn't). + // The file_registry is not passed to anyone, it is deleted when it goes out of scope. + auto status = file_registry->CheckNoRegistryFile(); + if (!status.ok()) { + return ToDBStatus(status); + } + } + // Call hooks to handle db_opts.extra_options. - auto hook_status = DBOpenHook(db_dir, db_opts); + auto hook_status = DBOpenHook(db_dir, db_opts, env_ctx.get()); if (!hook_status.ok()) { return ToDBStatus(hook_status); } + // TODO(mberhault): + // - check available ciphers somehow? + // We may have a encrypted files in the registry file but running without encryption flags. + // - pass read-only flag though, we should not be modifying file/key registries (including key + // rotation) in read-only mode. + // Register listener for tracking RocksDB stats. std::shared_ptr event_listener(new DBEventListener); options.listeners.emplace_back(event_listener); - // TODO(mberhault): we shouldn't need two separate env objects, - // options.env should be sufficient with SwitchingEnv owning any - // underlying Env. - std::unique_ptr memenv; - if (dir.len == 0) { - memenv.reset(rocksdb::NewMemEnv(rocksdb::Env::Default())); - options.env = memenv.get(); - } - - std::unique_ptr switching_env; - if (db_opts.use_switching_env) { - switching_env.reset(NewSwitchingEnv(options.env, options.info_log)); - options.env = switching_env.get(); - } + // Point rocksdb to the env to use. + options.env = env_ctx->db_env; rocksdb::DB* db_ptr; @@ -155,9 +187,8 @@ DBStatus DBOpen(DBEngine** db, DBSlice dir, DBOptions db_opts) { if (!status.ok()) { return ToDBStatus(status); } - *db = - new DBImpl(db_ptr, memenv.release(), db_opts.cache != nullptr ? db_opts.cache->rep : nullptr, - event_listener, switching_env.release()); + *db = new DBImpl(db_ptr, std::move(env_ctx), + db_opts.cache != nullptr ? db_opts.cache->rep : nullptr, event_listener); return kSuccess; } diff --git a/c-deps/libroach/db.h b/c-deps/libroach/db.h index 33d48043e43e..6a9d177276de 100644 --- a/c-deps/libroach/db.h +++ b/c-deps/libroach/db.h @@ -23,8 +23,10 @@ namespace cockroach { +struct EnvManager; + // DBOpenHook is called at the beginning of DBOpen. It can be implemented in CCL code. -rocksdb::Status DBOpenHook(const std::string& db_dir, const DBOptions opts); +rocksdb::Status DBOpenHook(const std::string& db_dir, const DBOptions opts, EnvManager* env_ctx); // ToDBSlice returns a DBSlice from a rocksdb::Slice inline DBSlice ToDBSlice(const rocksdb::Slice& s) { diff --git a/c-deps/libroach/db_test.cc b/c-deps/libroach/db_test.cc index f1fc34b53b3a..01a69de29f30 100644 --- a/c-deps/libroach/db_test.cc +++ b/c-deps/libroach/db_test.cc @@ -23,10 +23,10 @@ TEST(Libroach, DBOpenHook) { // Try an empty extra_options. db_opts.extra_options = ToDBSlice(""); - EXPECT_OK(DBOpenHook("", db_opts)); + EXPECT_OK(DBOpenHook("", db_opts, nullptr)); // Try extra_options with anything at all. db_opts.extra_options = ToDBSlice("blah"); - EXPECT_ERR(DBOpenHook("", db_opts), + EXPECT_ERR(DBOpenHook("", db_opts, nullptr), "DBOptions has extra_options, but OSS code cannot handle them"); } diff --git a/c-deps/libroach/engine.cc b/c-deps/libroach/engine.cc index cf8fe3d66d8f..d07ca2698858 100644 --- a/c-deps/libroach/engine.cc +++ b/c-deps/libroach/engine.cc @@ -15,6 +15,7 @@ #include "engine.h" #include "db.h" #include "encoding.h" +#include "env_manager.h" #include "fmt.h" #include "getter.h" #include "iterator.h" @@ -96,11 +97,10 @@ DBString DBEngine::GetUserProperties() { namespace cockroach { -DBImpl::DBImpl(rocksdb::DB* r, rocksdb::Env* m, std::shared_ptr bc, - std::shared_ptr event_listener, rocksdb::Env* s_env) +DBImpl::DBImpl(rocksdb::DB* r, std::unique_ptr e, std::shared_ptr bc, + std::shared_ptr event_listener) : DBEngine(r, &iters_count), - switching_env(s_env), - memenv(m), + env_mgr(std::move(e)), rep_deleter(r), block_cache(bc), event_listener(event_listener), diff --git a/c-deps/libroach/engine.h b/c-deps/libroach/engine.h index ce6066930830..90371193814a 100644 --- a/c-deps/libroach/engine.h +++ b/c-deps/libroach/engine.h @@ -49,9 +49,10 @@ struct DBEngine { namespace cockroach { +struct EnvManager; + struct DBImpl : public DBEngine { - std::unique_ptr switching_env; - std::unique_ptr memenv; + std::unique_ptr env_mgr; std::unique_ptr rep_deleter; std::shared_ptr block_cache; std::shared_ptr event_listener; @@ -60,8 +61,8 @@ struct DBImpl : public DBEngine { // Construct a new DBImpl from the specified DB. // The DB and passed Envs will be deleted when the DBImpl is deleted. // Either env can be NULL. - DBImpl(rocksdb::DB* r, rocksdb::Env* m, std::shared_ptr bc, - std::shared_ptr event_listener, rocksdb::Env* s_env); + DBImpl(rocksdb::DB* r, std::unique_ptr e, std::shared_ptr bc, + std::shared_ptr event_listener); virtual ~DBImpl(); virtual DBStatus AssertPreClose(); diff --git a/c-deps/libroach/env_manager.h b/c-deps/libroach/env_manager.h new file mode 100644 index 000000000000..dbb75de2d1c2 --- /dev/null +++ b/c-deps/libroach/env_manager.h @@ -0,0 +1,41 @@ +// Copyright 2018 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +#pragma once + +#include +#include "file_registry.h" +#include "rocksdbutils/env_encryption.h" + +namespace cockroach { + +// EnvManager manages all created Envs, as well as the file registry. +// Rocksdb owns Env::Default (global static). All other envs are owned by EnvManager. +// +// Some useful envs are kept: +// base_env: the env that all other envs must wrap. Usually a Env::Default or MemEnv. +// db_env: the env used by rocksdb. This can be an EncryptedEnv. +struct EnvManager { + EnvManager(rocksdb::Env* env) : base_env(env), db_env(env) {} + ~EnvManager() {} + + void TakeEnvOwnership(rocksdb::Env* env) { envs.push_back(std::unique_ptr(env)); } + + rocksdb::Env* base_env; + rocksdb::Env* db_env; + std::unique_ptr file_registry; + std::vector> envs; +}; + +} // namespace cockroach diff --git a/c-deps/libroach/env_switching.cc b/c-deps/libroach/env_switching.cc deleted file mode 100644 index cfbfba14d8df..000000000000 --- a/c-deps/libroach/env_switching.cc +++ /dev/null @@ -1,51 +0,0 @@ -// Copyright 2017 The Cockroach Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the License. - -#include "env_switching.h" - -/* - * - * SwitchingEnv switches between a base_env (usually a Env::Default or MemEnv) - * and an encrypted env. - * - * ----------------------------- - * | SwitchingEnv (EnvWrapper) | - * --- PLAIN - ENCRYPTED ------- - * | \ - * | \ - * | ------------------------------ - * | | encrypted_env (EnvWrapper) | - * | ------------------------------ - * | / - * | / - * ------------------ - * | base_env (Env) | - * ------------------ - * - * Any unimplemented methods are called on the base_env. - */ -class SwitchingEnv : public rocksdb::EnvWrapper { - public: - SwitchingEnv(rocksdb::Env* base_env, std::shared_ptr logger) - : rocksdb::EnvWrapper(base_env), logger(logger) { - rocksdb::Info(logger, "initialized switching env"); - } - - private: - std::shared_ptr logger; -}; - -rocksdb::Env* NewSwitchingEnv(rocksdb::Env* base_env, std::shared_ptr logger) { - return new SwitchingEnv(base_env ? base_env : rocksdb::Env::Default(), logger); -} diff --git a/c-deps/libroach/env_switching.h b/c-deps/libroach/env_switching.h deleted file mode 100644 index f62b80ef1172..000000000000 --- a/c-deps/libroach/env_switching.h +++ /dev/null @@ -1,20 +0,0 @@ -// Copyright 2017 The Cockroach Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. See the License for the specific language governing -// permissions and limitations under the License. - -#pragma once - -#include - -// Returns a new SwitchingEnv using the passed-in env as the base. -rocksdb::Env* NewSwitchingEnv(rocksdb::Env* base_env, std::shared_ptr logger); diff --git a/c-deps/libroach/file_registry.cc b/c-deps/libroach/file_registry.cc new file mode 100644 index 000000000000..1449df02c446 --- /dev/null +++ b/c-deps/libroach/file_registry.cc @@ -0,0 +1,248 @@ +// Copyright 2018 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +#include "file_registry.h" +#include "../fmt.h" +#include "../utils.h" + +using namespace cockroach; + +namespace cockroach { + +FileRegistry::FileRegistry(rocksdb::Env* env, const std::string& db_dir) + : env_(env), db_dir_(db_dir), registry_path_(db_dir_ + "/" + kFileRegistryFilename) {} + +rocksdb::Status FileRegistry::CheckNoRegistryFile() { + rocksdb::Status status = env_->FileExists(registry_path_); + if (status.ok()) { + return rocksdb::Status::InvalidArgument( + fmt::StringPrintf("registry file %s exists", registry_path_.c_str())); + } else if (!status.IsNotFound()) { + return status; + } + + return rocksdb::Status::OK(); +} + +rocksdb::Status FileRegistry::Load() { + rocksdb::Status status = env_->FileExists(registry_path_); + if (status.IsNotFound()) { + registry_.reset(new enginepb::FileRegistry()); + return rocksdb::Status::OK(); + } else if (!status.ok()) { + return status; + } + + std::string contents; + status = rocksdb::ReadFileToString(env_, registry_path_, &contents); + if (!status.ok()) { + return status; + } + + std::unique_ptr registry(new enginepb::FileRegistry()); + if (!registry->ParseFromString(contents)) { + return rocksdb::Status::InvalidArgument( + fmt::StringPrintf("failed to parse file registry %s", registry_path_.c_str())); + } + + // Make it the active registry. + registry_.swap(registry); + + return rocksdb::Status::OK(); +} + +std::unordered_set FileRegistry::GetUsedEnvTypes() { + std::unordered_set ret; + + std::unique_lock l(mu_); + assert(registry_ != nullptr); + + for (auto it : registry_->files()) { + ret.insert(it.second.env_type()); + } + + return ret; +} + +std::string FileRegistry::TransformPath(const std::string& filename) { + // Check for the rare case when we're referring to the db directory itself (without slash). + if (filename == db_dir_) { + return ""; + } + + // The db_dir usually does not include a trailing slash. We add it here. + auto prefixLength = db_dir_.size(); + + std::string db_dir(db_dir_); + if (prefixLength == 0 || db_dir[prefixLength - 1] != '/') { + db_dir.append("/"); + prefixLength++; + } + + auto nameLength = filename.size(); + if (nameLength < prefixLength) { + // Shorter than db_dir. + return filename; + } + + if (filename.compare(0, prefixLength, db_dir) != 0) { + // Does not start with db_dir. + return filename; + } + + // Matched rocksdbdir prefix. Check if there's a double slash (it happens): + if ((nameLength > prefixLength) && (filename[prefixLength] == '/')) { + prefixLength++; + } + + if (nameLength == prefixLength) { + // Exact match. + return ""; + } + + return filename.substr(prefixLength); +} + +std::unique_ptr FileRegistry::GetFileEntry(const std::string& filename) { + std::string newName = TransformPath(filename); + + std::unique_lock l(mu_); + assert(registry_ != nullptr); + + auto key = registry_->files().find(newName); + if (key == registry_->files().cend()) { + return nullptr; + } + return std::unique_ptr(new enginepb::FileEntry(key->second)); +} + +rocksdb::Status FileRegistry::SetFileEntry(const std::string& filename, + std::unique_ptr entry) { + std::string newName = TransformPath(filename); + + std::unique_lock l(mu_); + assert(registry_ != nullptr); + + // Make a copy of the registry and insert entry. + auto new_registry = + std::unique_ptr(new enginepb::FileRegistry(*registry_)); + (*new_registry->mutable_files())[newName] = *entry; + + return PersistRegistryLocked(std::move(new_registry)); +} + +rocksdb::Status FileRegistry::MaybeDeleteEntry(const std::string& filename) { + std::string newName = TransformPath(filename); + + std::unique_lock l(mu_); + assert(registry_ != nullptr); + + // It's cheaper to check first rather than copy and check existence after deletion. + auto key = registry_->files().find(newName); + if (key == registry_->files().cend()) { + // Entry does not exist: do nothing. + return rocksdb::Status::OK(); + } + + // Make a copy of the registry and delete entry. + auto new_registry = + std::unique_ptr(new enginepb::FileRegistry(*registry_)); + new_registry->mutable_files()->erase(newName); + + return PersistRegistryLocked(std::move(new_registry)); +} + +// Move the entry under 'src' to 'target' if 'src' exists, and persist the registry. +rocksdb::Status FileRegistry::MaybeRenameEntry(const std::string& src, const std::string& target) { + std::string newSrc = TransformPath(src); + std::string newTarget = TransformPath(target); + + std::unique_lock l(mu_); + assert(registry_ != nullptr); + + // It's cheaper to check first rather than copy-then-check. + auto src_key = registry_->files().find(newSrc); + auto target_key = registry_->files().find(newTarget); + if ((src_key == registry_->files().cend()) && (target_key == registry_->files().cend())) { + // Entries do not exist: do nothing. + return rocksdb::Status::OK(); + } + + // Make a copy of the registry. + auto new_registry = + std::unique_ptr(new enginepb::FileRegistry(*registry_)); + + if (src_key != registry_->files().cend()) { + // Source key exists: copy to target and delete source. + auto new_entry = std::unique_ptr(new enginepb::FileEntry(src_key->second)); + (*new_registry->mutable_files())[newTarget] = *new_entry; + new_registry->mutable_files()->erase(newSrc); + } else { + // Source doesn't exist, this means target does: delete it. + new_registry->mutable_files()->erase(newTarget); + } + + return PersistRegistryLocked(std::move(new_registry)); +} + +// Copy the entry under 'src' to 'target' if 'src' exists, and persist the registry. +rocksdb::Status FileRegistry::MaybeLinkEntry(const std::string& src, const std::string& target) { + std::string newSrc = TransformPath(src); + std::string newTarget = TransformPath(target); + + std::unique_lock l(mu_); + assert(registry_ != nullptr); + + // It's cheaper to check first rather than copy and check existence after deletion. + auto src_key = registry_->files().find(newSrc); + auto target_key = registry_->files().find(newTarget); + if ((src_key == registry_->files().cend()) && (target_key == registry_->files().cend())) { + // Entries do not exist: do nothing. + } + + // Make a copy of the registry. + auto new_registry = + std::unique_ptr(new enginepb::FileRegistry(*registry_)); + + if (src_key != registry_->files().cend()) { + // Source key exists: copy to target. + auto new_entry = std::unique_ptr(new enginepb::FileEntry(src_key->second)); + (*new_registry->mutable_files())[newTarget] = *new_entry; + } else { + // Source doesn't exist, this means target does: delete it. + new_registry->mutable_files()->erase(newTarget); + } + + return PersistRegistryLocked(std::move(new_registry)); +} + +rocksdb::Status FileRegistry::PersistRegistryLocked(std::unique_ptr reg) { + // Serialize and write to file. + std::string contents; + if (!reg->SerializeToString(&contents)) { + return rocksdb::Status::InvalidArgument("failed to serialize key registry"); + } + + auto status = SafeWriteStringToFile(env_, registry_path_, contents); + if (!status.ok()) { + return status; + } + + // Swap registry. + registry_.swap(reg); + + return rocksdb::Status::OK(); +} + +} // namespace cockroach diff --git a/c-deps/libroach/file_registry.h b/c-deps/libroach/file_registry.h new file mode 100644 index 000000000000..8133bdb5a440 --- /dev/null +++ b/c-deps/libroach/file_registry.h @@ -0,0 +1,92 @@ +// Copyright 2018 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +#pragma once + +#include +#include +#include +#include "protos/storage/engine/enginepb/file_registry.pb.h" + +namespace enginepb = cockroach::storage::engine::enginepb; + +namespace cockroach { + +// Name of the registry file. +static const std::string kFileRegistryFilename = "COCKROACHDB_REGISTRY"; + +// FileRegistry keeps track of encryption information per file. +// Plaintext files are not registered for two reasons: +// - they may be missing if transitioning from a non-registry version +// - we don't want to add the extra file rewrite in non-encrypted setups +// +// All paths given to FileRegistry should be absolute paths. It converts +// paths within `db_dir` to relative paths internally. +// Paths outside `db_dir` remain absolute. +class FileRegistry { + public: + FileRegistry(rocksdb::Env* env, const std::string& db_dir); + ~FileRegistry() {} + + // Returns OK if the registry file does not exist. + // An error could mean the file exists, or I/O error. + rocksdb::Status CheckNoRegistryFile(); + + // Load the file registry. Errors on file reading/parsing issues. + // OK if the file does not exist or is empty. + rocksdb::Status Load(); + + // Returns a hash-set of all used env types. + std::unordered_set GetUsedEnvTypes(); + + // Returns the FileEntry for the specified file. + std::unique_ptr GetFileEntry(const std::string& filename); + + // Insert 'entry' under 'filename' and persist the registry. + rocksdb::Status SetFileEntry(const std::string& filename, + std::unique_ptr entry); + + // Delete the entry for 'filename' if it exists and persist the registry. + rocksdb::Status MaybeDeleteEntry(const std::string& filename); + + // Move the entry under 'src' to 'target' if 'src' exists, and persist the registry. + // If 'src' does not exist (probably plaintext) but 'target' does, 'target' is deleted. + rocksdb::Status MaybeRenameEntry(const std::string& src, const std::string& target); + + // Copy the entry under 'src' to 'target' if 'src' exists, and persist the registry. + // If 'src' does not exist (probably plaintext) but 'target' does, 'target' is deleted. + rocksdb::Status MaybeLinkEntry(const std::string& src, const std::string& target); + + // TransformPath takes an absolute path. If 'path' is inside 'db_dir_', return the relative + // path, else return 'path'. + // This assumes sanitized paths (no symlinks, no relative paths) and uses a naive + // prefix removal. + std::string TransformPath(const std::string& path); + + private: + rocksdb::Env* env_; + std::string db_dir_; + std::string registry_path_; + + // Write 'reg' to the registry file, and swap with registry_ if successful. mu_ is held. + rocksdb::Status PersistRegistryLocked(std::unique_ptr reg); + + // The registry is read-only and can only be swapped for another one, it cannot be mutated in + // place. mu_ must be held for any registry access. + // TODO(mberhault): use a shared_mutex for multiple read-only holders. + std::mutex mu_; + std::unique_ptr registry_; +}; + +} // namespace cockroach diff --git a/c-deps/libroach/file_registry_test.cc b/c-deps/libroach/file_registry_test.cc new file mode 100644 index 000000000000..399b1fd3ac92 --- /dev/null +++ b/c-deps/libroach/file_registry_test.cc @@ -0,0 +1,51 @@ +// Copyright 2018 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +#include +#include "../fmt.h" +#include "../testutils.h" +#include "file_registry.h" + +using namespace cockroach; + +TEST(FileRegistry, TransformPath) { + struct TestCase { + std::string db_dir; + std::string input; + std::string output; + }; + + // The db_dir as sanitized does not have a trailing slash. + std::vector test_cases = { + {"/", "/foo", "foo"}, + {"/rocksdir", "/rocksdirfoo", "/rocksdirfoo"}, + {"/rocksdir", "/rocksdir/foo", "foo"}, + // We get the occasional double-slash. + {"/rocksdir", "/rocksdir//foo", "foo"}, + {"/mydir", "/mydir", ""}, + {"/mydir", "/mydir/", ""}, + {"/mydir", "/mydir//", ""}, + {"/mnt/otherdevice/", "/mnt/otherdevice/myfile", "myfile"}, + {"/mnt/otherdevice/myfile", "/mnt/otherdevice/myfile", ""}, + }; + + int test_num = 0; + for (auto t : test_cases) { + SCOPED_TRACE(fmt::StringPrintf("Testing #%d", test_num++)); + + FileRegistry reg(nullptr, t.db_dir); + auto out = reg.TransformPath(t.input); + EXPECT_EQ(t.output, out); + } +} diff --git a/c-deps/libroach/plaintext_stream.h b/c-deps/libroach/plaintext_stream.h new file mode 100644 index 000000000000..3b2be9722548 --- /dev/null +++ b/c-deps/libroach/plaintext_stream.h @@ -0,0 +1,45 @@ +// Copyright 2018 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. + +#pragma once + +#include "rocksdbutils/env_encryption.h" + +namespace cockroach { + +// PlaintextStream is a no-op stream cipher used for plaintext content. +class PlaintextStream final : public rocksdb_utils::BlockAccessCipherStream { + public: + PlaintextStream() {} + virtual ~PlaintextStream() {} + + virtual size_t BlockSize() override { return 0; } + virtual rocksdb::Status Encrypt(uint64_t fileOffset, char* data, size_t dataSize) override { + return rocksdb::Status::OK(); + } + virtual rocksdb::Status Decrypt(uint64_t fileOffset, char* data, size_t dataSize) override { + return rocksdb::Status::OK(); + } + + protected: + virtual void AllocateScratch(std::string&) override{}; + virtual rocksdb::Status EncryptBlock(uint64_t blockIndex, char* data, char* scratch) override { + return rocksdb::Status::OK(); + } + virtual rocksdb::Status DecryptBlock(uint64_t blockIndex, char* data, char* scratch) override { + return rocksdb::Status::OK(); + } +}; + +} // namespace cockroach diff --git a/c-deps/libroach/protos/storage/engine/enginepb/file_registry.pb.cc b/c-deps/libroach/protos/storage/engine/enginepb/file_registry.pb.cc index 8d515302d5f7..455ea1f0dc7a 100644 --- a/c-deps/libroach/protos/storage/engine/enginepb/file_registry.pb.cc +++ b/c-deps/libroach/protos/storage/engine/enginepb/file_registry.pb.cc @@ -18,11 +18,16 @@ namespace cockroach { namespace storage { namespace engine { namespace enginepb { -class RegistryDefaultTypeInternal { +class FileRegistry_FilesEntryDefaultTypeInternal { public: - ::google::protobuf::internal::ExplicitlyConstructed + ::google::protobuf::internal::ExplicitlyConstructed _instance; -} _Registry_default_instance_; +} _FileRegistry_FilesEntry_default_instance_; +class FileRegistryDefaultTypeInternal { +public: + ::google::protobuf::internal::ExplicitlyConstructed + _instance; +} _FileRegistry_default_instance_; class FileEntryDefaultTypeInternal { public: ::google::protobuf::internal::ExplicitlyConstructed @@ -44,17 +49,22 @@ PROTOBUF_CONSTEXPR_VAR ::google::protobuf::internal::ParseTable const TableStruct::schema[] GOOGLE_ATTRIBUTE_SECTION_VARIABLE(protodesc_cold) = { { NULL, NULL, 0, -1, -1, -1, -1, NULL, false }, { NULL, NULL, 0, -1, -1, -1, -1, NULL, false }, + { NULL, NULL, 0, -1, -1, -1, -1, NULL, false }, }; void TableStruct::InitDefaultsImpl() { GOOGLE_PROTOBUF_VERIFY_VERSION; ::google::protobuf::internal::InitProtobufDefaults(); - _Registry_default_instance_._instance.DefaultConstruct(); + _FileRegistry_FilesEntry_default_instance_._instance.DefaultConstruct(); + ::google::protobuf::internal::OnShutdownDestroyMessage( + &_FileRegistry_FilesEntry_default_instance_);_FileRegistry_default_instance_._instance.DefaultConstruct(); ::google::protobuf::internal::OnShutdownDestroyMessage( - &_Registry_default_instance_);_FileEntry_default_instance_._instance.DefaultConstruct(); + &_FileRegistry_default_instance_);_FileEntry_default_instance_._instance.DefaultConstruct(); ::google::protobuf::internal::OnShutdownDestroyMessage( - &_FileEntry_default_instance_);} + &_FileEntry_default_instance_);_FileRegistry_FilesEntry_default_instance_._instance.get_mutable()->set_default_instance(_FileRegistry_FilesEntry_default_instance_._instance.get_mutable()); + _FileRegistry_FilesEntry_default_instance_._instance.get_mutable()->InitAsDefaultInstance(); +} void InitDefaults() { static GOOGLE_PROTOBUF_DECLARE_ONCE(once); @@ -82,65 +92,81 @@ bool RegistryVersion_IsValid(int value) { } } +bool EnvType_IsValid(int value) { + switch (value) { + case 0: + case 1: + case 2: + return true; + default: + return false; + } +} + + +// =================================================================== + +#if PROTOBUF_INLINE_NOT_IN_HEADERS +#endif // PROTOBUF_INLINE_NOT_IN_HEADERS // =================================================================== #if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int Registry::kVersionFieldNumber; -const int Registry::kFilesFieldNumber; +const int FileRegistry::kVersionFieldNumber; +const int FileRegistry::kFilesFieldNumber; #endif // !defined(_MSC_VER) || _MSC_VER >= 1900 -Registry::Registry() +FileRegistry::FileRegistry() : ::google::protobuf::MessageLite(), _internal_metadata_(NULL) { if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) { protobuf_storage_2fengine_2fenginepb_2ffile_5fregistry_2eproto::InitDefaults(); } SharedCtor(); - // @@protoc_insertion_point(constructor:cockroach.storage.engine.enginepb.Registry) + // @@protoc_insertion_point(constructor:cockroach.storage.engine.enginepb.FileRegistry) } -Registry::Registry(const Registry& from) +FileRegistry::FileRegistry(const FileRegistry& from) : ::google::protobuf::MessageLite(), _internal_metadata_(NULL), - files_(from.files_), _cached_size_(0) { _internal_metadata_.MergeFrom(from._internal_metadata_); + files_.MergeFrom(from.files_); version_ = from.version_; - // @@protoc_insertion_point(copy_constructor:cockroach.storage.engine.enginepb.Registry) + // @@protoc_insertion_point(copy_constructor:cockroach.storage.engine.enginepb.FileRegistry) } -void Registry::SharedCtor() { +void FileRegistry::SharedCtor() { version_ = 0; _cached_size_ = 0; } -Registry::~Registry() { - // @@protoc_insertion_point(destructor:cockroach.storage.engine.enginepb.Registry) +FileRegistry::~FileRegistry() { + // @@protoc_insertion_point(destructor:cockroach.storage.engine.enginepb.FileRegistry) SharedDtor(); } -void Registry::SharedDtor() { +void FileRegistry::SharedDtor() { } -void Registry::SetCachedSize(int size) const { +void FileRegistry::SetCachedSize(int size) const { GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); _cached_size_ = size; GOOGLE_SAFE_CONCURRENT_WRITES_END(); } -const Registry& Registry::default_instance() { +const FileRegistry& FileRegistry::default_instance() { protobuf_storage_2fengine_2fenginepb_2ffile_5fregistry_2eproto::InitDefaults(); return *internal_default_instance(); } -Registry* Registry::New(::google::protobuf::Arena* arena) const { - Registry* n = new Registry; +FileRegistry* FileRegistry::New(::google::protobuf::Arena* arena) const { + FileRegistry* n = new FileRegistry; if (arena != NULL) { arena->Own(n); } return n; } -void Registry::Clear() { -// @@protoc_insertion_point(message_clear_start:cockroach.storage.engine.enginepb.Registry) +void FileRegistry::Clear() { +// @@protoc_insertion_point(message_clear_start:cockroach.storage.engine.enginepb.FileRegistry) ::google::protobuf::uint32 cached_has_bits = 0; // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; @@ -150,7 +176,7 @@ void Registry::Clear() { _internal_metadata_.Clear(); } -bool Registry::MergePartialFromCodedStream( +bool FileRegistry::MergePartialFromCodedStream( ::google::protobuf::io::CodedInputStream* input) { #define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure ::google::protobuf::uint32 tag; @@ -160,7 +186,7 @@ bool Registry::MergePartialFromCodedStream( mutable_unknown_fields)); ::google::protobuf::io::CodedOutputStream unknown_fields_stream( &unknown_fields_string, false); - // @@protoc_insertion_point(parse_start:cockroach.storage.engine.enginepb.Registry) + // @@protoc_insertion_point(parse_start:cockroach.storage.engine.enginepb.FileRegistry) for (;;) { ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); tag = p.first; @@ -181,12 +207,23 @@ bool Registry::MergePartialFromCodedStream( break; } - // repeated .cockroach.storage.engine.enginepb.FileEntry files = 2; + // map files = 2; case 2: { if (static_cast< ::google::protobuf::uint8>(tag) == static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { + FileRegistry_FilesEntry::Parser< ::google::protobuf::internal::MapFieldLite< + FileRegistry_FilesEntry, + ::std::string, ::cockroach::storage::engine::enginepb::FileEntry, + ::google::protobuf::internal::WireFormatLite::TYPE_STRING, + ::google::protobuf::internal::WireFormatLite::TYPE_MESSAGE, + 0 >, + ::google::protobuf::Map< ::std::string, ::cockroach::storage::engine::enginepb::FileEntry > > parser(&files_); DO_(::google::protobuf::internal::WireFormatLite::ReadMessageNoVirtual( - input, add_files())); + input, &parser)); + DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + parser.key().data(), static_cast(parser.key().length()), + ::google::protobuf::internal::WireFormatLite::PARSE, + "cockroach.storage.engine.enginepb.FileRegistry.FilesEntry.key")); } else { goto handle_unusual; } @@ -205,17 +242,17 @@ bool Registry::MergePartialFromCodedStream( } } success: - // @@protoc_insertion_point(parse_success:cockroach.storage.engine.enginepb.Registry) + // @@protoc_insertion_point(parse_success:cockroach.storage.engine.enginepb.FileRegistry) return true; failure: - // @@protoc_insertion_point(parse_failure:cockroach.storage.engine.enginepb.Registry) + // @@protoc_insertion_point(parse_failure:cockroach.storage.engine.enginepb.FileRegistry) return false; #undef DO_ } -void Registry::SerializeWithCachedSizes( +void FileRegistry::SerializeWithCachedSizes( ::google::protobuf::io::CodedOutputStream* output) const { - // @@protoc_insertion_point(serialize_start:cockroach.storage.engine.enginepb.Registry) + // @@protoc_insertion_point(serialize_start:cockroach.storage.engine.enginepb.FileRegistry) ::google::protobuf::uint32 cached_has_bits = 0; (void) cached_has_bits; @@ -225,32 +262,77 @@ void Registry::SerializeWithCachedSizes( 1, this->version(), output); } - // repeated .cockroach.storage.engine.enginepb.FileEntry files = 2; - for (unsigned int i = 0, - n = static_cast(this->files_size()); i < n; i++) { - ::google::protobuf::internal::WireFormatLite::WriteMessage( - 2, this->files(static_cast(i)), output); + // map files = 2; + if (!this->files().empty()) { + typedef ::google::protobuf::Map< ::std::string, ::cockroach::storage::engine::enginepb::FileEntry >::const_pointer + ConstPtr; + typedef ConstPtr SortItem; + typedef ::google::protobuf::internal::CompareByDerefFirst Less; + struct Utf8Check { + static void Check(ConstPtr p) { + ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + p->first.data(), static_cast(p->first.length()), + ::google::protobuf::internal::WireFormatLite::SERIALIZE, + "cockroach.storage.engine.enginepb.FileRegistry.FilesEntry.key"); + } + }; + + if (output->IsSerializationDeterministic() && + this->files().size() > 1) { + ::google::protobuf::scoped_array items( + new SortItem[this->files().size()]); + typedef ::google::protobuf::Map< ::std::string, ::cockroach::storage::engine::enginepb::FileEntry >::size_type size_type; + size_type n = 0; + for (::google::protobuf::Map< ::std::string, ::cockroach::storage::engine::enginepb::FileEntry >::const_iterator + it = this->files().begin(); + it != this->files().end(); ++it, ++n) { + items[static_cast(n)] = SortItem(&*it); + } + ::std::sort(&items[0], &items[static_cast(n)], Less()); + ::google::protobuf::scoped_ptr entry; + for (size_type i = 0; i < n; i++) { + entry.reset(files_.NewEntryWrapper( + items[static_cast(i)]->first, items[static_cast(i)]->second)); + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 2, *entry, output); + Utf8Check::Check(items[static_cast(i)]); + } + } else { + ::google::protobuf::scoped_ptr entry; + for (::google::protobuf::Map< ::std::string, ::cockroach::storage::engine::enginepb::FileEntry >::const_iterator + it = this->files().begin(); + it != this->files().end(); ++it) { + entry.reset(files_.NewEntryWrapper( + it->first, it->second)); + ::google::protobuf::internal::WireFormatLite::WriteMessage( + 2, *entry, output); + Utf8Check::Check(&*it); + } + } } output->WriteRaw((::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()).data(), static_cast((::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()).size())); - // @@protoc_insertion_point(serialize_end:cockroach.storage.engine.enginepb.Registry) + // @@protoc_insertion_point(serialize_end:cockroach.storage.engine.enginepb.FileRegistry) } -size_t Registry::ByteSizeLong() const { -// @@protoc_insertion_point(message_byte_size_start:cockroach.storage.engine.enginepb.Registry) +size_t FileRegistry::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:cockroach.storage.engine.enginepb.FileRegistry) size_t total_size = 0; total_size += (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()).size(); - // repeated .cockroach.storage.engine.enginepb.FileEntry files = 2; + // map files = 2; + total_size += 1 * + ::google::protobuf::internal::FromIntSize(this->files_size()); { - unsigned int count = static_cast(this->files_size()); - total_size += 1UL * count; - for (unsigned int i = 0; i < count; i++) { - total_size += - ::google::protobuf::internal::WireFormatLite::MessageSizeNoVirtual( - this->files(static_cast(i))); + ::google::protobuf::scoped_ptr entry; + for (::google::protobuf::Map< ::std::string, ::cockroach::storage::engine::enginepb::FileEntry >::const_iterator + it = this->files().begin(); + it != this->files().end(); ++it) { + entry.reset(files_.NewEntryWrapper(it->first, it->second)); + total_size += ::google::protobuf::internal::WireFormatLite:: + MessageSizeNoVirtual(*entry); } } @@ -267,13 +349,13 @@ size_t Registry::ByteSizeLong() const { return total_size; } -void Registry::CheckTypeAndMergeFrom( +void FileRegistry::CheckTypeAndMergeFrom( const ::google::protobuf::MessageLite& from) { - MergeFrom(*::google::protobuf::down_cast(&from)); + MergeFrom(*::google::protobuf::down_cast(&from)); } -void Registry::MergeFrom(const Registry& from) { -// @@protoc_insertion_point(class_specific_merge_from_start:cockroach.storage.engine.enginepb.Registry) +void FileRegistry::MergeFrom(const FileRegistry& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:cockroach.storage.engine.enginepb.FileRegistry) GOOGLE_DCHECK_NE(&from, this); _internal_metadata_.MergeFrom(from._internal_metadata_); ::google::protobuf::uint32 cached_has_bits = 0; @@ -285,78 +367,66 @@ void Registry::MergeFrom(const Registry& from) { } } -void Registry::CopyFrom(const Registry& from) { -// @@protoc_insertion_point(class_specific_copy_from_start:cockroach.storage.engine.enginepb.Registry) +void FileRegistry::CopyFrom(const FileRegistry& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:cockroach.storage.engine.enginepb.FileRegistry) if (&from == this) return; Clear(); MergeFrom(from); } -bool Registry::IsInitialized() const { +bool FileRegistry::IsInitialized() const { return true; } -void Registry::Swap(Registry* other) { +void FileRegistry::Swap(FileRegistry* other) { if (other == this) return; InternalSwap(other); } -void Registry::InternalSwap(Registry* other) { +void FileRegistry::InternalSwap(FileRegistry* other) { using std::swap; - files_.InternalSwap(&other->files_); + files_.Swap(&other->files_); swap(version_, other->version_); _internal_metadata_.Swap(&other->_internal_metadata_); swap(_cached_size_, other->_cached_size_); } -::std::string Registry::GetTypeName() const { - return "cockroach.storage.engine.enginepb.Registry"; +::std::string FileRegistry::GetTypeName() const { + return "cockroach.storage.engine.enginepb.FileRegistry"; } #if PROTOBUF_INLINE_NOT_IN_HEADERS -// Registry +// FileRegistry // .cockroach.storage.engine.enginepb.RegistryVersion version = 1; -void Registry::clear_version() { +void FileRegistry::clear_version() { version_ = 0; } -::cockroach::storage::engine::enginepb::RegistryVersion Registry::version() const { - // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.Registry.version) +::cockroach::storage::engine::enginepb::RegistryVersion FileRegistry::version() const { + // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.FileRegistry.version) return static_cast< ::cockroach::storage::engine::enginepb::RegistryVersion >(version_); } -void Registry::set_version(::cockroach::storage::engine::enginepb::RegistryVersion value) { +void FileRegistry::set_version(::cockroach::storage::engine::enginepb::RegistryVersion value) { version_ = value; - // @@protoc_insertion_point(field_set:cockroach.storage.engine.enginepb.Registry.version) + // @@protoc_insertion_point(field_set:cockroach.storage.engine.enginepb.FileRegistry.version) } -// repeated .cockroach.storage.engine.enginepb.FileEntry files = 2; -int Registry::files_size() const { +// map files = 2; +int FileRegistry::files_size() const { return files_.size(); } -void Registry::clear_files() { +void FileRegistry::clear_files() { files_.Clear(); } -const ::cockroach::storage::engine::enginepb::FileEntry& Registry::files(int index) const { - // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.Registry.files) - return files_.Get(index); -} -::cockroach::storage::engine::enginepb::FileEntry* Registry::mutable_files(int index) { - // @@protoc_insertion_point(field_mutable:cockroach.storage.engine.enginepb.Registry.files) - return files_.Mutable(index); -} -::cockroach::storage::engine::enginepb::FileEntry* Registry::add_files() { - // @@protoc_insertion_point(field_add:cockroach.storage.engine.enginepb.Registry.files) - return files_.Add(); -} -::google::protobuf::RepeatedPtrField< ::cockroach::storage::engine::enginepb::FileEntry >* -Registry::mutable_files() { - // @@protoc_insertion_point(field_mutable_list:cockroach.storage.engine.enginepb.Registry.files) - return &files_; + const ::google::protobuf::Map< ::std::string, ::cockroach::storage::engine::enginepb::FileEntry >& +FileRegistry::files() const { + // @@protoc_insertion_point(field_map:cockroach.storage.engine.enginepb.FileRegistry.files) + return files_.GetMap(); } -const ::google::protobuf::RepeatedPtrField< ::cockroach::storage::engine::enginepb::FileEntry >& -Registry::files() const { - // @@protoc_insertion_point(field_list:cockroach.storage.engine.enginepb.Registry.files) - return files_; + ::google::protobuf::Map< ::std::string, ::cockroach::storage::engine::enginepb::FileEntry >* +FileRegistry::mutable_files() { + // @@protoc_insertion_point(field_mutable_map:cockroach.storage.engine.enginepb.FileRegistry.files) + return files_.MutableMap(); } #endif // PROTOBUF_INLINE_NOT_IN_HEADERS @@ -364,8 +434,8 @@ Registry::files() const { // =================================================================== #if !defined(_MSC_VER) || _MSC_VER >= 1900 -const int FileEntry::kFilenameFieldNumber; -const int FileEntry::kEnvLevelFieldNumber; +const int FileEntry::kEnvTypeFieldNumber; +const int FileEntry::kEncryptionSettingsFieldNumber; #endif // !defined(_MSC_VER) || _MSC_VER >= 1900 FileEntry::FileEntry() @@ -381,17 +451,17 @@ FileEntry::FileEntry(const FileEntry& from) _internal_metadata_(NULL), _cached_size_(0) { _internal_metadata_.MergeFrom(from._internal_metadata_); - filename_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - if (from.filename().size() > 0) { - filename_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.filename_); + encryption_settings_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + if (from.encryption_settings().size() > 0) { + encryption_settings_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.encryption_settings_); } - env_level_ = from.env_level_; + env_type_ = from.env_type_; // @@protoc_insertion_point(copy_constructor:cockroach.storage.engine.enginepb.FileEntry) } void FileEntry::SharedCtor() { - filename_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - env_level_ = 0; + encryption_settings_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + env_type_ = 0; _cached_size_ = 0; } @@ -401,7 +471,7 @@ FileEntry::~FileEntry() { } void FileEntry::SharedDtor() { - filename_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + encryption_settings_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } void FileEntry::SetCachedSize(int size) const { @@ -428,8 +498,8 @@ void FileEntry::Clear() { // Prevent compiler warnings about cached_has_bits being unused (void) cached_has_bits; - filename_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); - env_level_ = 0; + encryption_settings_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + env_type_ = 0; _internal_metadata_.Clear(); } @@ -449,30 +519,27 @@ bool FileEntry::MergePartialFromCodedStream( tag = p.first; if (!p.second) goto handle_unusual; switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { - // string Filename = 1; + // .cockroach.storage.engine.enginepb.EnvType env_type = 1; case 1: { if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(10u /* 10 & 0xFF */)) { - DO_(::google::protobuf::internal::WireFormatLite::ReadString( - input, this->mutable_filename())); - DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->filename().data(), static_cast(this->filename().length()), - ::google::protobuf::internal::WireFormatLite::PARSE, - "cockroach.storage.engine.enginepb.FileEntry.Filename")); + static_cast< ::google::protobuf::uint8>(8u /* 8 & 0xFF */)) { + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + set_env_type(static_cast< ::cockroach::storage::engine::enginepb::EnvType >(value)); } else { goto handle_unusual; } break; } - // int32 env_level = 2; + // bytes encryption_settings = 2; case 2: { if (static_cast< ::google::protobuf::uint8>(tag) == - static_cast< ::google::protobuf::uint8>(16u /* 16 & 0xFF */)) { - - DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< - ::google::protobuf::int32, ::google::protobuf::internal::WireFormatLite::TYPE_INT32>( - input, &env_level_))); + static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { + DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( + input, this->mutable_encryption_settings())); } else { goto handle_unusual; } @@ -505,19 +572,16 @@ void FileEntry::SerializeWithCachedSizes( ::google::protobuf::uint32 cached_has_bits = 0; (void) cached_has_bits; - // string Filename = 1; - if (this->filename().size() > 0) { - ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( - this->filename().data(), static_cast(this->filename().length()), - ::google::protobuf::internal::WireFormatLite::SERIALIZE, - "cockroach.storage.engine.enginepb.FileEntry.Filename"); - ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( - 1, this->filename(), output); + // .cockroach.storage.engine.enginepb.EnvType env_type = 1; + if (this->env_type() != 0) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 1, this->env_type(), output); } - // int32 env_level = 2; - if (this->env_level() != 0) { - ::google::protobuf::internal::WireFormatLite::WriteInt32(2, this->env_level(), output); + // bytes encryption_settings = 2; + if (this->encryption_settings().size() > 0) { + ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased( + 2, this->encryption_settings(), output); } output->WriteRaw((::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()).data(), @@ -531,18 +595,17 @@ size_t FileEntry::ByteSizeLong() const { total_size += (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()).size(); - // string Filename = 1; - if (this->filename().size() > 0) { + // bytes encryption_settings = 2; + if (this->encryption_settings().size() > 0) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::StringSize( - this->filename()); + ::google::protobuf::internal::WireFormatLite::BytesSize( + this->encryption_settings()); } - // int32 env_level = 2; - if (this->env_level() != 0) { + // .cockroach.storage.engine.enginepb.EnvType env_type = 1; + if (this->env_type() != 0) { total_size += 1 + - ::google::protobuf::internal::WireFormatLite::Int32Size( - this->env_level()); + ::google::protobuf::internal::WireFormatLite::EnumSize(this->env_type()); } int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); @@ -564,12 +627,12 @@ void FileEntry::MergeFrom(const FileEntry& from) { ::google::protobuf::uint32 cached_has_bits = 0; (void) cached_has_bits; - if (from.filename().size() > 0) { + if (from.encryption_settings().size() > 0) { - filename_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.filename_); + encryption_settings_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.encryption_settings_); } - if (from.env_level() != 0) { - set_env_level(from.env_level()); + if (from.env_type() != 0) { + set_env_type(from.env_type()); } } @@ -590,8 +653,8 @@ void FileEntry::Swap(FileEntry* other) { } void FileEntry::InternalSwap(FileEntry* other) { using std::swap; - filename_.Swap(&other->filename_); - swap(env_level_, other->env_level_); + encryption_settings_.Swap(&other->encryption_settings_); + swap(env_type_, other->env_type_); _internal_metadata_.Swap(&other->_internal_metadata_); swap(_cached_size_, other->_cached_size_); } @@ -603,71 +666,71 @@ ::std::string FileEntry::GetTypeName() const { #if PROTOBUF_INLINE_NOT_IN_HEADERS // FileEntry -// string Filename = 1; -void FileEntry::clear_filename() { - filename_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +// .cockroach.storage.engine.enginepb.EnvType env_type = 1; +void FileEntry::clear_env_type() { + env_type_ = 0; +} +::cockroach::storage::engine::enginepb::EnvType FileEntry::env_type() const { + // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.FileEntry.env_type) + return static_cast< ::cockroach::storage::engine::enginepb::EnvType >(env_type_); +} +void FileEntry::set_env_type(::cockroach::storage::engine::enginepb::EnvType value) { + + env_type_ = value; + // @@protoc_insertion_point(field_set:cockroach.storage.engine.enginepb.FileEntry.env_type) +} + +// bytes encryption_settings = 2; +void FileEntry::clear_encryption_settings() { + encryption_settings_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } -const ::std::string& FileEntry::filename() const { - // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.FileEntry.Filename) - return filename_.GetNoArena(); +const ::std::string& FileEntry::encryption_settings() const { + // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) + return encryption_settings_.GetNoArena(); } -void FileEntry::set_filename(const ::std::string& value) { +void FileEntry::set_encryption_settings(const ::std::string& value) { - filename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:cockroach.storage.engine.enginepb.FileEntry.Filename) + encryption_settings_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) } #if LANG_CXX11 -void FileEntry::set_filename(::std::string&& value) { +void FileEntry::set_encryption_settings(::std::string&& value) { - filename_.SetNoArena( + encryption_settings_.SetNoArena( &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:cockroach.storage.engine.enginepb.FileEntry.Filename) + // @@protoc_insertion_point(field_set_rvalue:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) } #endif -void FileEntry::set_filename(const char* value) { +void FileEntry::set_encryption_settings(const char* value) { GOOGLE_DCHECK(value != NULL); - filename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:cockroach.storage.engine.enginepb.FileEntry.Filename) + encryption_settings_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) } -void FileEntry::set_filename(const char* value, size_t size) { +void FileEntry::set_encryption_settings(const void* value, size_t size) { - filename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + encryption_settings_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:cockroach.storage.engine.enginepb.FileEntry.Filename) + // @@protoc_insertion_point(field_set_pointer:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) } -::std::string* FileEntry::mutable_filename() { +::std::string* FileEntry::mutable_encryption_settings() { - // @@protoc_insertion_point(field_mutable:cockroach.storage.engine.enginepb.FileEntry.Filename) - return filename_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + // @@protoc_insertion_point(field_mutable:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) + return encryption_settings_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } -::std::string* FileEntry::release_filename() { - // @@protoc_insertion_point(field_release:cockroach.storage.engine.enginepb.FileEntry.Filename) +::std::string* FileEntry::release_encryption_settings() { + // @@protoc_insertion_point(field_release:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) - return filename_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return encryption_settings_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } -void FileEntry::set_allocated_filename(::std::string* filename) { - if (filename != NULL) { +void FileEntry::set_allocated_encryption_settings(::std::string* encryption_settings) { + if (encryption_settings != NULL) { } else { } - filename_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), filename); - // @@protoc_insertion_point(field_set_allocated:cockroach.storage.engine.enginepb.FileEntry.Filename) -} - -// int32 env_level = 2; -void FileEntry::clear_env_level() { - env_level_ = 0; -} -::google::protobuf::int32 FileEntry::env_level() const { - // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.FileEntry.env_level) - return env_level_; -} -void FileEntry::set_env_level(::google::protobuf::int32 value) { - - env_level_ = value; - // @@protoc_insertion_point(field_set:cockroach.storage.engine.enginepb.FileEntry.env_level) + encryption_settings_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), encryption_settings); + // @@protoc_insertion_point(field_set_allocated:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) } #endif // PROTOBUF_INLINE_NOT_IN_HEADERS diff --git a/c-deps/libroach/protos/storage/engine/enginepb/file_registry.pb.h b/c-deps/libroach/protos/storage/engine/enginepb/file_registry.pb.h index 49bddc38c97c..2db57eb16c6e 100644 --- a/c-deps/libroach/protos/storage/engine/enginepb/file_registry.pb.h +++ b/c-deps/libroach/protos/storage/engine/enginepb/file_registry.pb.h @@ -28,6 +28,9 @@ #include #include // IWYU pragma: export #include // IWYU pragma: export +#include // IWYU pragma: export +#include +#include #include // @@protoc_insertion_point(includes) namespace cockroach { @@ -37,9 +40,12 @@ namespace enginepb { class FileEntry; class FileEntryDefaultTypeInternal; extern FileEntryDefaultTypeInternal _FileEntry_default_instance_; -class Registry; -class RegistryDefaultTypeInternal; -extern RegistryDefaultTypeInternal _Registry_default_instance_; +class FileRegistry; +class FileRegistryDefaultTypeInternal; +extern FileRegistryDefaultTypeInternal _FileRegistry_default_instance_; +class FileRegistry_FilesEntry; +class FileRegistry_FilesEntryDefaultTypeInternal; +extern FileRegistry_FilesEntryDefaultTypeInternal _FileRegistry_FilesEntry_default_instance_; } // namespace enginepb } // namespace engine } // namespace storage @@ -75,26 +81,41 @@ const RegistryVersion RegistryVersion_MIN = Base; const RegistryVersion RegistryVersion_MAX = Base; const int RegistryVersion_ARRAYSIZE = RegistryVersion_MAX + 1; +enum EnvType { + Plaintext = 0, + Store = 1, + Data = 2, + EnvType_INT_MIN_SENTINEL_DO_NOT_USE_ = ::google::protobuf::kint32min, + EnvType_INT_MAX_SENTINEL_DO_NOT_USE_ = ::google::protobuf::kint32max +}; +bool EnvType_IsValid(int value); +const EnvType EnvType_MIN = Plaintext; +const EnvType EnvType_MAX = Data; +const int EnvType_ARRAYSIZE = EnvType_MAX + 1; + // =================================================================== -class Registry : public ::google::protobuf::MessageLite /* @@protoc_insertion_point(class_definition:cockroach.storage.engine.enginepb.Registry) */ { + +// ------------------------------------------------------------------- + +class FileRegistry : public ::google::protobuf::MessageLite /* @@protoc_insertion_point(class_definition:cockroach.storage.engine.enginepb.FileRegistry) */ { public: - Registry(); - virtual ~Registry(); + FileRegistry(); + virtual ~FileRegistry(); - Registry(const Registry& from); + FileRegistry(const FileRegistry& from); - inline Registry& operator=(const Registry& from) { + inline FileRegistry& operator=(const FileRegistry& from) { CopyFrom(from); return *this; } #if LANG_CXX11 - Registry(Registry&& from) noexcept - : Registry() { + FileRegistry(FileRegistry&& from) noexcept + : FileRegistry() { *this = ::std::move(from); } - inline Registry& operator=(Registry&& from) noexcept { + inline FileRegistry& operator=(FileRegistry&& from) noexcept { if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { if (this != &from) InternalSwap(&from); } else { @@ -103,29 +124,29 @@ class Registry : public ::google::protobuf::MessageLite /* @@protoc_insertion_po return *this; } #endif - static const Registry& default_instance(); + static const FileRegistry& default_instance(); - static inline const Registry* internal_default_instance() { - return reinterpret_cast( - &_Registry_default_instance_); + static inline const FileRegistry* internal_default_instance() { + return reinterpret_cast( + &_FileRegistry_default_instance_); } static PROTOBUF_CONSTEXPR int const kIndexInFileMessages = - 0; + 1; - void Swap(Registry* other); - friend void swap(Registry& a, Registry& b) { + void Swap(FileRegistry* other); + friend void swap(FileRegistry& a, FileRegistry& b) { a.Swap(&b); } // implements Message ---------------------------------------------- - inline Registry* New() const PROTOBUF_FINAL { return New(NULL); } + inline FileRegistry* New() const PROTOBUF_FINAL { return New(NULL); } - Registry* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL; + FileRegistry* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL; void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from) PROTOBUF_FINAL; - void CopyFrom(const Registry& from); - void MergeFrom(const Registry& from); + void CopyFrom(const FileRegistry& from); + void MergeFrom(const FileRegistry& from); void Clear() PROTOBUF_FINAL; bool IsInitialized() const PROTOBUF_FINAL; @@ -140,7 +161,7 @@ class Registry : public ::google::protobuf::MessageLite /* @@protoc_insertion_po void SharedCtor(); void SharedDtor(); void SetCachedSize(int size) const; - void InternalSwap(Registry* other); + void InternalSwap(FileRegistry* other); private: inline ::google::protobuf::Arena* GetArenaNoVirtual() const { return NULL; @@ -154,19 +175,17 @@ class Registry : public ::google::protobuf::MessageLite /* @@protoc_insertion_po // nested types ---------------------------------------------------- + // accessors ------------------------------------------------------- - // repeated .cockroach.storage.engine.enginepb.FileEntry files = 2; + // map files = 2; int files_size() const; void clear_files(); static const int kFilesFieldNumber = 2; - const ::cockroach::storage::engine::enginepb::FileEntry& files(int index) const; - ::cockroach::storage::engine::enginepb::FileEntry* mutable_files(int index); - ::cockroach::storage::engine::enginepb::FileEntry* add_files(); - ::google::protobuf::RepeatedPtrField< ::cockroach::storage::engine::enginepb::FileEntry >* - mutable_files(); - const ::google::protobuf::RepeatedPtrField< ::cockroach::storage::engine::enginepb::FileEntry >& + const ::google::protobuf::Map< ::std::string, ::cockroach::storage::engine::enginepb::FileEntry >& files() const; + ::google::protobuf::Map< ::std::string, ::cockroach::storage::engine::enginepb::FileEntry >* + mutable_files(); // .cockroach.storage.engine.enginepb.RegistryVersion version = 1; void clear_version(); @@ -174,11 +193,25 @@ class Registry : public ::google::protobuf::MessageLite /* @@protoc_insertion_po ::cockroach::storage::engine::enginepb::RegistryVersion version() const; void set_version(::cockroach::storage::engine::enginepb::RegistryVersion value); - // @@protoc_insertion_point(class_scope:cockroach.storage.engine.enginepb.Registry) + // @@protoc_insertion_point(class_scope:cockroach.storage.engine.enginepb.FileRegistry) private: ::google::protobuf::internal::InternalMetadataWithArenaLite _internal_metadata_; - ::google::protobuf::RepeatedPtrField< ::cockroach::storage::engine::enginepb::FileEntry > files_; + public: + typedef ::google::protobuf::internal::MapEntryLite< + ::std::string, ::cockroach::storage::engine::enginepb::FileEntry, + ::google::protobuf::internal::WireFormatLite::TYPE_STRING, + ::google::protobuf::internal::WireFormatLite::TYPE_MESSAGE, + 0 > + FileRegistry_FilesEntry; + private: + ::google::protobuf::internal::MapFieldLite< + FileRegistry_FilesEntry, + ::std::string, ::cockroach::storage::engine::enginepb::FileEntry, + ::google::protobuf::internal::WireFormatLite::TYPE_STRING, + ::google::protobuf::internal::WireFormatLite::TYPE_MESSAGE, + 0 > files_; + private: int version_; mutable int _cached_size_; friend struct protobuf_storage_2fengine_2fenginepb_2ffile_5fregistry_2eproto::TableStruct; @@ -218,7 +251,7 @@ class FileEntry : public ::google::protobuf::MessageLite /* @@protoc_insertion_p &_FileEntry_default_instance_); } static PROTOBUF_CONSTEXPR int const kIndexInFileMessages = - 1; + 2; void Swap(FileEntry* other); friend void swap(FileEntry& a, FileEntry& b) { @@ -264,32 +297,32 @@ class FileEntry : public ::google::protobuf::MessageLite /* @@protoc_insertion_p // accessors ------------------------------------------------------- - // string Filename = 1; - void clear_filename(); - static const int kFilenameFieldNumber = 1; - const ::std::string& filename() const; - void set_filename(const ::std::string& value); + // bytes encryption_settings = 2; + void clear_encryption_settings(); + static const int kEncryptionSettingsFieldNumber = 2; + const ::std::string& encryption_settings() const; + void set_encryption_settings(const ::std::string& value); #if LANG_CXX11 - void set_filename(::std::string&& value); + void set_encryption_settings(::std::string&& value); #endif - void set_filename(const char* value); - void set_filename(const char* value, size_t size); - ::std::string* mutable_filename(); - ::std::string* release_filename(); - void set_allocated_filename(::std::string* filename); - - // int32 env_level = 2; - void clear_env_level(); - static const int kEnvLevelFieldNumber = 2; - ::google::protobuf::int32 env_level() const; - void set_env_level(::google::protobuf::int32 value); + void set_encryption_settings(const char* value); + void set_encryption_settings(const void* value, size_t size); + ::std::string* mutable_encryption_settings(); + ::std::string* release_encryption_settings(); + void set_allocated_encryption_settings(::std::string* encryption_settings); + + // .cockroach.storage.engine.enginepb.EnvType env_type = 1; + void clear_env_type(); + static const int kEnvTypeFieldNumber = 1; + ::cockroach::storage::engine::enginepb::EnvType env_type() const; + void set_env_type(::cockroach::storage::engine::enginepb::EnvType value); // @@protoc_insertion_point(class_scope:cockroach.storage.engine.enginepb.FileEntry) private: ::google::protobuf::internal::InternalMetadataWithArenaLite _internal_metadata_; - ::google::protobuf::internal::ArenaStringPtr filename_; - ::google::protobuf::int32 env_level_; + ::google::protobuf::internal::ArenaStringPtr encryption_settings_; + int env_type_; mutable int _cached_size_; friend struct protobuf_storage_2fengine_2fenginepb_2ffile_5fregistry_2eproto::TableStruct; }; @@ -303,121 +336,111 @@ class FileEntry : public ::google::protobuf::MessageLite /* @@protoc_insertion_p #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wstrict-aliasing" #endif // __GNUC__ -// Registry +// ------------------------------------------------------------------- + +// FileRegistry // .cockroach.storage.engine.enginepb.RegistryVersion version = 1; -inline void Registry::clear_version() { +inline void FileRegistry::clear_version() { version_ = 0; } -inline ::cockroach::storage::engine::enginepb::RegistryVersion Registry::version() const { - // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.Registry.version) +inline ::cockroach::storage::engine::enginepb::RegistryVersion FileRegistry::version() const { + // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.FileRegistry.version) return static_cast< ::cockroach::storage::engine::enginepb::RegistryVersion >(version_); } -inline void Registry::set_version(::cockroach::storage::engine::enginepb::RegistryVersion value) { +inline void FileRegistry::set_version(::cockroach::storage::engine::enginepb::RegistryVersion value) { version_ = value; - // @@protoc_insertion_point(field_set:cockroach.storage.engine.enginepb.Registry.version) + // @@protoc_insertion_point(field_set:cockroach.storage.engine.enginepb.FileRegistry.version) } -// repeated .cockroach.storage.engine.enginepb.FileEntry files = 2; -inline int Registry::files_size() const { +// map files = 2; +inline int FileRegistry::files_size() const { return files_.size(); } -inline void Registry::clear_files() { +inline void FileRegistry::clear_files() { files_.Clear(); } -inline const ::cockroach::storage::engine::enginepb::FileEntry& Registry::files(int index) const { - // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.Registry.files) - return files_.Get(index); +inline const ::google::protobuf::Map< ::std::string, ::cockroach::storage::engine::enginepb::FileEntry >& +FileRegistry::files() const { + // @@protoc_insertion_point(field_map:cockroach.storage.engine.enginepb.FileRegistry.files) + return files_.GetMap(); } -inline ::cockroach::storage::engine::enginepb::FileEntry* Registry::mutable_files(int index) { - // @@protoc_insertion_point(field_mutable:cockroach.storage.engine.enginepb.Registry.files) - return files_.Mutable(index); -} -inline ::cockroach::storage::engine::enginepb::FileEntry* Registry::add_files() { - // @@protoc_insertion_point(field_add:cockroach.storage.engine.enginepb.Registry.files) - return files_.Add(); -} -inline ::google::protobuf::RepeatedPtrField< ::cockroach::storage::engine::enginepb::FileEntry >* -Registry::mutable_files() { - // @@protoc_insertion_point(field_mutable_list:cockroach.storage.engine.enginepb.Registry.files) - return &files_; -} -inline const ::google::protobuf::RepeatedPtrField< ::cockroach::storage::engine::enginepb::FileEntry >& -Registry::files() const { - // @@protoc_insertion_point(field_list:cockroach.storage.engine.enginepb.Registry.files) - return files_; +inline ::google::protobuf::Map< ::std::string, ::cockroach::storage::engine::enginepb::FileEntry >* +FileRegistry::mutable_files() { + // @@protoc_insertion_point(field_mutable_map:cockroach.storage.engine.enginepb.FileRegistry.files) + return files_.MutableMap(); } // ------------------------------------------------------------------- // FileEntry -// string Filename = 1; -inline void FileEntry::clear_filename() { - filename_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +// .cockroach.storage.engine.enginepb.EnvType env_type = 1; +inline void FileEntry::clear_env_type() { + env_type_ = 0; +} +inline ::cockroach::storage::engine::enginepb::EnvType FileEntry::env_type() const { + // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.FileEntry.env_type) + return static_cast< ::cockroach::storage::engine::enginepb::EnvType >(env_type_); +} +inline void FileEntry::set_env_type(::cockroach::storage::engine::enginepb::EnvType value) { + + env_type_ = value; + // @@protoc_insertion_point(field_set:cockroach.storage.engine.enginepb.FileEntry.env_type) +} + +// bytes encryption_settings = 2; +inline void FileEntry::clear_encryption_settings() { + encryption_settings_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } -inline const ::std::string& FileEntry::filename() const { - // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.FileEntry.Filename) - return filename_.GetNoArena(); +inline const ::std::string& FileEntry::encryption_settings() const { + // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) + return encryption_settings_.GetNoArena(); } -inline void FileEntry::set_filename(const ::std::string& value) { +inline void FileEntry::set_encryption_settings(const ::std::string& value) { - filename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); - // @@protoc_insertion_point(field_set:cockroach.storage.engine.enginepb.FileEntry.Filename) + encryption_settings_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) } #if LANG_CXX11 -inline void FileEntry::set_filename(::std::string&& value) { +inline void FileEntry::set_encryption_settings(::std::string&& value) { - filename_.SetNoArena( + encryption_settings_.SetNoArena( &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); - // @@protoc_insertion_point(field_set_rvalue:cockroach.storage.engine.enginepb.FileEntry.Filename) + // @@protoc_insertion_point(field_set_rvalue:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) } #endif -inline void FileEntry::set_filename(const char* value) { +inline void FileEntry::set_encryption_settings(const char* value) { GOOGLE_DCHECK(value != NULL); - filename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); - // @@protoc_insertion_point(field_set_char:cockroach.storage.engine.enginepb.FileEntry.Filename) + encryption_settings_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) } -inline void FileEntry::set_filename(const char* value, size_t size) { +inline void FileEntry::set_encryption_settings(const void* value, size_t size) { - filename_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + encryption_settings_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(reinterpret_cast(value), size)); - // @@protoc_insertion_point(field_set_pointer:cockroach.storage.engine.enginepb.FileEntry.Filename) + // @@protoc_insertion_point(field_set_pointer:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) } -inline ::std::string* FileEntry::mutable_filename() { +inline ::std::string* FileEntry::mutable_encryption_settings() { - // @@protoc_insertion_point(field_mutable:cockroach.storage.engine.enginepb.FileEntry.Filename) - return filename_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + // @@protoc_insertion_point(field_mutable:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) + return encryption_settings_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } -inline ::std::string* FileEntry::release_filename() { - // @@protoc_insertion_point(field_release:cockroach.storage.engine.enginepb.FileEntry.Filename) +inline ::std::string* FileEntry::release_encryption_settings() { + // @@protoc_insertion_point(field_release:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) - return filename_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + return encryption_settings_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); } -inline void FileEntry::set_allocated_filename(::std::string* filename) { - if (filename != NULL) { +inline void FileEntry::set_allocated_encryption_settings(::std::string* encryption_settings) { + if (encryption_settings != NULL) { } else { } - filename_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), filename); - // @@protoc_insertion_point(field_set_allocated:cockroach.storage.engine.enginepb.FileEntry.Filename) -} - -// int32 env_level = 2; -inline void FileEntry::clear_env_level() { - env_level_ = 0; -} -inline ::google::protobuf::int32 FileEntry::env_level() const { - // @@protoc_insertion_point(field_get:cockroach.storage.engine.enginepb.FileEntry.env_level) - return env_level_; -} -inline void FileEntry::set_env_level(::google::protobuf::int32 value) { - - env_level_ = value; - // @@protoc_insertion_point(field_set:cockroach.storage.engine.enginepb.FileEntry.env_level) + encryption_settings_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), encryption_settings); + // @@protoc_insertion_point(field_set_allocated:cockroach.storage.engine.enginepb.FileEntry.encryption_settings) } #ifdef __GNUC__ @@ -426,6 +449,8 @@ inline void FileEntry::set_env_level(::google::protobuf::int32 value) { #endif // !PROTOBUF_INLINE_NOT_IN_HEADERS // ------------------------------------------------------------------- +// ------------------------------------------------------------------- + // @@protoc_insertion_point(namespace_scope) @@ -439,6 +464,7 @@ namespace google { namespace protobuf { template <> struct is_proto_enum< ::cockroach::storage::engine::enginepb::RegistryVersion> : ::google::protobuf::internal::true_type {}; +template <> struct is_proto_enum< ::cockroach::storage::engine::enginepb::EnvType> : ::google::protobuf::internal::true_type {}; } // namespace protobuf } // namespace google diff --git a/c-deps/libroach/protosccl/ccl/storageccl/engineccl/enginepbccl/key_registry.pb.cc b/c-deps/libroach/protosccl/ccl/storageccl/engineccl/enginepbccl/key_registry.pb.cc index 09762932c8f0..ff6305264ffd 100644 --- a/c-deps/libroach/protosccl/ccl/storageccl/engineccl/enginepbccl/key_registry.pb.cc +++ b/c-deps/libroach/protosccl/ccl/storageccl/engineccl/enginepbccl/key_registry.pb.cc @@ -44,6 +44,11 @@ class SecretKeyDefaultTypeInternal { ::google::protobuf::internal::ExplicitlyConstructed _instance; } _SecretKey_default_instance_; +class EncryptionSettingsDefaultTypeInternal { +public: + ::google::protobuf::internal::ExplicitlyConstructed + _instance; +} _EncryptionSettings_default_instance_; namespace protobuf_ccl_2fstorageccl_2fengineccl_2fenginepbccl_2fkey_5fregistry_2eproto { @@ -63,6 +68,7 @@ PROTOBUF_CONSTEXPR_VAR ::google::protobuf::internal::ParseTable const { NULL, NULL, 0, -1, -1, -1, -1, NULL, false }, { NULL, NULL, 0, -1, -1, -1, -1, NULL, false }, { NULL, NULL, 0, -1, -1, -1, -1, NULL, false }, + { NULL, NULL, 0, -1, -1, -1, -1, NULL, false }, }; void TableStruct::InitDefaultsImpl() { @@ -79,7 +85,9 @@ void TableStruct::InitDefaultsImpl() { ::google::protobuf::internal::OnShutdownDestroyMessage( &_KeyInfo_default_instance_);_SecretKey_default_instance_._instance.DefaultConstruct(); ::google::protobuf::internal::OnShutdownDestroyMessage( - &_SecretKey_default_instance_);_DataKeysRegistry_StoreKeysEntry_default_instance_._instance.get_mutable()->set_default_instance(_DataKeysRegistry_StoreKeysEntry_default_instance_._instance.get_mutable()); + &_SecretKey_default_instance_);_EncryptionSettings_default_instance_._instance.DefaultConstruct(); + ::google::protobuf::internal::OnShutdownDestroyMessage( + &_EncryptionSettings_default_instance_);_DataKeysRegistry_StoreKeysEntry_default_instance_._instance.get_mutable()->set_default_instance(_DataKeysRegistry_StoreKeysEntry_default_instance_._instance.get_mutable()); _DataKeysRegistry_StoreKeysEntry_default_instance_._instance.get_mutable()->InitAsDefaultInstance(); _DataKeysRegistry_DataKeysEntry_default_instance_._instance.get_mutable()->set_default_instance(_DataKeysRegistry_DataKeysEntry_default_instance_._instance.get_mutable()); _DataKeysRegistry_DataKeysEntry_default_instance_._instance.get_mutable()->InitAsDefaultInstance(); @@ -1647,6 +1655,460 @@ void SecretKey::set_allocated_key(::std::string* key) { #endif // PROTOBUF_INLINE_NOT_IN_HEADERS +// =================================================================== + +#if !defined(_MSC_VER) || _MSC_VER >= 1900 +const int EncryptionSettings::kEncryptionTypeFieldNumber; +const int EncryptionSettings::kKeyIdFieldNumber; +const int EncryptionSettings::kNonceFieldNumber; +const int EncryptionSettings::kCounterFieldNumber; +#endif // !defined(_MSC_VER) || _MSC_VER >= 1900 + +EncryptionSettings::EncryptionSettings() + : ::google::protobuf::MessageLite(), _internal_metadata_(NULL) { + if (GOOGLE_PREDICT_TRUE(this != internal_default_instance())) { + protobuf_ccl_2fstorageccl_2fengineccl_2fenginepbccl_2fkey_5fregistry_2eproto::InitDefaults(); + } + SharedCtor(); + // @@protoc_insertion_point(constructor:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) +} +EncryptionSettings::EncryptionSettings(const EncryptionSettings& from) + : ::google::protobuf::MessageLite(), + _internal_metadata_(NULL), + _cached_size_(0) { + _internal_metadata_.MergeFrom(from._internal_metadata_); + key_id_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + if (from.key_id().size() > 0) { + key_id_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.key_id_); + } + nonce_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + if (from.nonce().size() > 0) { + nonce_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.nonce_); + } + ::memcpy(&encryption_type_, &from.encryption_type_, + static_cast(reinterpret_cast(&counter_) - + reinterpret_cast(&encryption_type_)) + sizeof(counter_)); + // @@protoc_insertion_point(copy_constructor:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) +} + +void EncryptionSettings::SharedCtor() { + key_id_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + nonce_.UnsafeSetDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::memset(&encryption_type_, 0, static_cast( + reinterpret_cast(&counter_) - + reinterpret_cast(&encryption_type_)) + sizeof(counter_)); + _cached_size_ = 0; +} + +EncryptionSettings::~EncryptionSettings() { + // @@protoc_insertion_point(destructor:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) + SharedDtor(); +} + +void EncryptionSettings::SharedDtor() { + key_id_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + nonce_.DestroyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} + +void EncryptionSettings::SetCachedSize(int size) const { + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); +} +const EncryptionSettings& EncryptionSettings::default_instance() { + protobuf_ccl_2fstorageccl_2fengineccl_2fenginepbccl_2fkey_5fregistry_2eproto::InitDefaults(); + return *internal_default_instance(); +} + +EncryptionSettings* EncryptionSettings::New(::google::protobuf::Arena* arena) const { + EncryptionSettings* n = new EncryptionSettings; + if (arena != NULL) { + arena->Own(n); + } + return n; +} + +void EncryptionSettings::Clear() { +// @@protoc_insertion_point(message_clear_start:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) + ::google::protobuf::uint32 cached_has_bits = 0; + // Prevent compiler warnings about cached_has_bits being unused + (void) cached_has_bits; + + key_id_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + nonce_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); + ::memset(&encryption_type_, 0, static_cast( + reinterpret_cast(&counter_) - + reinterpret_cast(&encryption_type_)) + sizeof(counter_)); + _internal_metadata_.Clear(); +} + +bool EncryptionSettings::MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) { +#define DO_(EXPRESSION) if (!GOOGLE_PREDICT_TRUE(EXPRESSION)) goto failure + ::google::protobuf::uint32 tag; + ::google::protobuf::io::LazyStringOutputStream unknown_fields_string( + ::google::protobuf::NewPermanentCallback(&_internal_metadata_, + &::google::protobuf::internal::InternalMetadataWithArenaLite:: + mutable_unknown_fields)); + ::google::protobuf::io::CodedOutputStream unknown_fields_stream( + &unknown_fields_string, false); + // @@protoc_insertion_point(parse_start:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) + for (;;) { + ::std::pair< ::google::protobuf::uint32, bool> p = input->ReadTagWithCutoffNoLastTag(127u); + tag = p.first; + if (!p.second) goto handle_unusual; + switch (::google::protobuf::internal::WireFormatLite::GetTagFieldNumber(tag)) { + // .cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionType encryption_type = 1; + case 1: { + if (static_cast< ::google::protobuf::uint8>(tag) == + static_cast< ::google::protobuf::uint8>(8u /* 8 & 0xFF */)) { + int value; + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + int, ::google::protobuf::internal::WireFormatLite::TYPE_ENUM>( + input, &value))); + set_encryption_type(static_cast< ::cockroach::ccl::storageccl::engineccl::enginepbccl::EncryptionType >(value)); + } else { + goto handle_unusual; + } + break; + } + + // string key_id = 2; + case 2: { + if (static_cast< ::google::protobuf::uint8>(tag) == + static_cast< ::google::protobuf::uint8>(18u /* 18 & 0xFF */)) { + DO_(::google::protobuf::internal::WireFormatLite::ReadString( + input, this->mutable_key_id())); + DO_(::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->key_id().data(), static_cast(this->key_id().length()), + ::google::protobuf::internal::WireFormatLite::PARSE, + "cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id")); + } else { + goto handle_unusual; + } + break; + } + + // bytes nonce = 3; + case 3: { + if (static_cast< ::google::protobuf::uint8>(tag) == + static_cast< ::google::protobuf::uint8>(26u /* 26 & 0xFF */)) { + DO_(::google::protobuf::internal::WireFormatLite::ReadBytes( + input, this->mutable_nonce())); + } else { + goto handle_unusual; + } + break; + } + + // uint32 counter = 4; + case 4: { + if (static_cast< ::google::protobuf::uint8>(tag) == + static_cast< ::google::protobuf::uint8>(32u /* 32 & 0xFF */)) { + + DO_((::google::protobuf::internal::WireFormatLite::ReadPrimitive< + ::google::protobuf::uint32, ::google::protobuf::internal::WireFormatLite::TYPE_UINT32>( + input, &counter_))); + } else { + goto handle_unusual; + } + break; + } + + default: { + handle_unusual: + if (tag == 0) { + goto success; + } + DO_(::google::protobuf::internal::WireFormatLite::SkipField( + input, tag, &unknown_fields_stream)); + break; + } + } + } +success: + // @@protoc_insertion_point(parse_success:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) + return true; +failure: + // @@protoc_insertion_point(parse_failure:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) + return false; +#undef DO_ +} + +void EncryptionSettings::SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const { + // @@protoc_insertion_point(serialize_start:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) + ::google::protobuf::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + // .cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionType encryption_type = 1; + if (this->encryption_type() != 0) { + ::google::protobuf::internal::WireFormatLite::WriteEnum( + 1, this->encryption_type(), output); + } + + // string key_id = 2; + if (this->key_id().size() > 0) { + ::google::protobuf::internal::WireFormatLite::VerifyUtf8String( + this->key_id().data(), static_cast(this->key_id().length()), + ::google::protobuf::internal::WireFormatLite::SERIALIZE, + "cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id"); + ::google::protobuf::internal::WireFormatLite::WriteStringMaybeAliased( + 2, this->key_id(), output); + } + + // bytes nonce = 3; + if (this->nonce().size() > 0) { + ::google::protobuf::internal::WireFormatLite::WriteBytesMaybeAliased( + 3, this->nonce(), output); + } + + // uint32 counter = 4; + if (this->counter() != 0) { + ::google::protobuf::internal::WireFormatLite::WriteUInt32(4, this->counter(), output); + } + + output->WriteRaw((::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()).data(), + static_cast((::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()).size())); + // @@protoc_insertion_point(serialize_end:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) +} + +size_t EncryptionSettings::ByteSizeLong() const { +// @@protoc_insertion_point(message_byte_size_start:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) + size_t total_size = 0; + + total_size += (::google::protobuf::internal::GetProto3PreserveUnknownsDefault() ? _internal_metadata_.unknown_fields() : _internal_metadata_.default_instance()).size(); + + // string key_id = 2; + if (this->key_id().size() > 0) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::StringSize( + this->key_id()); + } + + // bytes nonce = 3; + if (this->nonce().size() > 0) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::BytesSize( + this->nonce()); + } + + // .cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionType encryption_type = 1; + if (this->encryption_type() != 0) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::EnumSize(this->encryption_type()); + } + + // uint32 counter = 4; + if (this->counter() != 0) { + total_size += 1 + + ::google::protobuf::internal::WireFormatLite::UInt32Size( + this->counter()); + } + + int cached_size = ::google::protobuf::internal::ToCachedSize(total_size); + GOOGLE_SAFE_CONCURRENT_WRITES_BEGIN(); + _cached_size_ = cached_size; + GOOGLE_SAFE_CONCURRENT_WRITES_END(); + return total_size; +} + +void EncryptionSettings::CheckTypeAndMergeFrom( + const ::google::protobuf::MessageLite& from) { + MergeFrom(*::google::protobuf::down_cast(&from)); +} + +void EncryptionSettings::MergeFrom(const EncryptionSettings& from) { +// @@protoc_insertion_point(class_specific_merge_from_start:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) + GOOGLE_DCHECK_NE(&from, this); + _internal_metadata_.MergeFrom(from._internal_metadata_); + ::google::protobuf::uint32 cached_has_bits = 0; + (void) cached_has_bits; + + if (from.key_id().size() > 0) { + + key_id_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.key_id_); + } + if (from.nonce().size() > 0) { + + nonce_.AssignWithDefault(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), from.nonce_); + } + if (from.encryption_type() != 0) { + set_encryption_type(from.encryption_type()); + } + if (from.counter() != 0) { + set_counter(from.counter()); + } +} + +void EncryptionSettings::CopyFrom(const EncryptionSettings& from) { +// @@protoc_insertion_point(class_specific_copy_from_start:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) + if (&from == this) return; + Clear(); + MergeFrom(from); +} + +bool EncryptionSettings::IsInitialized() const { + return true; +} + +void EncryptionSettings::Swap(EncryptionSettings* other) { + if (other == this) return; + InternalSwap(other); +} +void EncryptionSettings::InternalSwap(EncryptionSettings* other) { + using std::swap; + key_id_.Swap(&other->key_id_); + nonce_.Swap(&other->nonce_); + swap(encryption_type_, other->encryption_type_); + swap(counter_, other->counter_); + _internal_metadata_.Swap(&other->_internal_metadata_); + swap(_cached_size_, other->_cached_size_); +} + +::std::string EncryptionSettings::GetTypeName() const { + return "cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings"; +} + +#if PROTOBUF_INLINE_NOT_IN_HEADERS +// EncryptionSettings + +// .cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionType encryption_type = 1; +void EncryptionSettings::clear_encryption_type() { + encryption_type_ = 0; +} +::cockroach::ccl::storageccl::engineccl::enginepbccl::EncryptionType EncryptionSettings::encryption_type() const { + // @@protoc_insertion_point(field_get:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.encryption_type) + return static_cast< ::cockroach::ccl::storageccl::engineccl::enginepbccl::EncryptionType >(encryption_type_); +} +void EncryptionSettings::set_encryption_type(::cockroach::ccl::storageccl::engineccl::enginepbccl::EncryptionType value) { + + encryption_type_ = value; + // @@protoc_insertion_point(field_set:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.encryption_type) +} + +// string key_id = 2; +void EncryptionSettings::clear_key_id() { + key_id_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +const ::std::string& EncryptionSettings::key_id() const { + // @@protoc_insertion_point(field_get:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) + return key_id_.GetNoArena(); +} +void EncryptionSettings::set_key_id(const ::std::string& value) { + + key_id_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) +} +#if LANG_CXX11 +void EncryptionSettings::set_key_id(::std::string&& value) { + + key_id_.SetNoArena( + &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) +} +#endif +void EncryptionSettings::set_key_id(const char* value) { + GOOGLE_DCHECK(value != NULL); + + key_id_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) +} +void EncryptionSettings::set_key_id(const char* value, size_t size) { + + key_id_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) +} +::std::string* EncryptionSettings::mutable_key_id() { + + // @@protoc_insertion_point(field_mutable:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) + return key_id_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +::std::string* EncryptionSettings::release_key_id() { + // @@protoc_insertion_point(field_release:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) + + return key_id_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +void EncryptionSettings::set_allocated_key_id(::std::string* key_id) { + if (key_id != NULL) { + + } else { + + } + key_id_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), key_id); + // @@protoc_insertion_point(field_set_allocated:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) +} + +// bytes nonce = 3; +void EncryptionSettings::clear_nonce() { + nonce_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +const ::std::string& EncryptionSettings::nonce() const { + // @@protoc_insertion_point(field_get:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) + return nonce_.GetNoArena(); +} +void EncryptionSettings::set_nonce(const ::std::string& value) { + + nonce_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) +} +#if LANG_CXX11 +void EncryptionSettings::set_nonce(::std::string&& value) { + + nonce_.SetNoArena( + &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) +} +#endif +void EncryptionSettings::set_nonce(const char* value) { + GOOGLE_DCHECK(value != NULL); + + nonce_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) +} +void EncryptionSettings::set_nonce(const void* value, size_t size) { + + nonce_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) +} +::std::string* EncryptionSettings::mutable_nonce() { + + // @@protoc_insertion_point(field_mutable:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) + return nonce_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +::std::string* EncryptionSettings::release_nonce() { + // @@protoc_insertion_point(field_release:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) + + return nonce_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +void EncryptionSettings::set_allocated_nonce(::std::string* nonce) { + if (nonce != NULL) { + + } else { + + } + nonce_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), nonce); + // @@protoc_insertion_point(field_set_allocated:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) +} + +// uint32 counter = 4; +void EncryptionSettings::clear_counter() { + counter_ = 0u; +} +::google::protobuf::uint32 EncryptionSettings::counter() const { + // @@protoc_insertion_point(field_get:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.counter) + return counter_; +} +void EncryptionSettings::set_counter(::google::protobuf::uint32 value) { + + counter_ = value; + // @@protoc_insertion_point(field_set:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.counter) +} + +#endif // PROTOBUF_INLINE_NOT_IN_HEADERS + // @@protoc_insertion_point(namespace_scope) } // namespace enginepbccl diff --git a/c-deps/libroach/protosccl/ccl/storageccl/engineccl/enginepbccl/key_registry.pb.h b/c-deps/libroach/protosccl/ccl/storageccl/engineccl/enginepbccl/key_registry.pb.h index cd4ac0e3c402..969a82caab96 100644 --- a/c-deps/libroach/protosccl/ccl/storageccl/engineccl/enginepbccl/key_registry.pb.h +++ b/c-deps/libroach/protosccl/ccl/storageccl/engineccl/enginepbccl/key_registry.pb.h @@ -47,6 +47,9 @@ extern DataKeysRegistry_DataKeysEntryDefaultTypeInternal _DataKeysRegistry_DataK class DataKeysRegistry_StoreKeysEntry; class DataKeysRegistry_StoreKeysEntryDefaultTypeInternal; extern DataKeysRegistry_StoreKeysEntryDefaultTypeInternal _DataKeysRegistry_StoreKeysEntry_default_instance_; +class EncryptionSettings; +class EncryptionSettingsDefaultTypeInternal; +extern EncryptionSettingsDefaultTypeInternal _EncryptionSettings_default_instance_; class KeyInfo; class KeyInfoDefaultTypeInternal; extern KeyInfoDefaultTypeInternal _KeyInfo_default_instance_; @@ -533,6 +536,138 @@ class SecretKey : public ::google::protobuf::MessageLite /* @@protoc_insertion_p mutable int _cached_size_; friend struct protobuf_ccl_2fstorageccl_2fengineccl_2fenginepbccl_2fkey_5fregistry_2eproto::TableStruct; }; +// ------------------------------------------------------------------- + +class EncryptionSettings : public ::google::protobuf::MessageLite /* @@protoc_insertion_point(class_definition:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) */ { + public: + EncryptionSettings(); + virtual ~EncryptionSettings(); + + EncryptionSettings(const EncryptionSettings& from); + + inline EncryptionSettings& operator=(const EncryptionSettings& from) { + CopyFrom(from); + return *this; + } + #if LANG_CXX11 + EncryptionSettings(EncryptionSettings&& from) noexcept + : EncryptionSettings() { + *this = ::std::move(from); + } + + inline EncryptionSettings& operator=(EncryptionSettings&& from) noexcept { + if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) { + if (this != &from) InternalSwap(&from); + } else { + CopyFrom(from); + } + return *this; + } + #endif + static const EncryptionSettings& default_instance(); + + static inline const EncryptionSettings* internal_default_instance() { + return reinterpret_cast( + &_EncryptionSettings_default_instance_); + } + static PROTOBUF_CONSTEXPR int const kIndexInFileMessages = + 5; + + void Swap(EncryptionSettings* other); + friend void swap(EncryptionSettings& a, EncryptionSettings& b) { + a.Swap(&b); + } + + // implements Message ---------------------------------------------- + + inline EncryptionSettings* New() const PROTOBUF_FINAL { return New(NULL); } + + EncryptionSettings* New(::google::protobuf::Arena* arena) const PROTOBUF_FINAL; + void CheckTypeAndMergeFrom(const ::google::protobuf::MessageLite& from) + PROTOBUF_FINAL; + void CopyFrom(const EncryptionSettings& from); + void MergeFrom(const EncryptionSettings& from); + void Clear() PROTOBUF_FINAL; + bool IsInitialized() const PROTOBUF_FINAL; + + size_t ByteSizeLong() const PROTOBUF_FINAL; + bool MergePartialFromCodedStream( + ::google::protobuf::io::CodedInputStream* input) PROTOBUF_FINAL; + void SerializeWithCachedSizes( + ::google::protobuf::io::CodedOutputStream* output) const PROTOBUF_FINAL; + void DiscardUnknownFields(); + int GetCachedSize() const PROTOBUF_FINAL { return _cached_size_; } + private: + void SharedCtor(); + void SharedDtor(); + void SetCachedSize(int size) const; + void InternalSwap(EncryptionSettings* other); + private: + inline ::google::protobuf::Arena* GetArenaNoVirtual() const { + return NULL; + } + inline void* MaybeArenaPtr() const { + return NULL; + } + public: + + ::std::string GetTypeName() const PROTOBUF_FINAL; + + // nested types ---------------------------------------------------- + + // accessors ------------------------------------------------------- + + // string key_id = 2; + void clear_key_id(); + static const int kKeyIdFieldNumber = 2; + const ::std::string& key_id() const; + void set_key_id(const ::std::string& value); + #if LANG_CXX11 + void set_key_id(::std::string&& value); + #endif + void set_key_id(const char* value); + void set_key_id(const char* value, size_t size); + ::std::string* mutable_key_id(); + ::std::string* release_key_id(); + void set_allocated_key_id(::std::string* key_id); + + // bytes nonce = 3; + void clear_nonce(); + static const int kNonceFieldNumber = 3; + const ::std::string& nonce() const; + void set_nonce(const ::std::string& value); + #if LANG_CXX11 + void set_nonce(::std::string&& value); + #endif + void set_nonce(const char* value); + void set_nonce(const void* value, size_t size); + ::std::string* mutable_nonce(); + ::std::string* release_nonce(); + void set_allocated_nonce(::std::string* nonce); + + // .cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionType encryption_type = 1; + void clear_encryption_type(); + static const int kEncryptionTypeFieldNumber = 1; + ::cockroach::ccl::storageccl::engineccl::enginepbccl::EncryptionType encryption_type() const; + void set_encryption_type(::cockroach::ccl::storageccl::engineccl::enginepbccl::EncryptionType value); + + // uint32 counter = 4; + void clear_counter(); + static const int kCounterFieldNumber = 4; + ::google::protobuf::uint32 counter() const; + void set_counter(::google::protobuf::uint32 value); + + // @@protoc_insertion_point(class_scope:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings) + private: + + ::google::protobuf::internal::InternalMetadataWithArenaLite _internal_metadata_; + ::google::protobuf::internal::ArenaStringPtr key_id_; + ::google::protobuf::internal::ArenaStringPtr nonce_; + int encryption_type_; + ::google::protobuf::uint32 counter_; + mutable int _cached_size_; + friend struct protobuf_ccl_2fstorageccl_2fengineccl_2fenginepbccl_2fkey_5fregistry_2eproto::TableStruct; +}; // =================================================================== @@ -993,6 +1128,144 @@ inline void SecretKey::set_allocated_key(::std::string* key) { // @@protoc_insertion_point(field_set_allocated:cockroach.ccl.storageccl.engineccl.enginepbccl.SecretKey.key) } +// ------------------------------------------------------------------- + +// EncryptionSettings + +// .cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionType encryption_type = 1; +inline void EncryptionSettings::clear_encryption_type() { + encryption_type_ = 0; +} +inline ::cockroach::ccl::storageccl::engineccl::enginepbccl::EncryptionType EncryptionSettings::encryption_type() const { + // @@protoc_insertion_point(field_get:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.encryption_type) + return static_cast< ::cockroach::ccl::storageccl::engineccl::enginepbccl::EncryptionType >(encryption_type_); +} +inline void EncryptionSettings::set_encryption_type(::cockroach::ccl::storageccl::engineccl::enginepbccl::EncryptionType value) { + + encryption_type_ = value; + // @@protoc_insertion_point(field_set:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.encryption_type) +} + +// string key_id = 2; +inline void EncryptionSettings::clear_key_id() { + key_id_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline const ::std::string& EncryptionSettings::key_id() const { + // @@protoc_insertion_point(field_get:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) + return key_id_.GetNoArena(); +} +inline void EncryptionSettings::set_key_id(const ::std::string& value) { + + key_id_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) +} +#if LANG_CXX11 +inline void EncryptionSettings::set_key_id(::std::string&& value) { + + key_id_.SetNoArena( + &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) +} +#endif +inline void EncryptionSettings::set_key_id(const char* value) { + GOOGLE_DCHECK(value != NULL); + + key_id_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) +} +inline void EncryptionSettings::set_key_id(const char* value, size_t size) { + + key_id_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) +} +inline ::std::string* EncryptionSettings::mutable_key_id() { + + // @@protoc_insertion_point(field_mutable:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) + return key_id_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline ::std::string* EncryptionSettings::release_key_id() { + // @@protoc_insertion_point(field_release:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) + + return key_id_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline void EncryptionSettings::set_allocated_key_id(::std::string* key_id) { + if (key_id != NULL) { + + } else { + + } + key_id_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), key_id); + // @@protoc_insertion_point(field_set_allocated:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.key_id) +} + +// bytes nonce = 3; +inline void EncryptionSettings::clear_nonce() { + nonce_.ClearToEmptyNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline const ::std::string& EncryptionSettings::nonce() const { + // @@protoc_insertion_point(field_get:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) + return nonce_.GetNoArena(); +} +inline void EncryptionSettings::set_nonce(const ::std::string& value) { + + nonce_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), value); + // @@protoc_insertion_point(field_set:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) +} +#if LANG_CXX11 +inline void EncryptionSettings::set_nonce(::std::string&& value) { + + nonce_.SetNoArena( + &::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::move(value)); + // @@protoc_insertion_point(field_set_rvalue:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) +} +#endif +inline void EncryptionSettings::set_nonce(const char* value) { + GOOGLE_DCHECK(value != NULL); + + nonce_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), ::std::string(value)); + // @@protoc_insertion_point(field_set_char:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) +} +inline void EncryptionSettings::set_nonce(const void* value, size_t size) { + + nonce_.SetNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), + ::std::string(reinterpret_cast(value), size)); + // @@protoc_insertion_point(field_set_pointer:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) +} +inline ::std::string* EncryptionSettings::mutable_nonce() { + + // @@protoc_insertion_point(field_mutable:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) + return nonce_.MutableNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline ::std::string* EncryptionSettings::release_nonce() { + // @@protoc_insertion_point(field_release:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) + + return nonce_.ReleaseNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited()); +} +inline void EncryptionSettings::set_allocated_nonce(::std::string* nonce) { + if (nonce != NULL) { + + } else { + + } + nonce_.SetAllocatedNoArena(&::google::protobuf::internal::GetEmptyStringAlreadyInited(), nonce); + // @@protoc_insertion_point(field_set_allocated:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.nonce) +} + +// uint32 counter = 4; +inline void EncryptionSettings::clear_counter() { + counter_ = 0u; +} +inline ::google::protobuf::uint32 EncryptionSettings::counter() const { + // @@protoc_insertion_point(field_get:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.counter) + return counter_; +} +inline void EncryptionSettings::set_counter(::google::protobuf::uint32 value) { + + counter_ = value; + // @@protoc_insertion_point(field_set:cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings.counter) +} + #ifdef __GNUC__ #pragma GCC diagnostic pop #endif // __GNUC__ @@ -1005,6 +1278,8 @@ inline void SecretKey::set_allocated_key(::std::string* key) { // ------------------------------------------------------------------- +// ------------------------------------------------------------------- + // @@protoc_insertion_point(namespace_scope) diff --git a/c-deps/libroach/rocksdbutils/aligned_buffer.h b/c-deps/libroach/rocksdbutils/aligned_buffer.h new file mode 100644 index 000000000000..5c71a0aa5880 --- /dev/null +++ b/c-deps/libroach/rocksdbutils/aligned_buffer.h @@ -0,0 +1,166 @@ +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found at +// https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +// and Apache 2.0 License (found in licenses/APL.txt in the root +// of this repository). +// +// Copyright (c) 2011 The LevelDB Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the licences/LevelDB.txt file in in the root of this repository. +// See the LevelDB.AUTHORS file for names of contributors. +#pragma once + +#include +#include +#include +#include + +namespace rocksdb_utils { + +// Needed parts from rocksdb/port/win/port_win.h +// VS < 2015 +#if defined(OS_WIN) && defined(_MSC_VER) && (_MSC_VER < 1900) +#define ROCKSDB_NOEXCEPT +#else // VS >= 2015 or MinGW +#define ROCKSDB_NOEXCEPT noexcept +#endif + +inline size_t TruncateToPageBoundary(size_t page_size, size_t s) { + s -= (s & (page_size - 1)); + assert((s % page_size) == 0); + return s; +} + +inline size_t Roundup(size_t x, size_t y) { return ((x + y - 1) / y) * y; } + +// This class is to manage an aligned user +// allocated buffer for direct I/O purposes +// though can be used for any purpose. +class AlignedBuffer { + size_t alignment_; + std::unique_ptr buf_; + size_t capacity_; + size_t cursize_; + char* bufstart_; + + public: + AlignedBuffer() : alignment_(), capacity_(0), cursize_(0), bufstart_(nullptr) {} + + AlignedBuffer(AlignedBuffer&& o) ROCKSDB_NOEXCEPT { *this = std::move(o); } + + AlignedBuffer& operator=(AlignedBuffer&& o) ROCKSDB_NOEXCEPT { + alignment_ = std::move(o.alignment_); + buf_ = std::move(o.buf_); + capacity_ = std::move(o.capacity_); + cursize_ = std::move(o.cursize_); + bufstart_ = std::move(o.bufstart_); + return *this; + } + + AlignedBuffer(const AlignedBuffer&) = delete; + + AlignedBuffer& operator=(const AlignedBuffer&) = delete; + + static bool isAligned(const void* ptr, size_t alignment) { + return reinterpret_cast(ptr) % alignment == 0; + } + + static bool isAligned(size_t n, size_t alignment) { return n % alignment == 0; } + + size_t Alignment() const { return alignment_; } + + size_t Capacity() const { return capacity_; } + + size_t CurrentSize() const { return cursize_; } + + const char* BufferStart() const { return bufstart_; } + + char* BufferStart() { return bufstart_; } + + void Clear() { cursize_ = 0; } + + void Alignment(size_t alignment) { + assert(alignment > 0); + assert((alignment & (alignment - 1)) == 0); + alignment_ = alignment; + } + + // Allocates a new buffer and sets bufstart_ to the aligned first byte + void AllocateNewBuffer(size_t requested_capacity, bool copy_data = false) { + assert(alignment_ > 0); + assert((alignment_ & (alignment_ - 1)) == 0); + + if (copy_data && requested_capacity < cursize_) { + // If we are downsizing to a capacity that is smaller than the current + // data in the buffer. Ignore the request. + return; + } + + size_t new_capacity = Roundup(requested_capacity, alignment_); + char* new_buf = new char[new_capacity + alignment_]; + char* new_bufstart = + reinterpret_cast((reinterpret_cast(new_buf) + (alignment_ - 1)) & + ~static_cast(alignment_ - 1)); + + if (copy_data) { + memcpy(new_bufstart, bufstart_, cursize_); + } else { + cursize_ = 0; + } + + bufstart_ = new_bufstart; + capacity_ = new_capacity; + buf_.reset(new_buf); + } + // Used for write + // Returns the number of bytes appended + size_t Append(const char* src, size_t append_size) { + size_t buffer_remaining = capacity_ - cursize_; + size_t to_copy = std::min(append_size, buffer_remaining); + + if (to_copy > 0) { + memcpy(bufstart_ + cursize_, src, to_copy); + cursize_ += to_copy; + } + return to_copy; + } + + size_t Read(char* dest, size_t offset, size_t read_size) const { + assert(offset < cursize_); + + size_t to_read = 0; + if (offset < cursize_) { + to_read = std::min(cursize_ - offset, read_size); + } + if (to_read > 0) { + memcpy(dest, bufstart_ + offset, to_read); + } + return to_read; + } + + /// Pad to alignment + void PadToAlignmentWith(int padding) { + size_t total_size = Roundup(cursize_, alignment_); + size_t pad_size = total_size - cursize_; + + if (pad_size > 0) { + assert((pad_size + cursize_) <= capacity_); + memset(bufstart_ + cursize_, padding, pad_size); + cursize_ += pad_size; + } + } + + // After a partial flush move the tail to the beginning of the buffer + void RefitTail(size_t tail_offset, size_t tail_size) { + if (tail_size > 0) { + memmove(bufstart_, bufstart_ + tail_offset, tail_size); + } + cursize_ = tail_size; + } + + // Returns place to start writing + char* Destination() { return bufstart_ + cursize_; } + + void Size(size_t cursize) { cursize_ = cursize; } +}; +} // namespace rocksdb_utils diff --git a/c-deps/libroach/rocksdbutils/env_encryption.cc b/c-deps/libroach/rocksdbutils/env_encryption.cc new file mode 100644 index 000000000000..8abbb2b5e926 --- /dev/null +++ b/c-deps/libroach/rocksdbutils/env_encryption.cc @@ -0,0 +1,691 @@ +// Copyright 2018 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. +// +// +// +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found at +// https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +// and Apache 2.0 License (found in licenses/APL.txt in the root +// of this repository). + +#include + +#include "../file_registry.h" +#include "../fmt.h" +#include "../plaintext_stream.h" +#include "aligned_buffer.h" +#include "env_encryption.h" + +using namespace cockroach; + +namespace rocksdb_utils { + +class EncryptedSequentialFile : public rocksdb::SequentialFile { + private: + std::unique_ptr file_; + std::unique_ptr stream_; + uint64_t offset_; + + public: + // Default constructor. + EncryptedSequentialFile(rocksdb::SequentialFile* f, BlockAccessCipherStream* s) + : file_(f), stream_(s), offset_(0) {} + + // Read up to "n" bytes from the file. "scratch[0..n-1]" may be + // written by this routine. Sets "*result" to the data that was + // read (including if fewer than "n" bytes were successfully read). + // May set "*result" to point at data in "scratch[0..n-1]", so + // "scratch[0..n-1]" must be live when "*result" is used. + // If an error was encountered, returns a non-OK status. + // + // REQUIRES: External synchronization + virtual rocksdb::Status Read(size_t n, rocksdb::Slice* result, char* scratch) override { + assert(scratch); + rocksdb::Status status = file_->Read(n, result, scratch); + if (!status.ok()) { + return status; + } + status = stream_->Decrypt(offset_, (char*)result->data(), result->size()); + offset_ += result->size(); // We've already ready data from disk, so update offset_ even if + // decryption fails. + return status; + } + + // Skip "n" bytes from the file. This is guaranteed to be no + // slower that reading the same data, but may be faster. + // + // If end of file is reached, skipping will stop at the end of the + // file, and Skip will return OK. + // + // REQUIRES: External synchronization + virtual rocksdb::Status Skip(uint64_t n) override { + auto status = file_->Skip(n); + if (!status.ok()) { + return status; + } + offset_ += n; + return status; + } + + // Indicates the upper layers if the current SequentialFile implementation + // uses direct IO. + virtual bool use_direct_io() const override { return file_->use_direct_io(); } + + // Use the returned alignment value to allocate + // aligned buffer for Direct I/O + virtual size_t GetRequiredBufferAlignment() const override { + return file_->GetRequiredBufferAlignment(); + } + + // Remove any kind of caching of data from the offset to offset+length + // of this file. If the length is 0, then it refers to the end of file. + // If the system is not caching the file contents, then this is a noop. + virtual rocksdb::Status InvalidateCache(size_t offset, size_t length) override { + return file_->InvalidateCache(offset, length); + } + + // Positioned Read for direct I/O + // If Direct I/O enabled, offset, n, and scratch should be properly aligned + virtual rocksdb::Status PositionedRead(uint64_t offset, size_t n, rocksdb::Slice* result, + char* scratch) override { + assert(scratch); + auto status = file_->PositionedRead(offset, n, result, scratch); + if (!status.ok()) { + return status; + } + offset_ = offset + result->size(); + status = stream_->Decrypt(offset, (char*)result->data(), result->size()); + return status; + } +}; + +// A file abstraction for randomly reading the contents of a file. +class EncryptedRandomAccessFile : public rocksdb::RandomAccessFile { + private: + std::unique_ptr file_; + std::unique_ptr stream_; + + public: + EncryptedRandomAccessFile(rocksdb::RandomAccessFile* f, BlockAccessCipherStream* s) + : file_(f), stream_(s) {} + + // Read up to "n" bytes from the file starting at "offset". + // "scratch[0..n-1]" may be written by this routine. Sets "*result" + // to the data that was read (including if fewer than "n" bytes were + // successfully read). May set "*result" to point at data in + // "scratch[0..n-1]", so "scratch[0..n-1]" must be live when + // "*result" is used. If an error was encountered, returns a non-OK + // status. + // + // Safe for concurrent use by multiple threads. + // If Direct I/O enabled, offset, n, and scratch should be aligned properly. + virtual rocksdb::Status Read(uint64_t offset, size_t n, rocksdb::Slice* result, + char* scratch) const override { + assert(scratch); + auto status = file_->Read(offset, n, result, scratch); + if (!status.ok()) { + return status; + } + status = stream_->Decrypt(offset, (char*)result->data(), result->size()); + return status; + } + + // Readahead the file starting from offset by n bytes for caching. + virtual rocksdb::Status Prefetch(uint64_t offset, size_t n) override { + // return rocksdb::Status::OK(); + return file_->Prefetch(offset, n); + } + + // Tries to get an unique ID for this file that will be the same each time + // the file is opened (and will stay the same while the file is open). + // Furthermore, it tries to make this ID at most "max_size" bytes. If such an + // ID can be created this function returns the length of the ID and places it + // in "id"; otherwise, this function returns 0, in which case "id" + // may not have been modified. + // + // This function guarantees, for IDs from a given environment, two unique ids + // cannot be made equal to eachother by adding arbitrary bytes to one of + // them. That is, no unique ID is the prefix of another. + // + // This function guarantees that the returned ID will not be interpretable as + // a single varint. + // + // Note: these IDs are only valid for the duration of the process. + virtual size_t GetUniqueId(char* id, size_t max_size) const override { + return file_->GetUniqueId(id, max_size); + }; + + virtual void Hint(AccessPattern pattern) override { file_->Hint(pattern); } + + // Indicates the upper layers if the current RandomAccessFile implementation + // uses direct IO. + virtual bool use_direct_io() const override { return file_->use_direct_io(); } + + // Use the returned alignment value to allocate + // aligned buffer for Direct I/O + virtual size_t GetRequiredBufferAlignment() const override { + return file_->GetRequiredBufferAlignment(); + } + + // Remove any kind of caching of data from the offset to offset+length + // of this file. If the length is 0, then it refers to the end of file. + // If the system is not caching the file contents, then this is a noop. + virtual rocksdb::Status InvalidateCache(size_t offset, size_t length) override { + return file_->InvalidateCache(offset, length); + } +}; + +// A file abstraction for sequential writing. The implementation +// must provide buffering since callers may append small fragments +// at a time to the file. +class EncryptedWritableFile : public rocksdb::WritableFileWrapper { + private: + std::unique_ptr file_; + std::unique_ptr stream_; + + public: + // Default constructor. + EncryptedWritableFile(rocksdb::WritableFile* f, BlockAccessCipherStream* s) + : rocksdb::WritableFileWrapper(f), file_(f), stream_(s) {} + + rocksdb::Status Append(const rocksdb::Slice& data) override { + AlignedBuffer buf; + rocksdb::Status status; + rocksdb::Slice dataToAppend(data); + if (data.size() > 0) { + auto offset = file_->GetFileSize(); // size including prefix + // Encrypt in cloned buffer + buf.Alignment(GetRequiredBufferAlignment()); + buf.AllocateNewBuffer(data.size()); + memmove(buf.BufferStart(), data.data(), data.size()); + status = stream_->Encrypt(offset, buf.BufferStart(), data.size()); + if (!status.ok()) { + return status; + } + dataToAppend = rocksdb::Slice(buf.BufferStart(), data.size()); + } + status = file_->Append(dataToAppend); + if (!status.ok()) { + return status; + } + return status; + } + + rocksdb::Status PositionedAppend(const rocksdb::Slice& data, uint64_t offset) override { + AlignedBuffer buf; + rocksdb::Status status; + rocksdb::Slice dataToAppend(data); + if (data.size() > 0) { + // Encrypt in cloned buffer + buf.Alignment(GetRequiredBufferAlignment()); + buf.AllocateNewBuffer(data.size()); + memmove(buf.BufferStart(), data.data(), data.size()); + status = stream_->Encrypt(offset, buf.BufferStart(), data.size()); + if (!status.ok()) { + return status; + } + dataToAppend = rocksdb::Slice(buf.BufferStart(), data.size()); + } + status = file_->PositionedAppend(dataToAppend, offset); + if (!status.ok()) { + return status; + } + return status; + } + + // Indicates the upper layers if the current WritableFile implementation + // uses direct IO. + virtual bool use_direct_io() const override { return file_->use_direct_io(); } + + // Use the returned alignment value to allocate + // aligned buffer for Direct I/O + virtual size_t GetRequiredBufferAlignment() const override { + return file_->GetRequiredBufferAlignment(); + } +}; + +// A file abstraction for random reading and writing. +class EncryptedRandomRWFile : public rocksdb::RandomRWFile { + private: + std::unique_ptr file_; + std::unique_ptr stream_; + + public: + EncryptedRandomRWFile(rocksdb::RandomRWFile* f, BlockAccessCipherStream* s) + : file_(f), stream_(s) {} + + // Indicates if the class makes use of direct I/O + // If false you must pass aligned buffer to Write() + virtual bool use_direct_io() const override { return file_->use_direct_io(); } + + // Use the returned alignment value to allocate + // aligned buffer for Direct I/O + virtual size_t GetRequiredBufferAlignment() const override { + return file_->GetRequiredBufferAlignment(); + } + + // Write bytes in `data` at offset `offset`, Returns rocksdb::Status::OK() on success. + // Pass aligned buffer when use_direct_io() returns true. + virtual rocksdb::Status Write(uint64_t offset, const rocksdb::Slice& data) override { + AlignedBuffer buf; + rocksdb::Status status; + rocksdb::Slice dataToWrite(data); + if (data.size() > 0) { + // Encrypt in cloned buffer + buf.Alignment(GetRequiredBufferAlignment()); + buf.AllocateNewBuffer(data.size()); + memmove(buf.BufferStart(), data.data(), data.size()); + status = stream_->Encrypt(offset, buf.BufferStart(), data.size()); + if (!status.ok()) { + return status; + } + dataToWrite = rocksdb::Slice(buf.BufferStart(), data.size()); + } + status = file_->Write(offset, dataToWrite); + return status; + } + + // Read up to `n` bytes starting from offset `offset` and store them in + // result, provided `scratch` size should be at least `n`. + // Returns rocksdb::Status::OK() on success. + virtual rocksdb::Status Read(uint64_t offset, size_t n, rocksdb::Slice* result, + char* scratch) const override { + assert(scratch); + auto status = file_->Read(offset, n, result, scratch); + if (!status.ok()) { + return status; + } + status = stream_->Decrypt(offset, (char*)result->data(), result->size()); + return status; + } + + virtual rocksdb::Status Flush() override { return file_->Flush(); } + + virtual rocksdb::Status Sync() override { return file_->Sync(); } + + virtual rocksdb::Status Fsync() override { return file_->Fsync(); } + + virtual rocksdb::Status Close() override { return file_->Close(); } +}; + +// EncryptedEnv implements an Env wrapper that adds encryption to files stored on disk. +class EncryptedEnv : public rocksdb::EnvWrapper { + public: + EncryptedEnv(rocksdb::Env* base_env, FileRegistry* file_registry, CipherStreamCreator* creator) + : rocksdb::EnvWrapper(base_env), file_registry_(file_registry), stream_creator_(creator) {} + + // NewSequentialFile opens a file for sequential reading. + virtual rocksdb::Status NewSequentialFile(const std::string& fname, + std::unique_ptr* result, + const rocksdb::EnvOptions& options) override { + result->reset(); + if (options.use_mmap_reads) { + return rocksdb::Status::InvalidArgument(); + } + // Open file using underlying Env implementation + std::unique_ptr underlying; + auto status = rocksdb::EnvWrapper::NewSequentialFile(fname, &underlying, options); + if (!status.ok()) { + return status; + } + + // Create cipher stream + std::unique_ptr stream; + status = CreateCipherStream(fname, false /* new_file */, &stream); + if (!status.ok()) { + return status; + } + (*result) = std::unique_ptr( + new EncryptedSequentialFile(underlying.release(), stream.release())); + return rocksdb::Status::OK(); + } + + // NewRandomAccessFile opens a file for random read access. + virtual rocksdb::Status NewRandomAccessFile(const std::string& fname, + std::unique_ptr* result, + const rocksdb::EnvOptions& options) override { + result->reset(); + if (options.use_mmap_reads) { + return rocksdb::Status::InvalidArgument(); + } + // Open file using underlying Env implementation + std::unique_ptr underlying; + auto status = rocksdb::EnvWrapper::NewRandomAccessFile(fname, &underlying, options); + if (!status.ok()) { + return status; + } + + // Create cipher stream + std::unique_ptr stream; + status = CreateCipherStream(fname, false /* new_file */, &stream); + if (!status.ok()) { + return status; + } + (*result) = std::unique_ptr( + new EncryptedRandomAccessFile(underlying.release(), stream.release())); + return rocksdb::Status::OK(); + } + + // NewWritableFile opens a file for sequential writing. + virtual rocksdb::Status NewWritableFile(const std::string& fname, + std::unique_ptr* result, + const rocksdb::EnvOptions& options) override { + result->reset(); + if (options.use_mmap_writes) { + return rocksdb::Status::InvalidArgument(); + } + // Open file using underlying Env implementation + std::unique_ptr underlying; + rocksdb::Status status = rocksdb::EnvWrapper::NewWritableFile(fname, &underlying, options); + if (!status.ok()) { + return status; + } + + // Create cipher stream + std::unique_ptr stream; + status = CreateCipherStream(fname, true /* new_file */, &stream); + if (!status.ok()) { + return status; + } + (*result) = std::unique_ptr( + new EncryptedWritableFile(underlying.release(), stream.release())); + return rocksdb::Status::OK(); + } + + // Create an object that writes to a new file with the specified + // name. Deletes any existing file with the same name and creates a + // new file. On success, stores a pointer to the new file in + // *result and returns OK. On failure stores nullptr in *result and + // returns non-OK. + // + // The returned file will only be accessed by one thread at a time. + virtual rocksdb::Status ReopenWritableFile(const std::string& fname, + std::unique_ptr* result, + const rocksdb::EnvOptions& options) override { + result->reset(); + if (options.use_mmap_writes) { + return rocksdb::Status::InvalidArgument(); + } + // Open file using underlying Env implementation + std::unique_ptr underlying; + rocksdb::Status status = rocksdb::EnvWrapper::ReopenWritableFile(fname, &underlying, options); + if (!status.ok()) { + return status; + } + + // Create cipher stream + std::unique_ptr stream; + status = CreateCipherStream(fname, true /* new_file */, &stream); + if (!status.ok()) { + return status; + } + (*result) = std::unique_ptr( + new EncryptedWritableFile(underlying.release(), stream.release())); + return rocksdb::Status::OK(); + } + + // Reuse an existing file by renaming it and opening it as writable. + virtual rocksdb::Status ReuseWritableFile(const std::string& fname, const std::string& old_fname, + std::unique_ptr* result, + const rocksdb::EnvOptions& options) override { + result->reset(); + if (options.use_mmap_writes) { + return rocksdb::Status::InvalidArgument(); + } + // Open file using underlying Env implementation + std::unique_ptr underlying; + rocksdb::Status status = + rocksdb::EnvWrapper::ReuseWritableFile(fname, old_fname, &underlying, options); + if (!status.ok()) { + return status; + } + + // Create cipher stream + std::unique_ptr stream; + status = CreateCipherStream(fname, true /* new_file */, &stream); + if (!status.ok()) { + return status; + } + (*result) = std::unique_ptr( + new EncryptedWritableFile(underlying.release(), stream.release())); + return rocksdb::Status::OK(); + } + + // Open `fname` for random read and write, if file dont exist the file + // will be created. On success, stores a pointer to the new file in + // *result and returns OK. On failure returns non-OK. + // + // The returned file will only be accessed by one thread at a time. + virtual rocksdb::Status NewRandomRWFile(const std::string& fname, + std::unique_ptr* result, + const rocksdb::EnvOptions& options) override { + result->reset(); + if (options.use_mmap_reads || options.use_mmap_writes) { + return rocksdb::Status::InvalidArgument(); + } + // Check file exists + bool isNewFile = !FileExists(fname).ok(); + + // Open file using underlying Env implementation + std::unique_ptr underlying; + rocksdb::Status status = rocksdb::EnvWrapper::NewRandomRWFile(fname, &underlying, options); + if (!status.ok()) { + return status; + } + + // Create cipher stream, indicating whether this is a new file. + std::unique_ptr stream; + status = CreateCipherStream(fname, isNewFile /* new_file */, &stream); + if (!status.ok()) { + return status; + } + (*result) = std::unique_ptr( + new EncryptedRandomRWFile(underlying.release(), stream.release())); + return rocksdb::Status::OK(); + } + + virtual rocksdb::Status DeleteFile(const std::string& fname) override { + auto status = file_registry_->MaybeDeleteEntry(fname); + if (!status.ok()) { + return status; + } + return EnvWrapper::DeleteFile(fname); + } + virtual rocksdb::Status RenameFile(const std::string& src, const std::string& target) override { + auto status = file_registry_->MaybeRenameEntry(src, target); + if (!status.ok()) { + return status; + } + return EnvWrapper::RenameFile(src, target); + } + + virtual rocksdb::Status LinkFile(const std::string& src, const std::string& target) override { + auto status = file_registry_->MaybeLinkEntry(src, target); + if (!status.ok()) { + return status; + } + return EnvWrapper::LinkFile(src, target); + } + + rocksdb::Status + CreateCipherStream(const std::string& fname, bool new_file, + std::unique_ptr* result) { + if (new_file) { + return InitAndCreateCipherStream(fname, result); + } else { + return LookupAndCreateCipherStream(fname, result); + } + } + + rocksdb::Status LookupAndCreateCipherStream(const std::string& fname, + std::unique_ptr* result) { + + // Look for file in registry. + auto file_entry = file_registry_->GetFileEntry(fname); + if (file_entry == nullptr) { + // No entry: plaintext. + (*result) = std::unique_ptr(new PlaintextStream()); + return rocksdb::Status::OK(); + } + + // File entry exists: check that the env_type corresponds to the requested one. + if (file_entry->env_type() != stream_creator_->GetEnvType()) { + return rocksdb::Status::InvalidArgument( + fmt::StringPrintf("file %s was written using env_type %d, but we are trying to read it " + "using env_type %d: problems will occur", + fname.c_str(), file_entry->env_type(), stream_creator_->GetEnvType())); + } + + return stream_creator_->CreateCipherStreamFromSettings(file_entry->encryption_settings(), + result); + } + + rocksdb::Status InitAndCreateCipherStream(const std::string& fname, + std::unique_ptr* result) { + + // Look for file in registry. + auto file_entry = file_registry_->GetFileEntry(fname); + if (file_entry != nullptr && file_entry->env_type() != stream_creator_->GetEnvType()) { + // File entry exists but the env_type does not match the requested one. + return rocksdb::Status::InvalidArgument( + fmt::StringPrintf("file %s was written using env_type %d, but we are overwriting it " + "using env_type %d: problems will occur", + fname.c_str(), file_entry->env_type(), stream_creator_->GetEnvType())); + } + + std::string encryption_settings; + auto status = stream_creator_->InitSettingsAndCreateCipherStream(&encryption_settings, result); + if (!status.ok()) { + return status; + } + + if (encryption_settings.size() == 0) { + // Plaintext: delete registry entry if is exists. + return file_registry_->MaybeDeleteEntry(fname); + } else { + // Encryption settings specified: create a FileEntry and save it, overwriting any existing + // one. + std::unique_ptr new_entry(new enginepb::FileEntry()); + new_entry->set_env_type(stream_creator_->GetEnvType()); + new_entry->set_encryption_settings(encryption_settings); + return file_registry_->SetFileEntry(fname, std::move(new_entry)); + } + + return rocksdb::Status::OK(); + } + + private: + FileRegistry* file_registry_; + std::unique_ptr stream_creator_; +}; + +// Returns an Env that encrypts data when stored on disk and decrypts data when +// read from disk. +rocksdb::Env* NewEncryptedEnv(rocksdb::Env* base_env, FileRegistry* file_registry, + CipherStreamCreator* creator) { + return new EncryptedEnv(base_env, file_registry, creator); +} + +// Encrypt one or more (partial) blocks of data at the file offset. +// Length of data is given in dataSize. +rocksdb::Status BlockAccessCipherStream::Encrypt(uint64_t fileOffset, char* data, size_t dataSize) { + // Calculate block index + auto blockSize = BlockSize(); + uint64_t blockIndex = fileOffset / blockSize; + size_t blockOffset = fileOffset % blockSize; + std::unique_ptr blockBuffer; + + std::string scratch; + AllocateScratch(scratch); + + // Encrypt individual blocks. + while (1) { + char* block = data; + size_t n = std::min(dataSize, blockSize - blockOffset); + if (n != blockSize) { + // We're not encrypting a full block. + // Copy data to blockBuffer + if (!blockBuffer.get()) { + // Allocate buffer + blockBuffer = std::unique_ptr(new char[blockSize]); + } + block = blockBuffer.get(); + // Copy plain data to block buffer + memmove(block + blockOffset, data, n); + } + auto status = EncryptBlock(blockIndex, block, (char*)scratch.data()); + if (!status.ok()) { + return status; + } + if (block != data) { + // Copy encrypted data back to `data`. + memmove(data, block + blockOffset, n); + } + dataSize -= n; + if (dataSize == 0) { + return rocksdb::Status::OK(); + } + data += n; + blockOffset = 0; + blockIndex++; + } +} + +// Decrypt one or more (partial) blocks of data at the file offset. +// Length of data is given in dataSize. +rocksdb::Status BlockAccessCipherStream::Decrypt(uint64_t fileOffset, char* data, size_t dataSize) { + // Calculate block index + auto blockSize = BlockSize(); + uint64_t blockIndex = fileOffset / blockSize; + size_t blockOffset = fileOffset % blockSize; + std::unique_ptr blockBuffer; + + std::string scratch; + AllocateScratch(scratch); + + // Decrypt individual blocks. + while (1) { + char* block = data; + size_t n = std::min(dataSize, blockSize - blockOffset); + if (n != blockSize) { + // We're not decrypting a full block. + // Copy data to blockBuffer + if (!blockBuffer.get()) { + // Allocate buffer + blockBuffer = std::unique_ptr(new char[blockSize]); + } + block = blockBuffer.get(); + // Copy encrypted data to block buffer + memmove(block + blockOffset, data, n); + } + auto status = DecryptBlock(blockIndex, block, (char*)scratch.data()); + if (!status.ok()) { + return status; + } + if (block != data) { + // Copy decrypted data back to `data`. + memmove(data, block + blockOffset, n); + } + dataSize -= n; + if (dataSize == 0) { + return rocksdb::Status::OK(); + } + data += n; + blockOffset = 0; + blockIndex++; + } +} + +} // namespace rocksdb_utils diff --git a/c-deps/libroach/rocksdbutils/env_encryption.h b/c-deps/libroach/rocksdbutils/env_encryption.h new file mode 100644 index 000000000000..5b59602bf413 --- /dev/null +++ b/c-deps/libroach/rocksdbutils/env_encryption.h @@ -0,0 +1,115 @@ +// Copyright 2018 The Cockroach Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +// implied. See the License for the specific language governing +// permissions and limitations under the License. +// +// +// +// Copyright (c) 2011-present, Facebook, Inc. All rights reserved. +// This source code is licensed under both the GPLv2 (found at +// https://www.gnu.org/licenses/old-licenses/gpl-2.0.en.html) +// and Apache 2.0 License (found in licenses/APL.txt in the root +// of this repository). + +#pragma once + +#include + +#include "../file_registry.h" +#include "../protos/storage/engine/enginepb/file_registry.pb.h" +#include "rocksdb/env.h" +#include "rocksdb/status.h" + +namespace enginepb = cockroach::storage::engine::enginepb; + +namespace rocksdb_utils { + +class CipherStreamCreator; + +// Returns an Env that encrypts data when stored on disk and decrypts data when +// read from disk. +// The env takes ownership of the CipherStreamCreator. +rocksdb::Env* NewEncryptedEnv(rocksdb::Env* base_env, cockroach::FileRegistry* file_registry, + CipherStreamCreator* creator); + +// BlockAccessCipherStream is the base class for any cipher stream that +// supports random access at block level (without requiring data from other blocks). +// E.g. CTR (Counter operation mode) supports this requirement. +class BlockAccessCipherStream { + public: + virtual ~BlockAccessCipherStream() {} + + // BlockSize returns the size of each block supported by this cipher stream. + virtual size_t BlockSize() = 0; + + // Encrypt one or more (partial) blocks of data at the file offset. + // Length of data is given in dataSize. + virtual rocksdb::Status Encrypt(uint64_t fileOffset, char* data, size_t dataSize); + + // Decrypt one or more (partial) blocks of data at the file offset. + // Length of data is given in dataSize. + virtual rocksdb::Status Decrypt(uint64_t fileOffset, char* data, size_t dataSize); + + protected: + // Allocate scratch space which is passed to EncryptBlock/DecryptBlock. + virtual void AllocateScratch(std::string&) = 0; + + // Encrypt a block of data at the given block index. + // Length of data is equal to BlockSize(); + virtual rocksdb::Status EncryptBlock(uint64_t blockIndex, char* data, char* scratch) = 0; + + // Decrypt a block of data at the given block index. + // Length of data is equal to BlockSize(); + virtual rocksdb::Status DecryptBlock(uint64_t blockIndex, char* data, char* scratch) = 0; +}; + +// BlockCipher +class BlockCipher { + public: + virtual ~BlockCipher() {} + + // BlockSize returns the size of each block supported by this cipher stream. + virtual size_t BlockSize() = 0; + + // Encrypt a block of data. + // Length of data is equal to BlockSize(). + virtual rocksdb::Status Encrypt(char* data) = 0; + + // Decrypt a block of data. + // Length of data is equal to BlockSize(). + virtual rocksdb::Status Decrypt(char* data) = 0; +}; + +// CipherStreamCreator is the abstract class used by EncryptedEnv. +// It performs two functions: +// * initializes encryption settings and create a cipher stream with them +// * creates a cipher stream given encryption settings +class CipherStreamCreator { + public: + virtual ~CipherStreamCreator() {} + // Create new encryption settings and a cipher stream using them. + // Assigns new objects to 'settings' and 'result'. + // If plaintext, settings will be nullptr. + virtual rocksdb::Status + InitSettingsAndCreateCipherStream(std::string* settings, + std::unique_ptr* result) = 0; + + // Create a cipher stream given encryption settings. + virtual rocksdb::Status + CreateCipherStreamFromSettings(const std::string& settings, + std::unique_ptr* result) = 0; + + // Return the EnvType for this stream creator. It should match files being operated on. + virtual enginepb::EnvType GetEnvType() = 0; +}; + +} // namespace rocksdb_utils diff --git a/licenses/LevelDB.AUTHORS b/licenses/LevelDB.AUTHORS new file mode 100644 index 000000000000..a451875f1a5e --- /dev/null +++ b/licenses/LevelDB.AUTHORS @@ -0,0 +1,12 @@ +Facebook Inc. +Facebook Engineering Team + +Google Inc. +# Initial version authors: +Jeffrey Dean +Sanjay Ghemawat + +# Partial list of contributors: +Kevin Regan +Johan Bilien +Matthew Von-Maszewski (Basho Technologies) diff --git a/licenses/LevelDB.txt b/licenses/LevelDB.txt new file mode 100644 index 000000000000..7108b0bfba7b --- /dev/null +++ b/licenses/LevelDB.txt @@ -0,0 +1,29 @@ +This contains code that is from LevelDB, and that code is under the following license: + +Copyright (c) 2011 The LevelDB Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pkg/ccl/cmdccl/enc_utils/.gitignore b/pkg/ccl/cmdccl/enc_utils/.gitignore new file mode 100644 index 000000000000..f0a7e8fd81a4 --- /dev/null +++ b/pkg/ccl/cmdccl/enc_utils/.gitignore @@ -0,0 +1 @@ +enc_utils diff --git a/pkg/ccl/cmdccl/enc_utils/README.md b/pkg/ccl/cmdccl/enc_utils/README.md new file mode 100644 index 000000000000..44f66415b291 --- /dev/null +++ b/pkg/ccl/cmdccl/enc_utils/README.md @@ -0,0 +1,25 @@ +This is a small utility to read encrypted rocksdb files. + + +Given a key and a cockroach node started with: +``` +$ openssl rand 48 > aes-128.key +$ cockroach start \ + --enterprise-encryption=path=cockroach-data,key=aes-128.key,old-key=plain +``` + +We can run this utility by pointing it to the key and data directory: +``` +$ go run main.go --db-dir cockroach-data --store-key aes-128.key +I180119 13:40:58.021531 1 ccl/cmdccl/enc_utils/main.go:125 store key: AES128_CTR len: 16 +I180119 13:40:58.021575 1 ccl/cmdccl/enc_utils/main.go:77 file registry version: Base +I180119 13:40:58.021594 1 ccl/cmdccl/enc_utils/main.go:92 file registry contains 20 entries +I180119 13:40:58.021612 1 ccl/cmdccl/enc_utils/main.go:198 decrypting COCKROACHDB_DATA_KEYS with AES128_CTR key 74bc2e29... +I180119 13:40:58.021627 1 ccl/cmdccl/enc_utils/main.go:142 data key registry contains 1 data key(s) +I180119 13:40:58.021642 1 ccl/cmdccl/enc_utils/main.go:198 decrypting CURRENT with AES128_CTR key 75b27bcc... +I180119 13:40:58.021650 1 ccl/cmdccl/enc_utils/main.go:151 current: MANIFEST-000008 +I180119 13:40:58.021733 1 ccl/cmdccl/enc_utils/main.go:198 decrypting OPTIONS-000005 with AES128_CTR key 75b27bcc... +I180119 13:40:58.021757 1 ccl/cmdccl/enc_utils/main.go:167 options file: OPTIONS-000005 starts with: # This is a RocksDB option file. +# +# For detailed file format spec, please refer to the example file +``` diff --git a/pkg/ccl/cmdccl/enc_utils/main.go b/pkg/ccl/cmdccl/enc_utils/main.go new file mode 100644 index 000000000000..6771751189db --- /dev/null +++ b/pkg/ccl/cmdccl/enc_utils/main.go @@ -0,0 +1,248 @@ +// Copyright 2017 The Cockroach Authors. +// +// Licensed as a CockroachDB Enterprise file under the Cockroach Community +// License (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt + +package main + +import ( + "context" + "crypto/aes" + "encoding/binary" + "encoding/hex" + "flag" + "fmt" + "io/ioutil" + "path/filepath" + + "github.com/cockroachdb/cockroach/pkg/ccl/storageccl/engineccl/enginepbccl" + "github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb" + "github.com/cockroachdb/cockroach/pkg/util/log" + "github.com/cockroachdb/cockroach/pkg/util/protoutil" + "github.com/pkg/errors" +) + +const fileRegistryPath = "COCKROACHDB_REGISTRY" +const keyRegistryPath = "COCKROACHDB_DATA_KEYS" +const currentPath = "CURRENT" +const optionsPathGlob = "OPTIONS-*" + +var dbDir = flag.String("db-dir", "", "path to the db directory") +var storeKeyPath = flag.String("store-key", "", "path to the active store key") + +type fileEntry struct { + envType enginepb.EnvType + settings enginepbccl.EncryptionSettings +} + +func (f fileEntry) String() string { + ret := fmt.Sprintf(" env type: %d, %s\n", + f.envType, f.settings.EncryptionType) + if f.settings.EncryptionType != enginepbccl.EncryptionType_Plaintext { + ret += fmt.Sprintf(" keyID: %s\n nonce: % x\n counter: %d\n", + f.settings.KeyId, + f.settings.Nonce, + f.settings.Counter) + } + return ret +} + +type keyEntry struct { + encryptionType enginepbccl.EncryptionType + rawKey []byte +} + +func (k keyEntry) String() string { + return fmt.Sprintf("%s len: %d", k.encryptionType, len(k.rawKey)) +} + +var fileRegistry = map[string]fileEntry{} +var keyRegistry = map[string]keyEntry{} + +func loadFileRegistry() { + data, err := ioutil.ReadFile(filepath.Join(*dbDir, fileRegistryPath)) + if err != nil { + log.Fatalf(context.Background(), "could not read %s: %v", fileRegistryPath, err) + } + + var reg enginepb.FileRegistry + if err := protoutil.Unmarshal(data, ®); err != nil { + log.Fatalf(context.Background(), "could not unmarshal %s: %v", fileRegistryPath, err) + } + + log.Infof(context.Background(), "file registry version: %s", reg.Version) + log.Infof(context.Background(), "file registry contains %d entries", len(reg.Files)) + for name, entry := range reg.Files { + var encSettings enginepbccl.EncryptionSettings + settings := entry.EncryptionSettings + if err := protoutil.Unmarshal(settings, &encSettings); err != nil { + log.Fatalf(context.Background(), "could not unmarshal encryption setting for %s: %v", name, err) + } + + fileRegistry[name] = fileEntry{entry.EnvType, encSettings} + + log.Infof(context.Background(), " %-30s level: %-8s type: %-12s keyID: %s", name, entry.EnvType, encSettings.EncryptionType, encSettings.KeyId[:8]) + } +} + +func loadStoreKey() { + if len(*storeKeyPath) == 0 || *storeKeyPath == "plain" { + log.Infof(context.Background(), "No store key specified") + return + } + + data, err := ioutil.ReadFile(*storeKeyPath) + if err != nil { + log.Fatalf(context.Background(), "could not read %s: %v", *storeKeyPath, err) + } + + var k keyEntry + switch len(data) { + case 48: + k.encryptionType = enginepbccl.EncryptionType_AES128_CTR + case 56: + k.encryptionType = enginepbccl.EncryptionType_AES192_CTR + case 64: + k.encryptionType = enginepbccl.EncryptionType_AES256_CTR + default: + log.Fatalf(context.Background(), "wrong key length %d, want 32 bytes + AES length", len(data)) + } + + // Hexadecimal representation of the first 32 bytes. + id := hex.EncodeToString(data[0:32]) + // Raw key is the rest. + k.rawKey = data[32:] + + keyRegistry[id] = k + + log.Infof(context.Background(), "store key: %s", k) +} + +func loadKeyRegistry() { + data, err := readFile(keyRegistryPath) + if err != nil { + log.Fatalf(context.Background(), "could not read %s: %v", keyRegistryPath, err) + } + + var reg enginepbccl.DataKeysRegistry + if err := protoutil.Unmarshal(data, ®); err != nil { + log.Fatalf(context.Background(), "could not unmarshal %s: %v", keyRegistryPath, err) + } + + log.Infof(context.Background(), "key registry contains %d store keys(s) and %d data key(s)", + len(reg.StoreKeys), len(reg.DataKeys)) + for _, e := range reg.StoreKeys { + log.Infof(context.Background(), " store key: type: %-12s %v", e.EncryptionType, e) + } + for _, e := range reg.DataKeys { + log.Infof(context.Background(), " data key: type: %-12s %v", e.Info.EncryptionType, e.Info) + } + for k, e := range reg.DataKeys { + keyRegistry[k] = keyEntry{e.Info.EncryptionType, e.Key} + } +} + +func loadCurrent() { + data, err := readFile(currentPath) + if err != nil { + log.Fatalf(context.Background(), "could not read %s: %v", currentPath, err) + } + + log.Infof(context.Background(), "current: %s", string(data)) +} + +func loadOptions() { + absGlob := filepath.Join(*dbDir, optionsPathGlob) + paths, err := filepath.Glob(absGlob) + if err != nil { + log.Fatalf(context.Background(), "problem finding files matching %s: %v", absGlob, err) + } + + for _, f := range paths { + fname := filepath.Base(f) + data, err := readFile(fname) + if err != nil { + log.Fatalf(context.Background(), "could not read %s: %v", fname, err) + } + log.Infof(context.Background(), "options file: %s starts with: %s", fname, string(data[:100])) + } +} + +func readFile(filename string) ([]byte, error) { + if len(filename) == 0 { + return nil, errors.Errorf("filename is empty") + } + + absPath := filename + if filename[0] != '/' { + absPath = filepath.Join(*dbDir, filename) + } + + data, err := ioutil.ReadFile(absPath) + if err != nil { + return nil, errors.Errorf("could not read %s: %v", absPath, err) + } + + reg, ok := fileRegistry[filename] + if !ok || reg.settings.EncryptionType == enginepbccl.EncryptionType_Plaintext { + // Plaintext: do nothing. + log.Infof(context.Background(), "reading plaintext %s", absPath) + return data, nil + } + + // Encrypted: find the key. + key, ok := keyRegistry[reg.settings.KeyId] + if !ok { + return nil, errors.Errorf("could not find key %s for file %s", reg.settings.KeyId, absPath) + } + log.Infof(context.Background(), "decrypting %s with %s key %s...", filename, reg.settings.EncryptionType, reg.settings.KeyId[:8]) + + cipher, err := aes.NewCipher(key.rawKey) + if err != nil { + return nil, errors.Errorf("could not build AES cipher for file %s: %v", absPath, err) + } + + size := len(data) + counter := reg.settings.Counter + nonce := reg.settings.Nonce + if len(nonce) != 12 { + log.Fatalf(context.Background(), "nonce has wrong length: %d, expected 12", len(nonce)) + } + + iv := make([]byte, aes.BlockSize) + for offset := 0; offset < size; offset += aes.BlockSize { + // Put nonce at beginning of IV. + copy(iv, nonce) + // Write counter to end of IV in network byte order. + binary.BigEndian.PutUint32(iv[12:], counter) + // Increment counter for next block. + counter++ + + // Encrypt IV (AES CTR mode is always encrypt). + cipher.Encrypt(iv, iv) + + // XOR data with decrypted IV. We may have a partial block at the end of 'data'. + for i := 0; i < aes.BlockSize; i++ { + pos := offset + i + if pos >= size { + // Partial block. + break + } + data[pos] = data[pos] ^ iv[i] + } + } + + return data, nil +} + +func main() { + flag.Parse() + loadStoreKey() + loadFileRegistry() + loadKeyRegistry() + loadCurrent() + loadOptions() +} diff --git a/pkg/ccl/storageccl/engineccl/enginepbccl/key_registry.pb.go b/pkg/ccl/storageccl/engineccl/enginepbccl/key_registry.pb.go index 49ed4b43bd38..8134a3292f8d 100644 --- a/pkg/ccl/storageccl/engineccl/enginepbccl/key_registry.pb.go +++ b/pkg/ccl/storageccl/engineccl/enginepbccl/key_registry.pb.go @@ -11,6 +11,7 @@ DataKeysRegistry KeyInfo SecretKey + EncryptionSettings */ package enginepbccl @@ -62,6 +63,9 @@ func (x EncryptionType) String() string { } func (EncryptionType) EnumDescriptor() ([]byte, []int) { return fileDescriptorKeyRegistry, []int{0} } +// DataKeysRegistry contains all data keys (including the raw key) as well +// as store key information (excluding raw key). +// This is written to disk. type DataKeysRegistry struct { // Map of key_id to KeyInfo (raw key is not included). StoreKeys map[string]*KeyInfo `protobuf:"bytes,1,rep,name=store_keys,json=storeKeys" json:"store_keys,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` @@ -115,10 +119,28 @@ func (m *SecretKey) String() string { return proto.CompactTextString( func (*SecretKey) ProtoMessage() {} func (*SecretKey) Descriptor() ([]byte, []int) { return fileDescriptorKeyRegistry, []int{2} } +// EncryptionSettings describes the encryption settings for a file. +// This is stored as a protobuf.Any inside the FileEntry as described in: +// pkg/storage/engine/enginepb/file_registry.proto +type EncryptionSettings struct { + EncryptionType EncryptionType `protobuf:"varint,1,opt,name=encryption_type,json=encryptionType,proto3,enum=cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionType" json:"encryption_type,omitempty"` + // Fields for AES-CTR. Empty when encryption_type = Plaintext. + KeyId string `protobuf:"bytes,2,opt,name=key_id,json=keyId,proto3" json:"key_id,omitempty"` + // len(nonce) + sizeof(counter) should add up to AES_Blocksize (128 bits). + Nonce []byte `protobuf:"bytes,3,opt,name=nonce,proto3" json:"nonce,omitempty"` + Counter uint32 `protobuf:"varint,4,opt,name=counter,proto3" json:"counter,omitempty"` +} + +func (m *EncryptionSettings) Reset() { *m = EncryptionSettings{} } +func (m *EncryptionSettings) String() string { return proto.CompactTextString(m) } +func (*EncryptionSettings) ProtoMessage() {} +func (*EncryptionSettings) Descriptor() ([]byte, []int) { return fileDescriptorKeyRegistry, []int{3} } + func init() { proto.RegisterType((*DataKeysRegistry)(nil), "cockroach.ccl.storageccl.engineccl.enginepbccl.DataKeysRegistry") proto.RegisterType((*KeyInfo)(nil), "cockroach.ccl.storageccl.engineccl.enginepbccl.KeyInfo") proto.RegisterType((*SecretKey)(nil), "cockroach.ccl.storageccl.engineccl.enginepbccl.SecretKey") + proto.RegisterType((*EncryptionSettings)(nil), "cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionSettings") proto.RegisterEnum("cockroach.ccl.storageccl.engineccl.enginepbccl.EncryptionType", EncryptionType_name, EncryptionType_value) } func (m *DataKeysRegistry) Marshal() (dAtA []byte, err error) { @@ -307,6 +329,46 @@ func (m *SecretKey) MarshalTo(dAtA []byte) (int, error) { return i, nil } +func (m *EncryptionSettings) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalTo(dAtA) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *EncryptionSettings) MarshalTo(dAtA []byte) (int, error) { + var i int + _ = i + var l int + _ = l + if m.EncryptionType != 0 { + dAtA[i] = 0x8 + i++ + i = encodeVarintKeyRegistry(dAtA, i, uint64(m.EncryptionType)) + } + if len(m.KeyId) > 0 { + dAtA[i] = 0x12 + i++ + i = encodeVarintKeyRegistry(dAtA, i, uint64(len(m.KeyId))) + i += copy(dAtA[i:], m.KeyId) + } + if len(m.Nonce) > 0 { + dAtA[i] = 0x1a + i++ + i = encodeVarintKeyRegistry(dAtA, i, uint64(len(m.Nonce))) + i += copy(dAtA[i:], m.Nonce) + } + if m.Counter != 0 { + dAtA[i] = 0x20 + i++ + i = encodeVarintKeyRegistry(dAtA, i, uint64(m.Counter)) + } + return i, nil +} + func encodeVarintKeyRegistry(dAtA []byte, offset int, v uint64) int { for v >= 1<<7 { dAtA[offset] = uint8(v&0x7f | 0x80) @@ -397,6 +459,26 @@ func (m *SecretKey) Size() (n int) { return n } +func (m *EncryptionSettings) Size() (n int) { + var l int + _ = l + if m.EncryptionType != 0 { + n += 1 + sovKeyRegistry(uint64(m.EncryptionType)) + } + l = len(m.KeyId) + if l > 0 { + n += 1 + l + sovKeyRegistry(uint64(l)) + } + l = len(m.Nonce) + if l > 0 { + n += 1 + l + sovKeyRegistry(uint64(l)) + } + if m.Counter != 0 { + n += 1 + sovKeyRegistry(uint64(m.Counter)) + } + return n +} + func sovKeyRegistry(x uint64) (n int) { for { n++ @@ -1073,6 +1155,154 @@ func (m *SecretKey) Unmarshal(dAtA []byte) error { } return nil } +func (m *EncryptionSettings) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowKeyRegistry + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: EncryptionSettings: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: EncryptionSettings: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EncryptionType", wireType) + } + m.EncryptionType = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowKeyRegistry + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.EncryptionType |= (EncryptionType(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field KeyId", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowKeyRegistry + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthKeyRegistry + } + postIndex := iNdEx + intStringLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.KeyId = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Nonce", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowKeyRegistry + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthKeyRegistry + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Nonce = append(m.Nonce[:0], dAtA[iNdEx:postIndex]...) + if m.Nonce == nil { + m.Nonce = []byte{} + } + iNdEx = postIndex + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Counter", wireType) + } + m.Counter = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowKeyRegistry + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Counter |= (uint32(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipKeyRegistry(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthKeyRegistry + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func skipKeyRegistry(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 @@ -1183,39 +1413,42 @@ func init() { } var fileDescriptorKeyRegistry = []byte{ - // 543 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x94, 0xcb, 0x6e, 0xd3, 0x4c, - 0x14, 0xc7, 0xeb, 0xdc, 0xbe, 0xfa, 0xa4, 0x71, 0xa3, 0xd1, 0x07, 0x8a, 0xb2, 0x30, 0x51, 0x10, - 0x28, 0x62, 0xe1, 0x0a, 0x23, 0x68, 0xcb, 0x02, 0x89, 0x4b, 0x16, 0x55, 0x04, 0x41, 0x4e, 0x56, - 0x6c, 0xac, 0xe9, 0xf8, 0xd4, 0x98, 0xa4, 0x1e, 0xcb, 0x9e, 0xa4, 0x9d, 0x67, 0x60, 0xc3, 0xa3, - 0xf0, 0x18, 0x5d, 0xb2, 0x64, 0x09, 0xe1, 0x45, 0xd0, 0x8c, 0xed, 0x90, 0x20, 0x36, 0x51, 0x77, - 0xe7, 0x7f, 0x74, 0xe6, 0xff, 0x3b, 0x17, 0xd9, 0x70, 0xcc, 0xd8, 0xfc, 0x28, 0x13, 0x3c, 0xa5, - 0x21, 0xaa, 0x10, 0xe3, 0x30, 0x8a, 0x37, 0xa2, 0xe4, 0x5c, 0xc5, 0x33, 0x94, 0x7e, 0x8a, 0x61, - 0x94, 0x89, 0x54, 0x3a, 0x49, 0xca, 0x05, 0x27, 0x0e, 0xe3, 0x6c, 0x96, 0x72, 0xca, 0x3e, 0x3a, - 0x8c, 0xcd, 0x9d, 0x3f, 0x16, 0xce, 0xda, 0xc2, 0xd9, 0xb0, 0xe8, 0xfe, 0x1f, 0xf2, 0x90, 0xeb, - 0xa7, 0x47, 0x2a, 0xca, 0x5d, 0xfa, 0x5f, 0x6b, 0xd0, 0x7e, 0x43, 0x05, 0x1d, 0xa1, 0xcc, 0xbc, - 0x02, 0x40, 0x62, 0x00, 0x65, 0x87, 0xfe, 0x0c, 0x65, 0xd6, 0x31, 0x7a, 0xd5, 0x41, 0xd3, 0x1d, - 0xef, 0xc8, 0x73, 0xfe, 0x76, 0x75, 0x26, 0xca, 0x52, 0x65, 0x86, 0xb1, 0x48, 0xa5, 0x67, 0x66, - 0xa5, 0x26, 0x33, 0x30, 0x03, 0x2a, 0x68, 0x8e, 0xab, 0x68, 0xdc, 0xbb, 0x5b, 0xe3, 0xca, 0x44, - 0x4e, 0xdb, 0x0f, 0x0a, 0x49, 0x06, 0xd0, 0xa6, 0x4c, 0x44, 0x4b, 0xf4, 0xd7, 0x33, 0x76, 0xaa, - 0x3d, 0x63, 0x60, 0x7a, 0x56, 0x9e, 0x2f, 0xfb, 0x24, 0x0f, 0xe1, 0xb0, 0xa8, 0x2c, 0xbb, 0xeb, - 0xd4, 0x74, 0x61, 0x2b, 0x4f, 0x17, 0x84, 0xee, 0x02, 0xac, 0xed, 0xd9, 0x48, 0x1b, 0xaa, 0xaa, - 0xda, 0xd0, 0xd5, 0x2a, 0x24, 0x6f, 0xa1, 0xbe, 0xa4, 0xf3, 0x05, 0x76, 0x2a, 0x3d, 0x63, 0xd0, - 0x74, 0x8f, 0x77, 0x1d, 0x6f, 0x84, 0xf2, 0x2c, 0xbe, 0xe0, 0x5e, 0xee, 0xf2, 0xbc, 0x72, 0x62, - 0x74, 0x97, 0xd0, 0xda, 0x9a, 0xf1, 0x1f, 0xd4, 0xf1, 0x36, 0xf5, 0x74, 0x57, 0xea, 0x04, 0x59, - 0x8a, 0x62, 0x84, 0x72, 0x83, 0xdb, 0xff, 0x5c, 0x81, 0xff, 0x8a, 0x76, 0x48, 0x08, 0x87, 0x18, - 0xb3, 0x54, 0x26, 0x22, 0xe2, 0xb1, 0x2f, 0x64, 0x82, 0x1a, 0x6f, 0xb9, 0x2f, 0x76, 0x45, 0x0d, - 0xd7, 0x36, 0x53, 0x99, 0xa0, 0x67, 0xe1, 0x96, 0x26, 0x77, 0xa0, 0xa1, 0xbe, 0x81, 0x28, 0xd0, - 0xa3, 0x98, 0x5e, 0x7d, 0x86, 0xf2, 0x2c, 0x20, 0xf7, 0xa1, 0xc5, 0x52, 0xa4, 0x39, 0x3d, 0xba, - 0x44, 0x7d, 0xc9, 0xaa, 0x77, 0x50, 0x26, 0xa7, 0xd1, 0x25, 0x92, 0xbb, 0xd0, 0xc8, 0xf8, 0x22, - 0x65, 0x58, 0x9c, 0xaf, 0x50, 0xe4, 0x1e, 0x34, 0xaf, 0x68, 0xe6, 0xe3, 0x75, 0xc2, 0x33, 0x0c, - 0x3a, 0xf5, 0x9e, 0x31, 0xd8, 0xf7, 0xe0, 0x8a, 0x66, 0xc3, 0x3c, 0x43, 0xfa, 0xd0, 0x4a, 0x68, - 0x8a, 0xb1, 0xf0, 0x0b, 0x76, 0x43, 0xbf, 0x6f, 0xe6, 0x49, 0xb5, 0x83, 0xa0, 0xff, 0x09, 0xcc, - 0xf5, 0x96, 0xc8, 0x08, 0x6a, 0x51, 0x7c, 0xc1, 0xf5, 0x0e, 0x6e, 0x71, 0x64, 0x6d, 0x52, 0x9e, - 0x53, 0xcd, 0x7b, 0xa0, 0xcf, 0xf9, 0x68, 0x0c, 0xd6, 0xf6, 0x9a, 0x48, 0x0b, 0xcc, 0xf7, 0x73, - 0x1a, 0xc5, 0x02, 0xaf, 0x45, 0x7b, 0x8f, 0x58, 0x00, 0x2f, 0x87, 0x93, 0xc7, 0xee, 0x89, 0xff, - 0x7a, 0xea, 0xb5, 0x8d, 0x52, 0x9f, 0xba, 0x5a, 0x57, 0x0a, 0xed, 0x3e, 0x7d, 0xa6, 0x75, 0xf5, - 0xd5, 0x83, 0x9b, 0x9f, 0xf6, 0xde, 0xcd, 0xca, 0x36, 0xbe, 0xad, 0x6c, 0xe3, 0xfb, 0xca, 0x36, - 0x7e, 0xac, 0x6c, 0xe3, 0xcb, 0x2f, 0x7b, 0xef, 0x43, 0x73, 0xa3, 0xaf, 0xf3, 0x86, 0xfe, 0x57, - 0x3c, 0xf9, 0x1d, 0x00, 0x00, 0xff, 0xff, 0x1c, 0x48, 0xfa, 0x96, 0xac, 0x04, 0x00, 0x00, + // 583 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x94, 0xcf, 0x6e, 0xd3, 0x40, + 0x10, 0xc6, 0xbb, 0x49, 0x93, 0x36, 0x93, 0x26, 0x8d, 0x56, 0x05, 0x59, 0x3d, 0x98, 0x28, 0x08, + 0x14, 0x71, 0x70, 0x85, 0x11, 0xb4, 0xe5, 0x80, 0xc4, 0x9f, 0x1c, 0xaa, 0x08, 0x8a, 0x9c, 0x9e, + 0xb8, 0x58, 0xdb, 0xf5, 0xd4, 0x98, 0xa4, 0xbb, 0xd6, 0x7a, 0xd3, 0xd6, 0xcf, 0xc0, 0x85, 0x47, + 0xe1, 0x21, 0x38, 0xf4, 0xc8, 0x91, 0x23, 0x84, 0x17, 0x41, 0x5e, 0xdb, 0x69, 0x82, 0xb8, 0x44, + 0x3d, 0x70, 0x9b, 0x6f, 0x34, 0xfe, 0x7e, 0xfb, 0xcd, 0x28, 0x81, 0x7d, 0xce, 0x27, 0x7b, 0x89, + 0x96, 0x8a, 0x85, 0x98, 0x95, 0x28, 0xc2, 0x48, 0x2c, 0x54, 0xf1, 0x69, 0x56, 0x8f, 0x31, 0xf5, + 0x15, 0x86, 0x51, 0xa2, 0x55, 0xea, 0xc4, 0x4a, 0x6a, 0x49, 0x1d, 0x2e, 0xf9, 0x58, 0x49, 0xc6, + 0x3f, 0x3a, 0x9c, 0x4f, 0x9c, 0x1b, 0x0b, 0x67, 0x6e, 0xe1, 0x2c, 0x58, 0xec, 0xee, 0x84, 0x32, + 0x94, 0xe6, 0xd3, 0xbd, 0xac, 0xca, 0x5d, 0x7a, 0x5f, 0xd7, 0xa1, 0xf3, 0x86, 0x69, 0x36, 0xc4, + 0x34, 0xf1, 0x0a, 0x00, 0x15, 0x00, 0x99, 0x1d, 0xfa, 0x63, 0x4c, 0x13, 0x8b, 0x74, 0xab, 0xfd, + 0xa6, 0x7b, 0xbc, 0x22, 0xcf, 0xf9, 0xdb, 0xd5, 0x19, 0x65, 0x96, 0x59, 0x67, 0x20, 0xb4, 0x4a, + 0xbd, 0x46, 0x52, 0x6a, 0x3a, 0x86, 0x46, 0xc0, 0x34, 0xcb, 0x71, 0x15, 0x83, 0x7b, 0x77, 0x6b, + 0x5c, 0xd9, 0xc8, 0x69, 0x9b, 0x41, 0x21, 0x69, 0x1f, 0x3a, 0x8c, 0xeb, 0xe8, 0x02, 0xfd, 0x79, + 0x46, 0xab, 0xda, 0x25, 0xfd, 0x86, 0xd7, 0xce, 0xfb, 0xe5, 0x3b, 0xe9, 0x43, 0xd8, 0x2e, 0x26, + 0xcb, 0xd7, 0x59, 0xeb, 0x66, 0xb0, 0x95, 0xb7, 0x0b, 0xc2, 0xee, 0x14, 0xda, 0xcb, 0xd9, 0x68, + 0x07, 0xaa, 0xd9, 0x34, 0x31, 0xd3, 0x59, 0x49, 0xdf, 0x42, 0xed, 0x82, 0x4d, 0xa6, 0x68, 0x55, + 0xba, 0xa4, 0xdf, 0x74, 0xf7, 0x57, 0x8d, 0x37, 0xc4, 0xf4, 0x48, 0x9c, 0x49, 0x2f, 0x77, 0x79, + 0x5e, 0x39, 0x20, 0xbb, 0x17, 0xd0, 0x5a, 0xca, 0xf8, 0x0f, 0xea, 0xf1, 0x32, 0xf5, 0x70, 0x55, + 0xea, 0x08, 0xb9, 0x42, 0x3d, 0xc4, 0x74, 0x81, 0xdb, 0xfb, 0x5c, 0x81, 0x8d, 0xe2, 0x39, 0x34, + 0x84, 0x6d, 0x14, 0x5c, 0xa5, 0xb1, 0x8e, 0xa4, 0xf0, 0x75, 0x1a, 0xa3, 0xc1, 0xb7, 0xdd, 0x17, + 0xab, 0xa2, 0x06, 0x73, 0x9b, 0x93, 0x34, 0x46, 0xaf, 0x8d, 0x4b, 0x9a, 0xde, 0x81, 0x7a, 0xf6, + 0x1b, 0x88, 0x02, 0x13, 0xa5, 0xe1, 0xd5, 0xc6, 0x98, 0x1e, 0x05, 0xf4, 0x3e, 0xb4, 0xb8, 0x42, + 0x96, 0xd3, 0xa3, 0x73, 0x34, 0x97, 0xac, 0x7a, 0x5b, 0x65, 0xf3, 0x24, 0x3a, 0x47, 0x7a, 0x17, + 0xea, 0x89, 0x9c, 0x2a, 0x8e, 0xc5, 0xf9, 0x0a, 0x45, 0xef, 0x41, 0xf3, 0x92, 0x25, 0x3e, 0x5e, + 0xc5, 0x32, 0xc1, 0xc0, 0xaa, 0x75, 0x49, 0x7f, 0xd3, 0x83, 0x4b, 0x96, 0x0c, 0xf2, 0x0e, 0xed, + 0x41, 0x2b, 0x66, 0x0a, 0x85, 0xf6, 0x0b, 0x76, 0xdd, 0x7c, 0xdf, 0xcc, 0x9b, 0xd9, 0x0e, 0x82, + 0xde, 0x27, 0x68, 0xcc, 0xb7, 0x44, 0x87, 0xb0, 0x1e, 0x89, 0x33, 0x69, 0x76, 0x70, 0x8b, 0x23, + 0x1b, 0x93, 0xf2, 0x9c, 0x59, 0xde, 0x2d, 0x73, 0xce, 0xde, 0x37, 0x02, 0xf4, 0x66, 0x4f, 0x23, + 0xd4, 0x3a, 0x12, 0x61, 0xf2, 0xdf, 0x8f, 0xb0, 0x03, 0x35, 0x21, 0x05, 0xcf, 0x97, 0xbf, 0xe5, + 0xe5, 0x82, 0x5a, 0xb0, 0xc1, 0xe5, 0x54, 0x68, 0x54, 0x66, 0xed, 0x2d, 0xaf, 0x94, 0x8f, 0x8e, + 0xa1, 0xbd, 0x0c, 0xa2, 0x2d, 0x68, 0xbc, 0x9f, 0xb0, 0x48, 0x68, 0xbc, 0xd2, 0x9d, 0x35, 0xda, + 0x06, 0x78, 0x39, 0x18, 0x3d, 0x76, 0x0f, 0xfc, 0xd7, 0x27, 0x5e, 0x87, 0x94, 0xfa, 0xd0, 0x35, + 0xba, 0x52, 0x68, 0xf7, 0xe9, 0x33, 0xa3, 0xab, 0xaf, 0x1e, 0x5c, 0xff, 0xb2, 0xd7, 0xae, 0x67, + 0x36, 0xf9, 0x3e, 0xb3, 0xc9, 0x8f, 0x99, 0x4d, 0x7e, 0xce, 0x6c, 0xf2, 0xe5, 0xb7, 0xbd, 0xf6, + 0xa1, 0xb9, 0x90, 0xee, 0xb4, 0x6e, 0xfe, 0xf2, 0x9e, 0xfc, 0x09, 0x00, 0x00, 0xff, 0xff, 0x87, + 0xce, 0xa5, 0xe2, 0x73, 0x05, 0x00, 0x00, } diff --git a/pkg/ccl/storageccl/engineccl/enginepbccl/key_registry.proto b/pkg/ccl/storageccl/engineccl/enginepbccl/key_registry.proto index 4290a0087e03..27272ff2350e 100644 --- a/pkg/ccl/storageccl/engineccl/enginepbccl/key_registry.proto +++ b/pkg/ccl/storageccl/engineccl/enginepbccl/key_registry.proto @@ -21,6 +21,9 @@ enum EncryptionType { AES256_CTR = 3; } +// DataKeysRegistry contains all data keys (including the raw key) as well +// as store key information (excluding raw key). +// This is written to disk. message DataKeysRegistry { // Map of key_id to KeyInfo (raw key is not included). map store_keys = 1; @@ -59,3 +62,16 @@ message SecretKey { // The raw key. bytes key = 2; } + +// EncryptionSettings describes the encryption settings for a file. +// This is stored as a protobuf.Any inside the FileEntry as described in: +// pkg/storage/engine/enginepb/file_registry.proto +message EncryptionSettings { + EncryptionType encryption_type = 1; + + // Fields for AES-CTR. Empty when encryption_type = Plaintext. + string key_id = 2; + // len(nonce) + sizeof(counter) should add up to AES_Blocksize (128 bits). + bytes nonce = 3; // 12 bytes + uint32 counter = 4; // 4 bytes +} diff --git a/pkg/sql/logictest/testdata/logic_test/order_by b/pkg/sql/logictest/testdata/logic_test/order_by index af564869093f..bc7a66dea031 100644 --- a/pkg/sql/logictest/testdata/logic_test/order_by +++ b/pkg/sql/logictest/testdata/logic_test/order_by @@ -876,3 +876,15 @@ limit · · (block_id, writer_id, · table blocks@primary · · · spans ALL · · · limit 1 · · + +# Regression test for #25197: make sure that order by with an offset works in +# the top-k sorter. + +statement ok +CREATE TABLE t25197 (id BIGSERIAL PRIMARY KEY, d TEXT) + +statement ok +INSERT INTO t25197 (d) VALUES ('test1') + +query TT +SELECT * FROM t25197 ORDER BY d ASC OFFSET 1 LIMIT 10 diff --git a/pkg/storage/engine/enginepb/file_registry.pb.go b/pkg/storage/engine/enginepb/file_registry.pb.go index dceb031d43da..c489f450d63a 100644 --- a/pkg/storage/engine/enginepb/file_registry.pb.go +++ b/pkg/storage/engine/enginepb/file_registry.pb.go @@ -11,7 +11,7 @@ storage/engine/enginepb/rocksdb.proto It has these top-level messages: - Registry + FileRegistry FileEntry MVCCMetadata MVCCStats @@ -28,6 +28,8 @@ import proto "github.com/gogo/protobuf/proto" import fmt "fmt" import math "math" +import sortkeys "github.com/gogo/protobuf/sortkeys" + import io "io" // Reference imports to suppress errors if they are not otherwise used. @@ -60,26 +62,61 @@ func (x RegistryVersion) String() string { } func (RegistryVersion) EnumDescriptor() ([]byte, []int) { return fileDescriptorFileRegistry, []int{0} } +// EnvType determines which rocksdb::Env is used and for what purpose. +type EnvType int32 + +const ( + // The default Env when no encryption is used. + // File using Plaintext are not recorded in the file registry. + EnvType_Plaintext EnvType = 0 + // The Env using store-level keys. + // Used only to read/write the data key registry. + EnvType_Store EnvType = 1 + // The Env using data-level keys. + // Used as the default rocksdb Env when encryption is enabled. + EnvType_Data EnvType = 2 +) + +var EnvType_name = map[int32]string{ + 0: "Plaintext", + 1: "Store", + 2: "Data", +} +var EnvType_value = map[string]int32{ + "Plaintext": 0, + "Store": 1, + "Data": 2, +} + +func (x EnvType) String() string { + return proto.EnumName(EnvType_name, int32(x)) +} +func (EnvType) EnumDescriptor() ([]byte, []int) { return fileDescriptorFileRegistry, []int{1} } + // Registry describes how a files are handled. This includes the // rockdb::Env responsible for each file as well as opaque env details. -// TODO(mberhault): this is still in flux. -type Registry struct { +type FileRegistry struct { // version is currently always Base. Version RegistryVersion `protobuf:"varint,1,opt,name=version,proto3,enum=cockroach.storage.engine.enginepb.RegistryVersion" json:"version,omitempty"` - Files []*FileEntry `protobuf:"bytes,2,rep,name=files" json:"files,omitempty"` + // Map of filename -> FileEntry. + // Filename is relative to the rocksdb dir if the file is inside it. + // Otherwise it is an absolute path. + // TODO(mberhault): figure out if we need anything special for Windows. + Files map[string]*FileEntry `protobuf:"bytes,2,rep,name=files" json:"files,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value"` } -func (m *Registry) Reset() { *m = Registry{} } -func (m *Registry) String() string { return proto.CompactTextString(m) } -func (*Registry) ProtoMessage() {} -func (*Registry) Descriptor() ([]byte, []int) { return fileDescriptorFileRegistry, []int{0} } +func (m *FileRegistry) Reset() { *m = FileRegistry{} } +func (m *FileRegistry) String() string { return proto.CompactTextString(m) } +func (*FileRegistry) ProtoMessage() {} +func (*FileRegistry) Descriptor() ([]byte, []int) { return fileDescriptorFileRegistry, []int{0} } type FileEntry struct { - // File path relative to the DB directory. - // TODO(mberhault): figure out if we need anything special for Windows. - Filename string `protobuf:"bytes,1,opt,name=Filename,proto3" json:"Filename,omitempty"` - // Env level identifies which rocksdb::Env is responsible for this file. - EnvLevel int32 `protobuf:"varint,2,opt,name=env_level,json=envLevel,proto3" json:"env_level,omitempty"` + // Env type identifies which rocksdb::Env is responsible for this file. + EnvType EnvType `protobuf:"varint,1,opt,name=env_type,json=envType,proto3,enum=cockroach.storage.engine.enginepb.EnvType" json:"env_type,omitempty"` + // Env-specific fields for non-0 env. These are known by CCL code only. + // This is a serialized protobuf. We cannot use protobuf.Any since we use + // MessageLite in C++. + EncryptionSettings []byte `protobuf:"bytes,2,opt,name=encryption_settings,json=encryptionSettings,proto3" json:"encryption_settings,omitempty"` } func (m *FileEntry) Reset() { *m = FileEntry{} } @@ -88,11 +125,12 @@ func (*FileEntry) ProtoMessage() {} func (*FileEntry) Descriptor() ([]byte, []int) { return fileDescriptorFileRegistry, []int{1} } func init() { - proto.RegisterType((*Registry)(nil), "cockroach.storage.engine.enginepb.Registry") + proto.RegisterType((*FileRegistry)(nil), "cockroach.storage.engine.enginepb.FileRegistry") proto.RegisterType((*FileEntry)(nil), "cockroach.storage.engine.enginepb.FileEntry") proto.RegisterEnum("cockroach.storage.engine.enginepb.RegistryVersion", RegistryVersion_name, RegistryVersion_value) + proto.RegisterEnum("cockroach.storage.engine.enginepb.EnvType", EnvType_name, EnvType_value) } -func (m *Registry) Marshal() (dAtA []byte, err error) { +func (m *FileRegistry) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalTo(dAtA) @@ -102,7 +140,7 @@ func (m *Registry) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Registry) MarshalTo(dAtA []byte) (int, error) { +func (m *FileRegistry) MarshalTo(dAtA []byte) (int, error) { var i int _ = i var l int @@ -113,15 +151,36 @@ func (m *Registry) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintFileRegistry(dAtA, i, uint64(m.Version)) } if len(m.Files) > 0 { - for _, msg := range m.Files { + keysForFiles := make([]string, 0, len(m.Files)) + for k := range m.Files { + keysForFiles = append(keysForFiles, string(k)) + } + sortkeys.Strings(keysForFiles) + for _, k := range keysForFiles { dAtA[i] = 0x12 i++ - i = encodeVarintFileRegistry(dAtA, i, uint64(msg.Size())) - n, err := msg.MarshalTo(dAtA[i:]) - if err != nil { - return 0, err + v := m.Files[string(k)] + msgSize := 0 + if v != nil { + msgSize = v.Size() + msgSize += 1 + sovFileRegistry(uint64(msgSize)) + } + mapSize := 1 + len(k) + sovFileRegistry(uint64(len(k))) + msgSize + i = encodeVarintFileRegistry(dAtA, i, uint64(mapSize)) + dAtA[i] = 0xa + i++ + i = encodeVarintFileRegistry(dAtA, i, uint64(len(k))) + i += copy(dAtA[i:], k) + if v != nil { + dAtA[i] = 0x12 + i++ + i = encodeVarintFileRegistry(dAtA, i, uint64(v.Size())) + n1, err := v.MarshalTo(dAtA[i:]) + if err != nil { + return 0, err + } + i += n1 } - i += n } } return i, nil @@ -142,16 +201,16 @@ func (m *FileEntry) MarshalTo(dAtA []byte) (int, error) { _ = i var l int _ = l - if len(m.Filename) > 0 { - dAtA[i] = 0xa + if m.EnvType != 0 { + dAtA[i] = 0x8 i++ - i = encodeVarintFileRegistry(dAtA, i, uint64(len(m.Filename))) - i += copy(dAtA[i:], m.Filename) + i = encodeVarintFileRegistry(dAtA, i, uint64(m.EnvType)) } - if m.EnvLevel != 0 { - dAtA[i] = 0x10 + if len(m.EncryptionSettings) > 0 { + dAtA[i] = 0x12 i++ - i = encodeVarintFileRegistry(dAtA, i, uint64(m.EnvLevel)) + i = encodeVarintFileRegistry(dAtA, i, uint64(len(m.EncryptionSettings))) + i += copy(dAtA[i:], m.EncryptionSettings) } return i, nil } @@ -165,16 +224,23 @@ func encodeVarintFileRegistry(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return offset + 1 } -func (m *Registry) Size() (n int) { +func (m *FileRegistry) Size() (n int) { var l int _ = l if m.Version != 0 { n += 1 + sovFileRegistry(uint64(m.Version)) } if len(m.Files) > 0 { - for _, e := range m.Files { - l = e.Size() - n += 1 + l + sovFileRegistry(uint64(l)) + for k, v := range m.Files { + _ = k + _ = v + l = 0 + if v != nil { + l = v.Size() + l += 1 + sovFileRegistry(uint64(l)) + } + mapEntrySize := 1 + len(k) + sovFileRegistry(uint64(len(k))) + l + n += mapEntrySize + 1 + sovFileRegistry(uint64(mapEntrySize)) } } return n @@ -183,13 +249,13 @@ func (m *Registry) Size() (n int) { func (m *FileEntry) Size() (n int) { var l int _ = l - l = len(m.Filename) + if m.EnvType != 0 { + n += 1 + sovFileRegistry(uint64(m.EnvType)) + } + l = len(m.EncryptionSettings) if l > 0 { n += 1 + l + sovFileRegistry(uint64(l)) } - if m.EnvLevel != 0 { - n += 1 + sovFileRegistry(uint64(m.EnvLevel)) - } return n } @@ -206,7 +272,7 @@ func sovFileRegistry(x uint64) (n int) { func sozFileRegistry(x uint64) (n int) { return sovFileRegistry(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *Registry) Unmarshal(dAtA []byte) error { +func (m *FileRegistry) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -229,10 +295,10 @@ func (m *Registry) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Registry: wiretype end group for non-group") + return fmt.Errorf("proto: FileRegistry: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Registry: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: FileRegistry: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -280,10 +346,102 @@ func (m *Registry) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Files = append(m.Files, &FileEntry{}) - if err := m.Files[len(m.Files)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err + if m.Files == nil { + m.Files = make(map[string]*FileEntry) } + var mapkey string + var mapvalue *FileEntry + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileRegistry + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileRegistry + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= (uint64(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthFileRegistry + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var mapmsglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowFileRegistry + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + mapmsglen |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + if mapmsglen < 0 { + return ErrInvalidLengthFileRegistry + } + postmsgIndex := iNdEx + mapmsglen + if mapmsglen < 0 { + return ErrInvalidLengthFileRegistry + } + if postmsgIndex > l { + return io.ErrUnexpectedEOF + } + mapvalue = &FileEntry{} + if err := mapvalue.Unmarshal(dAtA[iNdEx:postmsgIndex]); err != nil { + return err + } + iNdEx = postmsgIndex + } else { + iNdEx = entryPreIndex + skippy, err := skipFileRegistry(dAtA[iNdEx:]) + if err != nil { + return err + } + if skippy < 0 { + return ErrInvalidLengthFileRegistry + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Files[mapkey] = mapvalue iNdEx = postIndex default: iNdEx = preIndex @@ -336,10 +494,10 @@ func (m *FileEntry) Unmarshal(dAtA []byte) error { } switch fieldNum { case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Filename", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EnvType", wireType) } - var stringLen uint64 + m.EnvType = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowFileRegistry @@ -349,26 +507,16 @@ func (m *FileEntry) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= (uint64(b) & 0x7F) << shift + m.EnvType |= (EnvType(b) & 0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { - return ErrInvalidLengthFileRegistry - } - postIndex := iNdEx + intStringLen - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Filename = string(dAtA[iNdEx:postIndex]) - iNdEx = postIndex case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field EnvLevel", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field EncryptionSettings", wireType) } - m.EnvLevel = 0 + var byteLen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowFileRegistry @@ -378,11 +526,23 @@ func (m *FileEntry) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.EnvLevel |= (int32(b) & 0x7F) << shift + byteLen |= (int(b) & 0x7F) << shift if b < 0x80 { break } } + if byteLen < 0 { + return ErrInvalidLengthFileRegistry + } + postIndex := iNdEx + byteLen + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.EncryptionSettings = append(m.EncryptionSettings[:0], dAtA[iNdEx:postIndex]...) + if m.EncryptionSettings == nil { + m.EncryptionSettings = []byte{} + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipFileRegistry(dAtA[iNdEx:]) @@ -514,22 +674,29 @@ func init() { } var fileDescriptorFileRegistry = []byte{ - // 267 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xd2, 0x2e, 0x2e, 0xc9, 0x2f, - 0x4a, 0x4c, 0x4f, 0xd5, 0x4f, 0xcd, 0x4b, 0xcf, 0xcc, 0x83, 0x51, 0x05, 0x49, 0xfa, 0x69, 0x99, - 0x39, 0xa9, 0xf1, 0x45, 0xa9, 0xe9, 0x99, 0xc5, 0x25, 0x45, 0x95, 0x7a, 0x05, 0x45, 0xf9, 0x25, - 0xf9, 0x42, 0x8a, 0xc9, 0xf9, 0xc9, 0xd9, 0x45, 0xf9, 0x89, 0xc9, 0x19, 0x7a, 0x50, 0x6d, 0x7a, - 0x10, 0xf5, 0x7a, 0x30, 0x6d, 0x52, 0x22, 0xe9, 0xf9, 0xe9, 0xf9, 0x60, 0xd5, 0xfa, 0x20, 0x16, - 0x44, 0xa3, 0xd2, 0x1c, 0x46, 0x2e, 0x8e, 0x20, 0xa8, 0x59, 0x42, 0x3e, 0x5c, 0xec, 0x65, 0xa9, - 0x45, 0xc5, 0x99, 0xf9, 0x79, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x7c, 0x46, 0x46, 0x7a, 0x04, 0xcd, - 0xd5, 0x83, 0xe9, 0x0e, 0x83, 0xe8, 0x0c, 0x82, 0x19, 0x21, 0xe4, 0xc4, 0xc5, 0x0a, 0x72, 0x6a, - 0xb1, 0x04, 0x93, 0x02, 0xb3, 0x06, 0xb7, 0x91, 0x0e, 0x11, 0x66, 0xb9, 0x65, 0xe6, 0xa4, 0xba, - 0xe6, 0x95, 0x14, 0x55, 0x06, 0x41, 0xb4, 0x2a, 0xb9, 0x70, 0x71, 0xc2, 0xc5, 0x84, 0xa4, 0xb8, - 0x38, 0x40, 0x9c, 0xbc, 0xc4, 0xdc, 0x54, 0xb0, 0xfb, 0x38, 0x83, 0xe0, 0x7c, 0x21, 0x69, 0x2e, - 0xce, 0xd4, 0xbc, 0xb2, 0xf8, 0x9c, 0xd4, 0xb2, 0xd4, 0x1c, 0x09, 0x26, 0x05, 0x46, 0x0d, 0xd6, - 0x20, 0x8e, 0xd4, 0xbc, 0x32, 0x1f, 0x10, 0x5f, 0x4b, 0x9a, 0x8b, 0x1f, 0xcd, 0x95, 0x42, 0x1c, - 0x5c, 0x2c, 0x4e, 0x89, 0xc5, 0xa9, 0x02, 0x0c, 0x4e, 0x4a, 0x27, 0x1e, 0xca, 0x31, 0x9c, 0x78, - 0x24, 0xc7, 0x78, 0xe1, 0x91, 0x1c, 0xe3, 0x8d, 0x47, 0x72, 0x8c, 0x0f, 0x1e, 0xc9, 0x31, 0x4e, - 0x78, 0x2c, 0xc7, 0x10, 0xc5, 0x01, 0x73, 0x57, 0x12, 0x1b, 0x38, 0xb0, 0x8c, 0x01, 0x01, 0x00, - 0x00, 0xff, 0xff, 0x97, 0xf2, 0x8e, 0xf1, 0x94, 0x01, 0x00, 0x00, + // 371 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x92, 0xbf, 0x8e, 0xd3, 0x40, + 0x10, 0xc6, 0xbd, 0x0e, 0x21, 0xf1, 0x24, 0x80, 0xb5, 0x50, 0x44, 0x41, 0xb2, 0x42, 0xaa, 0x28, + 0x80, 0x2d, 0x99, 0x06, 0xa5, 0x8c, 0x08, 0x15, 0x45, 0xe4, 0x20, 0x0a, 0x9a, 0xc8, 0xb1, 0x26, + 0x66, 0x15, 0x6b, 0xd7, 0xda, 0x5d, 0x2c, 0x5c, 0xf3, 0x02, 0x57, 0xdd, 0x33, 0xa5, 0xbc, 0xf2, + 0xca, 0x3b, 0xdf, 0x8b, 0x9c, 0xfc, 0xef, 0x72, 0xba, 0xe6, 0x52, 0xed, 0xec, 0xce, 0x7c, 0xbf, + 0xef, 0xd3, 0x68, 0xe1, 0xa3, 0xd2, 0x42, 0x86, 0x31, 0x7a, 0xc8, 0x63, 0xc6, 0xdb, 0x23, 0xdd, + 0x79, 0x7b, 0x96, 0xe0, 0x56, 0x62, 0xcc, 0x94, 0x96, 0xb9, 0x9b, 0x4a, 0xa1, 0x05, 0xfd, 0x10, + 0x89, 0xe8, 0x20, 0x45, 0x18, 0xfd, 0x71, 0x1b, 0x99, 0x5b, 0xcf, 0xbb, 0xad, 0x6c, 0xfc, 0x2e, + 0x16, 0xb1, 0xa8, 0xa6, 0xbd, 0xb2, 0xaa, 0x85, 0xd3, 0x4b, 0x13, 0x86, 0xdf, 0x59, 0x82, 0x41, + 0xc3, 0xa3, 0x3f, 0xa0, 0x97, 0xa1, 0x54, 0x4c, 0xf0, 0x11, 0x99, 0x90, 0xd9, 0x6b, 0xdf, 0x77, + 0x9f, 0x65, 0xbb, 0xad, 0xfa, 0x57, 0xad, 0x0c, 0x5a, 0x04, 0x5d, 0x43, 0xb7, 0x8c, 0xab, 0x46, + 0xe6, 0xa4, 0x33, 0x1b, 0xf8, 0x8b, 0x33, 0x58, 0x8f, 0xd3, 0x54, 0x17, 0xb5, 0xe2, 0x5a, 0xe6, + 0x41, 0x0d, 0x1a, 0xef, 0x01, 0x4e, 0x8f, 0xd4, 0x86, 0xce, 0x01, 0xf3, 0x2a, 0xa9, 0x15, 0x94, + 0x25, 0x5d, 0x42, 0x37, 0x0b, 0x93, 0xbf, 0x38, 0x32, 0x27, 0x64, 0x36, 0xf0, 0x3f, 0x9d, 0xe9, + 0xd8, 0x78, 0x54, 0xd2, 0x85, 0xf9, 0x95, 0x4c, 0xff, 0x13, 0xb0, 0x1e, 0x1a, 0x74, 0x05, 0x7d, + 0xe4, 0xd9, 0x56, 0xe7, 0x29, 0x36, 0x6b, 0x99, 0x9f, 0x01, 0x5e, 0xf1, 0xec, 0x67, 0x9e, 0x62, + 0xd0, 0xc3, 0xba, 0xa0, 0x1e, 0xbc, 0x45, 0x1e, 0xc9, 0x3c, 0xd5, 0x4c, 0xf0, 0xad, 0x42, 0xad, + 0x19, 0x8f, 0x55, 0x15, 0x75, 0x18, 0xd0, 0x53, 0x6b, 0xd3, 0x74, 0xe6, 0xef, 0xe1, 0xcd, 0x93, + 0xdd, 0xd2, 0x3e, 0xbc, 0x58, 0x86, 0x0a, 0x6d, 0x63, 0xfe, 0x19, 0x7a, 0x8d, 0x03, 0x7d, 0x05, + 0xd6, 0x3a, 0x09, 0x19, 0xd7, 0xf8, 0x4f, 0xdb, 0x06, 0xb5, 0xa0, 0xbb, 0xd1, 0x42, 0xa2, 0x4d, + 0xca, 0xf1, 0x6f, 0xa1, 0x0e, 0x6d, 0x73, 0x39, 0x3d, 0xde, 0x3a, 0xc6, 0xb1, 0x70, 0xc8, 0x55, + 0xe1, 0x90, 0xeb, 0xc2, 0x21, 0x37, 0x85, 0x43, 0x2e, 0xee, 0x1c, 0xe3, 0x77, 0xbf, 0x4d, 0xbc, + 0x7b, 0x59, 0xfd, 0x8a, 0x2f, 0xf7, 0x01, 0x00, 0x00, 0xff, 0xff, 0xe4, 0xcf, 0x6f, 0xbd, 0x7d, + 0x02, 0x00, 0x00, } diff --git a/pkg/storage/engine/enginepb/file_registry.proto b/pkg/storage/engine/enginepb/file_registry.proto index 4fb48bfeb5b7..27b5761515ab 100644 --- a/pkg/storage/engine/enginepb/file_registry.proto +++ b/pkg/storage/engine/enginepb/file_registry.proto @@ -23,22 +23,37 @@ enum RegistryVersion { Base = 0; } +// EnvType determines which rocksdb::Env is used and for what purpose. +enum EnvType { + // The default Env when no encryption is used. + // File using Plaintext are not recorded in the file registry. + Plaintext = 0; + // The Env using store-level keys. + // Used only to read/write the data key registry. + Store = 1; + // The Env using data-level keys. + // Used as the default rocksdb Env when encryption is enabled. + Data = 2; +} + // Registry describes how a files are handled. This includes the // rockdb::Env responsible for each file as well as opaque env details. -// TODO(mberhault): this is still in flux. -message Registry { +message FileRegistry { // version is currently always Base. RegistryVersion version = 1; - repeated FileEntry files = 2; + // Map of filename -> FileEntry. + // Filename is relative to the rocksdb dir if the file is inside it. + // Otherwise it is an absolute path. + // TODO(mberhault): figure out if we need anything special for Windows. + map files = 2; } message FileEntry { - // File path relative to the DB directory. - // TODO(mberhault): figure out if we need anything special for Windows. - string Filename = 1; - // Env level identifies which rocksdb::Env is responsible for this file. - int32 env_level = 2; + // Env type identifies which rocksdb::Env is responsible for this file. + EnvType env_type = 1; // Env-specific fields for non-0 env. These are known by CCL code only. - // TODO(mberhault): determine format. Maybe google.protobuf.Any + // This is a serialized protobuf. We cannot use protobuf.Any since we use + // MessageLite in C++. + bytes encryption_settings = 2; }