Merge #21580 #25240

21580: libroach: encrypt data at rest r=mberhault a=mberhault

Part of encryption-at-rest (see #19783)

Quick overview:
* file registry records encryption settings for encrypted files
* an encrypted env has a "env level" (plain, store, data) and a key
manager as a source
* the data key manager uses an encrypted env with the store key manager as
key source
* rocksdb uses an encrypted env with the data key manager as key source

Release note: none

25240: distsqlrun: forward-port regression test for topk panic r=jordanlewis a=jordanlewis

Release note: None

Co-authored-by: marc <[email protected]>
Co-authored-by: Jordan Lewis <[email protected]>

c-deps/libroach/CMakeLists.txt

@@ -29,8 +29,8 @@ add_library(roach
   db.cc
   encoding.cc
   engine.cc
-  env_switching.cc
   eventlistener.cc
+  file_registry.cc
   getter.cc
   godefs.cc
   ldb.cc
@@ -49,6 +49,7 @@ add_library(roach
   protos/util/hlc/legacy_timestamp.pb.cc
   protos/util/hlc/timestamp.pb.cc
   protos/util/unresolved_addr.pb.cc
+  rocksdbutils/env_encryption.cc
 )
 target_include_directories(roach
   PUBLIC  ./include
@@ -59,6 +60,7 @@ target_include_directories(roach
 
 add_library(roachccl
   ccl/crypto_utils.cc
+  ccl/ctr_stream.cc
   ccl/db.cc
   ccl/key_manager.cc
   protosccl/ccl/baseccl/encryption_options.pb.cc
@@ -87,6 +89,7 @@ enable_testing()
 set(tests
   db_test.cc
   encoding_test.cc
+  file_registry_test.cc
   ccl/db_test.cc
   ccl/key_manager_test.cc
 )
@@ -132,7 +135,7 @@ foreach(tsrc ${tests})
       ${CRYPTOPP_LIB}
     )
     target_include_directories(${tname}
-      PRIVATE ../cryptopp
+      PRIVATE .. # CryptoPP headers are directly in the directory. Include .. to be able to include <cryptopp/....h>
       PRIVATE protosccl
     )
   endif()

c-deps/libroach/ccl/crypto_utils.cc

@@ -27,3 +27,17 @@ std::string RandomBytes(size_t length) {
   CryptoPP::OS_GenerateRandomBlock(false /* blocking */, data, length);
   return std::string(reinterpret_cast<const char*>(data.data()), data.size());
 }
+
+AESCipher::~AESCipher() {}
+
+size_t AESCipher::BlockSize() { return CryptoPP::AES::BLOCKSIZE; }
+
+rocksdb::Status AESCipher::Encrypt(char* data) {
+  enc_.ProcessBlock((byte*)data);
+  return rocksdb::Status::OK();
+}
+
+rocksdb::Status AESCipher::Decrypt(char* data) {
+  enc_.ProcessBlock((byte*)data);
+  return rocksdb::Status::OK();
+}

c-deps/libroach/ccl/crypto_utils.h

@@ -8,7 +8,9 @@
 
 #pragma once
 
+#include <cryptopp/aes.h>
 #include <string>
+#include "../rocksdbutils/env_encryption.h"
 
 /*
  * These provide various crypto primitives. They currently use CryptoPP.
@@ -23,3 +25,27 @@ std::string HexString(const std::string& s);
 // TODO(mberhault): it would be good to have a blocking version (/dev/random on *nix),
 // but to do it properly we might want to pre-read in the background.
 std::string RandomBytes(size_t length);
+
+// AES block cipher using CryptoPP.
+class AESCipher : public rocksdb_utils::BlockCipher {
+ public:
+  // The key must have a valid length (16/24/32 bytes) or CryptoPP will fail.
+  AESCipher(std::string key)
+      : enc_((byte*)key.data(), key.size()), dec_((byte*)key.data(), key.size()) {}
+  virtual ~AESCipher();
+
+  // Blocksize is fixed for AES.
+  virtual size_t BlockSize() override;
+
+  // Encrypt a block of data.
+  // Length of data is equal to BlockSize().
+  virtual rocksdb::Status Encrypt(char* data) override;
+
+  // Decrypt a block of data.
+  // Length of data is equal to BlockSize().
+  virtual rocksdb::Status Decrypt(char* data) override;
+
+ private:
+  CryptoPP::AES::Encryption enc_;
+  CryptoPP::AES::Decryption dec_;
+};

c-deps/libroach/ccl/ctr_stream.cc

@@ -0,0 +1,148 @@
+// Copyright 2018 The Cockroach Authors.
+//
+// Licensed as a CockroachDB Enterprise file under the Cockroach Community
+// License (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+//     https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt
+//  (found in the LICENSE.Apache file in the root directory).
+
+#include "ctr_stream.h"
+#include <arpa/inet.h>
+#include "../fmt.h"
+#include "../plaintext_stream.h"
+#include "crypto_utils.h"
+
+using namespace cockroach;
+
+namespace cockroach {
+
+rocksdb::Status BuildCipherStream(const enginepbccl::EncryptionSettings& settings,
+                                  const enginepbccl::SecretKey* key,
+                                  std::unique_ptr<rocksdb_utils::BlockAccessCipherStream>* result) {
+  // We should not be getting called for plaintext, and we only have AES.
+  if (settings.encryption_type() != enginepbccl::AES128_CTR &&
+      settings.encryption_type() != enginepbccl::AES192_CTR &&
+      settings.encryption_type() != enginepbccl::AES256_CTR) {
+    return rocksdb::Status::InvalidArgument(
+        fmt::StringPrintf("unknown encryption type %d", settings.encryption_type()));
+  }
+
+  (*result) = std::unique_ptr<rocksdb_utils::BlockAccessCipherStream>(
+      new CTRCipherStream(new AESCipher(key->key()), settings.nonce(), settings.counter()));
+
+  return rocksdb::Status::OK();
+}
+
+CTRCipherStreamCreator::~CTRCipherStreamCreator() {}
+
+rocksdb::Status CTRCipherStreamCreator::InitSettingsAndCreateCipherStream(
+    std::string* settings, std::unique_ptr<rocksdb_utils::BlockAccessCipherStream>* result) {
+  auto key = key_manager_->CurrentKey();
+  if (key == nullptr || key->info().encryption_type() == enginepbccl::Plaintext) {
+    // Plaintext: don't set "settings".
+    (*result) = std::unique_ptr<rocksdb_utils::BlockAccessCipherStream>(new PlaintextStream());
+    return rocksdb::Status::OK();
+  }
+
+  // Create the settings.
+  enginepbccl::EncryptionSettings enc_settings;
+  enc_settings.set_encryption_type(key->info().encryption_type());
+  enc_settings.set_key_id(key->info().key_id());
+
+  // Let's get 16 random bytes. 12 for the nonce, 4 for the counter.
+  std::string random_bytes = RandomBytes(16);
+  assert(random_bytes.size() == 16);
+
+  // First 12 bytes for the nonce.
+  enc_settings.set_nonce(random_bytes.substr(0, 12));
+  // Last 4 as an unsigned int32 for the counter.
+  uint32_t counter;
+  memcpy(&counter, random_bytes.data() + 12, 4);
+  enc_settings.set_counter(counter);
+
+  // Serialize enc_settings directly into the passed settings pointer. This will be ignored
+  // on error.
+  if (!enc_settings.SerializeToString(settings)) {
+    return rocksdb::Status::InvalidArgument("failed to serialize encryption settings");
+  }
+
+  return BuildCipherStream(enc_settings, key.get(), result);
+}
+
+// Create a cipher stream given encryption settings.
+rocksdb::Status CTRCipherStreamCreator::CreateCipherStreamFromSettings(
+    const std::string& settings, std::unique_ptr<rocksdb_utils::BlockAccessCipherStream>* result) {
+  enginepbccl::EncryptionSettings enc_settings;
+  if (!enc_settings.ParseFromString(settings)) {
+    return rocksdb::Status::InvalidArgument("failed to parse encryption settings");
+  }
+
+  if (settings.size() == 0 || enc_settings.encryption_type() == enginepbccl::Plaintext) {
+    // Plaintext.
+    (*result) = std::unique_ptr<rocksdb_utils::BlockAccessCipherStream>(new PlaintextStream());
+    return rocksdb::Status::OK();
+  }
+
+  // Get the key from the manager.
+  auto key = key_manager_->GetKey(enc_settings.key_id());
+  if (key == nullptr) {
+    return rocksdb::Status::InvalidArgument(fmt::StringPrintf(
+        "key_manager does not have a key with ID %s", enc_settings.key_id().c_str()));
+  }
+
+  return BuildCipherStream(enc_settings, key.get(), result);
+}
+
+enginepb::EnvType CTRCipherStreamCreator::GetEnvType() { return env_type_; }
+
+CTRCipherStream::CTRCipherStream(rocksdb_utils::BlockCipher* c, const std::string& nonce,
+                                 uint32_t counter)
+    : cipher_(c), nonce_(nonce), counter_(counter) {
+  assert(iv_.size() == (cipher_->BlockSize() - 4));
+}
+
+CTRCipherStream::~CTRCipherStream() {}
+
+size_t CTRCipherStream::BlockSize() { return cipher_->BlockSize(); }
+
+void CTRCipherStream::AllocateScratch(std::string& scratch) {
+  auto blockSize = cipher_->BlockSize();
+  scratch.reserve(blockSize);
+}
+
+rocksdb::Status CTRCipherStream::EncryptBlock(uint64_t blockIndex, char* data, char* scratch) {
+  // Create IV = nonce + counter
+  auto blockSize = cipher_->BlockSize();
+  auto nonce_size = blockSize - 4;
+  // Write the nonce at the beginning of the scratch space.
+  memcpy(scratch, nonce_.data(), nonce_size);
+
+  // Counter value for this block, converted to network byte order.
+  uint32_t block_counter = htonl(counter_ + blockIndex);
+  // Write after the nonce.
+  memcpy(scratch + nonce_size, &block_counter, 4);
+
+  // Encrypt nonce+counter
+  auto status = cipher_->Encrypt(scratch);
+  if (!status.ok()) {
+    return status;
+  }
+
+  // XOR data with ciphertext.
+  // TODO(mberhault): this is not an efficient XOR. Instead, we could move
+  // this into the cipher and use something like CryptoPP::ProcessAndXorBlock.
+  for (size_t i = 0; i < blockSize; i++) {
+    data[i] = data[i] ^ scratch[i];
+  }
+  return rocksdb::Status::OK();
+}
+
+// Decrypt a block of data at the given block index.
+// Length of data is equal to BlockSize();
+rocksdb::Status CTRCipherStream::DecryptBlock(uint64_t blockIndex, char* data, char* scratch) {
+  // For CTR decryption & encryption are the same
+  return EncryptBlock(blockIndex, data, scratch);
+}
+
+}  // namespace cockroach

c-deps/libroach/ccl/ctr_stream.h

@@ -0,0 +1,71 @@
+// Copyright 2018 The Cockroach Authors.
+//
+// Licensed as a CockroachDB Enterprise file under the Cockroach Community
+// License (the "License"); you may not use this file except in compliance with
+// the License. You may obtain a copy of the License at
+//
+//     https://github.com/cockroachdb/cockroach/blob/master/licenses/CCL.txt
+
+#pragma once
+
+#include <string>
+#include "../rocksdbutils/env_encryption.h"
+#include "key_manager.h"
+
+namespace cockroach {
+
+// CTRCipherStreamCreator creates a CTRCipherStream using a KeyManager.
+// Takes ownership of the KeyManager.
+class CTRCipherStreamCreator final : public rocksdb_utils::CipherStreamCreator {
+ public:
+  CTRCipherStreamCreator(KeyManager* key_mgr, enginepb::EnvType env_type)
+      : key_manager_(key_mgr), env_type_(env_type) {}
+  virtual ~CTRCipherStreamCreator();
+
+  virtual rocksdb::Status InitSettingsAndCreateCipherStream(
+      std::string* settings,
+      std::unique_ptr<rocksdb_utils::BlockAccessCipherStream>* result) override;
+
+  // Create a cipher stream given encryption settings.
+  virtual rocksdb::Status CreateCipherStreamFromSettings(
+      const std::string& settings,
+      std::unique_ptr<rocksdb_utils::BlockAccessCipherStream>* result) override;
+
+  virtual enginepb::EnvType GetEnvType() override;
+
+ private:
+  std::unique_ptr<KeyManager> key_manager_;
+  enginepb::EnvType env_type_;
+};
+
+class CTRCipherStream final : public rocksdb_utils::BlockAccessCipherStream {
+ public:
+  // Create a CTR cipher stream given:
+  // - a block cipher (takes ownership)
+  // - nonce of size 'cipher.BlockSize - sizeof(counter)' (eg: 16-4 = 12 bytes for AES)
+  // - counter
+  CTRCipherStream(rocksdb_utils::BlockCipher* c, const std::string& nonce, uint32_t counter);
+  virtual ~CTRCipherStream();
+
+  // BlockSize returns the size of each block supported by this cipher stream.
+  virtual size_t BlockSize() override;
+
+ protected:
+  // Allocate scratch space which is passed to EncryptBlock/DecryptBlock.
+  virtual void AllocateScratch(std::string&) override;
+
+  // Encrypt a block of data at the given block index.
+  // Length of data is equal to BlockSize();
+  virtual rocksdb::Status EncryptBlock(uint64_t blockIndex, char* data, char* scratch) override;
+
+  // Decrypt a block of data at the given block index.
+  // Length of data is equal to BlockSize();
+  virtual rocksdb::Status DecryptBlock(uint64_t blockIndex, char* data, char* scratch) override;
+
+ private:
+  std::unique_ptr<rocksdb_utils::BlockCipher> cipher_;
+  std::string nonce_;
+  uint32_t counter_;
+};
+
+}  // namespace cockroach