From 2523ccd0dc4d2fe1151aaedcde4ca76669d0670e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felix=20M=C3=BCller?= Date: Tue, 12 Apr 2022 13:11:00 +0200 Subject: [PATCH] Introduce state snapshot history (#698) Closes #688 --- .github/workflows/build_and_test.yml | 4 +- Cargo.lock | 2 + core-primitives/enclave-api/ffi/src/lib.rs | 7 + .../enclave-api/src/enclave_base.rs | 15 + core-primitives/settings/src/lib.rs | 1 + core-primitives/sgx/crypto/src/aes.rs | 25 +- core-primitives/sgx/crypto/src/ed25519.rs | 21 +- core-primitives/sgx/crypto/src/lib.rs | 2 +- core-primitives/sgx/crypto/src/mocks.rs | 61 ++- core-primitives/sgx/crypto/src/rsa3072.rs | 25 +- core-primitives/sgx/io/src/lib.rs | 34 +- core-primitives/stf-executor/src/executor.rs | 30 +- .../stf-executor/src/executor_tests.rs | 27 +- core-primitives/stf-state-handler/Cargo.toml | 6 + .../stf-state-handler/src/error.rs | 14 +- .../stf-state-handler/src/file_io.rs | 455 +++++++++++++----- .../src/global_file_state_handler.rs | 83 ---- .../stf-state-handler/src/handle_state.rs | 32 +- .../src/in_memory_state_file_io.rs | 380 +++++++++++++++ core-primitives/stf-state-handler/src/lib.rs | 23 +- .../src/query_shard_state.rs | 2 +- .../stf-state-handler/src/state_handler.rs | 180 +++++++ .../src/state_key_repository.rs | 96 ++++ .../src/state_snapshot_primitives.rs | 55 +++ .../src/state_snapshot_repository.rs | 443 +++++++++++++++++ .../src/state_snapshot_repository_loader.rs | 202 ++++++++ .../stf-state-handler/src/test/mocks/mod.rs | 19 + .../test/mocks/state_key_repository_mock.rs | 68 +++ .../test/mocks/versioned_state_access_mock.rs | 100 ++++ .../stf-state-handler/src/test/mod.rs | 25 + .../stf-state-handler/src/test/sgx_tests.rs | 340 +++++++++++++ .../stf-state-handler/src/tests.rs | 175 ------- .../test/src/mock/handle_state_mock.rs | 79 +-- core-primitives/time-utils/src/lib.rs | 5 + core-primitives/top-pool-author/src/author.rs | 13 +- .../top-pool-author/src/author_tests.rs | 4 +- .../src/triggered_dispatcher.rs | 7 + .../light-client/src/concurrent_access.rs | 16 +- core/parentchain/light-client/src/io.rs | 12 +- .../src/mocks/validator_mock_seal.rs | 8 +- enclave-runtime/Cargo.lock | 1 + enclave-runtime/Enclave.edl | 4 + enclave-runtime/src/attestation.rs | 6 +- enclave-runtime/src/global_components.rs | 20 +- enclave-runtime/src/initialization.rs | 56 ++- enclave-runtime/src/lib.rs | 21 +- .../test/fixtures/initialize_test_state.rs | 3 +- .../src/test/sidechain_aura_tests.rs | 4 +- enclave-runtime/src/tests.rs | 23 +- enclave-runtime/src/tls_ra/seal_handler.rs | 70 +-- enclave-runtime/src/tls_ra/tls_ra_client.rs | 25 +- enclave-runtime/src/tls_ra/tls_ra_server.rs | 26 +- enclave-runtime/src/top_pool_execution.rs | 10 +- service/Cargo.toml | 1 + service/src/main.rs | 35 +- service/src/tests/ecalls.rs | 7 +- service/src/tests/mocks/enclave_api_mock.rs | 4 + .../consensus/aura/src/block_importer.rs | 4 +- .../aura/src/test/block_importer_tests.rs | 4 +- .../consensus/common/src/block_import.rs | 8 +- sidechain/consensus/slots/src/slots.rs | 24 +- 61 files changed, 2814 insertions(+), 638 deletions(-) delete mode 100644 core-primitives/stf-state-handler/src/global_file_state_handler.rs create mode 100644 core-primitives/stf-state-handler/src/in_memory_state_file_io.rs create mode 100644 core-primitives/stf-state-handler/src/state_handler.rs create mode 100644 core-primitives/stf-state-handler/src/state_key_repository.rs create mode 100644 core-primitives/stf-state-handler/src/state_snapshot_primitives.rs create mode 100644 core-primitives/stf-state-handler/src/state_snapshot_repository.rs create mode 100644 core-primitives/stf-state-handler/src/state_snapshot_repository_loader.rs create mode 100644 core-primitives/stf-state-handler/src/test/mocks/mod.rs create mode 100644 core-primitives/stf-state-handler/src/test/mocks/state_key_repository_mock.rs create mode 100644 core-primitives/stf-state-handler/src/test/mocks/versioned_state_access_mock.rs create mode 100644 core-primitives/stf-state-handler/src/test/mod.rs create mode 100644 core-primitives/stf-state-handler/src/test/sgx_tests.rs delete mode 100644 core-primitives/stf-state-handler/src/tests.rs diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index b8ba7c087f..08061fd854 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -209,8 +209,8 @@ jobs: # * `set -eo pipefail` is needed to return an error even if piped to `tee`. shell: bash --noprofile --norc -eo pipefail {0} run: | - touch ${{ env.LOG_DIR}}/local-setup.log - ./local-setup/launch.py local-setup/github-action-config.json 2>&1 | tee ${{ env.LOG_DIR}}/local-setup.log & + touch ${{ env.LOG_DIR }}/local-setup.log + ./local-setup/launch.py local-setup/github-action-config.json 2>&1 | tee -i ${{ env.LOG_DIR }}/local-setup.log & sleep 150 - name: ${{ matrix.demo_name }} diff --git a/Cargo.lock b/Cargo.lock index f88568f79d..95fc0ce167 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2205,6 +2205,7 @@ dependencies = [ "itp-enclave-metrics", "itp-node-api-extensions", "itp-settings", + "itp-stf-state-handler", "itp-test", "itp-types", "its-consensus-slots", @@ -2717,6 +2718,7 @@ dependencies = [ "itp-settings", "itp-sgx-crypto", "itp-sgx-io", + "itp-time-utils", "itp-types", "lazy_static", "log 0.4.14 (registry+https://github.com/rust-lang/crates.io-index)", diff --git a/core-primitives/enclave-api/ffi/src/lib.rs b/core-primitives/enclave-api/ffi/src/lib.rs index 99af857b59..d569d07d1a 100644 --- a/core-primitives/enclave-api/ffi/src/lib.rs +++ b/core-primitives/enclave-api/ffi/src/lib.rs @@ -49,6 +49,13 @@ extern "C" { latest_header_size: usize, ) -> sgx_status_t; + pub fn init_shard( + eid: sgx_enclave_id_t, + retval: *mut sgx_status_t, + shard: *const u8, + shard_size: u32, + ) -> sgx_status_t; + pub fn trigger_parentchain_block_import( eid: sgx_enclave_id_t, retval: *mut sgx_status_t, diff --git a/core-primitives/enclave-api/src/enclave_base.rs b/core-primitives/enclave-api/src/enclave_base.rs index 34fc370bf3..d2b688369e 100644 --- a/core-primitives/enclave-api/src/enclave_base.rs +++ b/core-primitives/enclave-api/src/enclave_base.rs @@ -49,6 +49,9 @@ pub trait EnclaveBase: Send + Sync + 'static { authority_proof: Vec>, ) -> EnclaveResult; + /// Initialize a new shard. + fn init_shard(&self, shard: Vec) -> EnclaveResult<()>; + /// Trigger the import of parentchain block explicitly. Used when initializing a light-client /// with a triggered import dispatcher. fn trigger_parentchain_block_import(&self) -> EnclaveResult<()>; @@ -145,6 +148,18 @@ impl EnclaveBase for Enclave { Ok(latest) } + fn init_shard(&self, shard: Vec) -> EnclaveResult<()> { + let mut retval = sgx_status_t::SGX_SUCCESS; + + let result = + unsafe { ffi::init_shard(self.eid, &mut retval, shard.as_ptr(), shard.len() as u32) }; + + ensure!(result == sgx_status_t::SGX_SUCCESS, Error::Sgx(result)); + ensure!(retval == sgx_status_t::SGX_SUCCESS, Error::Sgx(retval)); + + Ok(()) + } + fn trigger_parentchain_block_import(&self) -> EnclaveResult<()> { let mut retval = sgx_status_t::SGX_SUCCESS; diff --git a/core-primitives/settings/src/lib.rs b/core-primitives/settings/src/lib.rs index f1933b0041..d607e329d5 100644 --- a/core-primitives/settings/src/lib.rs +++ b/core-primitives/settings/src/lib.rs @@ -53,6 +53,7 @@ pub mod files { pub static RA_API_KEY_FILE: &str = "key.txt"; pub const SPID_MIN_LENGTH: usize = 32; + pub const STATE_SNAPSHOTS_CACHE_SIZE: usize = 120; } /// Settings concerning the worker diff --git a/core-primitives/sgx/crypto/src/aes.rs b/core-primitives/sgx/crypto/src/aes.rs index 861146d0e4..e9d3ebef0a 100644 --- a/core-primitives/sgx/crypto/src/aes.rs +++ b/core-primitives/sgx/crypto/src/aes.rs @@ -30,7 +30,7 @@ use std::convert::{TryFrom, TryInto}; type AesOfb = Ofb; -#[derive(Debug, Default, Encode, Decode, Clone, Copy)] +#[derive(Debug, Default, Encode, Decode, Clone, Copy, PartialEq, Eq)] pub struct Aes { pub key: [u8; 16], pub init_vec: [u8; 16], @@ -78,24 +78,37 @@ pub mod sgx { use super::*; use itp_settings::files::AES_KEY_FILE_AND_INIT_V; - use itp_sgx_io::{seal, unseal, SealedIO}; + use itp_sgx_io::{seal, unseal, SealedIO, StaticSealedIO}; use log::info; use sgx_rand::{Rng, StdRng}; use std::sgxfs::SgxFile; - impl SealedIO for AesSeal { + impl StaticSealedIO for AesSeal { type Error = Error; type Unsealed = Aes; - fn unseal() -> Result { + fn unseal_from_static_file() -> Result { Ok(unseal(AES_KEY_FILE_AND_INIT_V).map(|b| Decode::decode(&mut b.as_slice()))??) } - fn seal(unsealed: Self::Unsealed) -> Result<()> { + fn seal_to_static_file(unsealed: Self::Unsealed) -> Result<()> { Ok(unsealed.using_encoded(|bytes| seal(bytes, AES_KEY_FILE_AND_INIT_V))?) } } + impl SealedIO for AesSeal { + type Error = Error; + type Unsealed = Aes; + + fn unseal(&self) -> Result { + Self::unseal_from_static_file() + } + + fn seal(&self, unsealed: Self::Unsealed) -> Result<()> { + Self::seal_to_static_file(unsealed) + } + } + pub fn create_sealed_if_absent() -> Result<()> { if SgxFile::open(AES_KEY_FILE_AND_INIT_V).is_err() { info!("[Enclave] Keyfile not found, creating new! {}", AES_KEY_FILE_AND_INIT_V); @@ -112,6 +125,6 @@ pub mod sgx { rand.fill_bytes(&mut key); rand.fill_bytes(&mut iv); - AesSeal::seal(Aes::new(key, iv)) + AesSeal::seal_to_static_file(Aes::new(key, iv)) } } diff --git a/core-primitives/sgx/crypto/src/ed25519.rs b/core-primitives/sgx/crypto/src/ed25519.rs index b179e10c8f..c9752cadb5 100644 --- a/core-primitives/sgx/crypto/src/ed25519.rs +++ b/core-primitives/sgx/crypto/src/ed25519.rs @@ -30,17 +30,17 @@ pub mod sgx { use crate::error::{Error, Result}; use codec::Encode; use itp_settings::files::SEALED_SIGNER_SEED_FILE; - use itp_sgx_io::{seal, unseal, SealedIO}; + use itp_sgx_io::{seal, unseal, SealedIO, StaticSealedIO}; use log::*; use sgx_rand::{Rng, StdRng}; use sp_core::{crypto::Pair, ed25519}; use std::{path::Path, sgxfs::SgxFile}; - impl SealedIO for Ed25519Seal { + impl StaticSealedIO for Ed25519Seal { type Error = Error; type Unsealed = ed25519::Pair; - fn unseal() -> Result { + fn unseal_from_static_file() -> Result { let raw = unseal(SEALED_SIGNER_SEED_FILE)?; let key = ed25519::Pair::from_seed_slice(&raw) @@ -49,11 +49,24 @@ pub mod sgx { Ok(key.into()) } - fn seal(unsealed: Self::Unsealed) -> Result<()> { + fn seal_to_static_file(unsealed: Self::Unsealed) -> Result<()> { Ok(unsealed.seed().using_encoded(|bytes| seal(bytes, SEALED_SIGNER_SEED_FILE))?) } } + impl SealedIO for Ed25519Seal { + type Error = Error; + type Unsealed = ed25519::Pair; + + fn unseal(&self) -> Result { + Self::unseal_from_static_file() + } + + fn seal(&self, unsealed: Self::Unsealed) -> Result<()> { + Self::seal_to_static_file(unsealed) + } + } + pub fn create_sealed_if_absent() -> Result<()> { if SgxFile::open(SEALED_SIGNER_SEED_FILE).is_err() { if Path::new(SEALED_SIGNER_SEED_FILE).exists() { diff --git a/core-primitives/sgx/crypto/src/lib.rs b/core-primitives/sgx/crypto/src/lib.rs index 68c7579b3d..58721611fa 100644 --- a/core-primitives/sgx/crypto/src/lib.rs +++ b/core-primitives/sgx/crypto/src/lib.rs @@ -33,5 +33,5 @@ pub use self::rsa3072::*; pub use error::*; pub use traits::*; -#[cfg(all(feature = "mocks", feature = "sgx"))] +#[cfg(feature = "mocks")] pub mod mocks; diff --git a/core-primitives/sgx/crypto/src/mocks.rs b/core-primitives/sgx/crypto/src/mocks.rs index 539b0a234b..fdb4864989 100644 --- a/core-primitives/sgx/crypto/src/mocks.rs +++ b/core-primitives/sgx/crypto/src/mocks.rs @@ -15,41 +15,72 @@ */ +#[cfg(feature = "sgx")] +use std::sync::SgxRwLock as RwLock; + +#[cfg(feature = "std")] +use std::sync::RwLock; + use crate::{ + aes::Aes, error::{Error, Result}, - Aes, }; -use itp_sgx_io::SealedIO; -use sgx_crypto_helper::rsa3072::Rsa3072KeyPair; +use itp_sgx_io::{SealedIO, StaticSealedIO}; #[derive(Default)] -pub struct AesSealMock {} +pub struct AesSealMock { + aes: RwLock, +} -impl SealedIO for AesSealMock { +impl StaticSealedIO for AesSealMock { type Error = Error; type Unsealed = Aes; - fn unseal() -> Result { + fn unseal_from_static_file() -> Result { Ok(Aes::default()) } - fn seal(_unsealed: Self::Unsealed) -> Result<()> { + fn seal_to_static_file(_unsealed: Self::Unsealed) -> Result<()> { Ok(()) } } -#[derive(Default)] -pub struct Rsa3072SealMock {} - -impl SealedIO for Rsa3072SealMock { +impl SealedIO for AesSealMock { type Error = Error; - type Unsealed = Rsa3072KeyPair; + type Unsealed = Aes; - fn unseal() -> Result { - Ok(Rsa3072KeyPair::default()) + fn unseal(&self) -> std::result::Result { + self.aes + .read() + .map_err(|e| Error::Other(format!("{:?}", e).into())) + .map(|k| k.clone()) } - fn seal(_unsealed: Self::Unsealed) -> Result<()> { + fn seal(&self, unsealed: Self::Unsealed) -> std::result::Result<(), Self::Error> { + let mut aes_lock = self.aes.write().map_err(|e| Error::Other(format!("{:?}", e).into()))?; + *aes_lock = unsealed; Ok(()) } } + +#[cfg(feature = "sgx")] +pub mod sgx { + use super::*; + use sgx_crypto_helper::rsa3072::Rsa3072KeyPair; + + #[derive(Default)] + pub struct Rsa3072SealMock {} + + impl StaticSealedIO for Rsa3072SealMock { + type Error = Error; + type Unsealed = Rsa3072KeyPair; + + fn unseal_from_static_file() -> Result { + Ok(Rsa3072KeyPair::default()) + } + + fn seal_to_static_file(_unsealed: Self::Unsealed) -> Result<()> { + Ok(()) + } + } +} diff --git a/core-primitives/sgx/crypto/src/rsa3072.rs b/core-primitives/sgx/crypto/src/rsa3072.rs index 05a71ac198..2db909d436 100644 --- a/core-primitives/sgx/crypto/src/rsa3072.rs +++ b/core-primitives/sgx/crypto/src/rsa3072.rs @@ -23,7 +23,7 @@ use crate::{ }; use derive_more::Display; use itp_settings::files::RSA3072_SEALED_KEY_FILE; -use itp_sgx_io::{seal, unseal, SealedIO}; +use itp_sgx_io::{seal, unseal, SealedIO, StaticSealedIO}; use log::*; use sgx_crypto_helper::{ rsa3072::{Rsa3072KeyPair, Rsa3072PubKey}, @@ -34,23 +34,36 @@ use std::{sgxfs::SgxFile, vec::Vec}; #[derive(Copy, Clone, Debug, Display)] pub struct Rsa3072Seal; -impl SealedIO for Rsa3072Seal { +impl StaticSealedIO for Rsa3072Seal { type Error = Error; type Unsealed = Rsa3072KeyPair; - fn unseal() -> Result { + fn unseal_from_static_file() -> Result { let raw = unseal(RSA3072_SEALED_KEY_FILE)?; let key: Rsa3072KeyPair = serde_json::from_slice(&raw).map_err(|e| Error::Other(format!("{:?}", e).into()))?; Ok(key.into()) } - fn seal(unsealed: Rsa3072KeyPair) -> Result<()> { + fn seal_to_static_file(unsealed: Rsa3072KeyPair) -> Result<()> { let key_json = serde_json::to_vec(&unsealed).map_err(|e| Error::Other(format!("{:?}", e).into()))?; Ok(seal(&key_json, RSA3072_SEALED_KEY_FILE)?) } } +impl SealedIO for Rsa3072Seal { + type Error = Error; + type Unsealed = Rsa3072KeyPair; + + fn unseal(&self) -> Result { + Self::unseal_from_static_file() + } + + fn seal(&self, unsealed: Self::Unsealed) -> Result<()> { + Self::seal_to_static_file(unsealed) + } +} + impl ShieldingCrypto for Rsa3072KeyPair { type Error = Error; @@ -71,7 +84,7 @@ impl ShieldingCrypto for Rsa3072KeyPair { impl Rsa3072Seal { pub fn unseal_pubkey() -> Result { - let pair = Self::unseal()?; + let pair = Self::unseal_from_static_file()?; let pubkey = pair.export_pubkey().map_err(|e| Error::Other(format!("{:?}", e).into()))?; Ok(pubkey) } @@ -88,5 +101,5 @@ pub fn create_sealed_if_absent() -> Result<()> { pub fn create_sealed() -> Result<()> { let rsa_keypair = Rsa3072KeyPair::new().map_err(|e| Error::Other(format!("{:?}", e).into()))?; // println!("[Enclave] generated RSA3072 key pair. Cleartext: {}", rsa_key_json); - Rsa3072Seal::seal(rsa_keypair) + Rsa3072Seal::seal_to_static_file(rsa_keypair) } diff --git a/core-primitives/sgx/io/src/lib.rs b/core-primitives/sgx/io/src/lib.rs index 86463e5eb6..278ab31cd7 100644 --- a/core-primitives/sgx/io/src/lib.rs +++ b/core-primitives/sgx/io/src/lib.rs @@ -9,8 +9,10 @@ compile_error!("feature \"std\" and feature \"sgx\" cannot be enabled at the sam extern crate sgx_tstd as std; use std::{ + convert::AsRef, fs, io::{Read, Result as IOResult, Write}, + path::Path, string::String, vec::Vec, }; @@ -28,27 +30,43 @@ pub trait IO: Sized { /// Abstraction around IO that is supposed to use `SgxFile`. We expose it also in `std` to /// be able to put it as trait bounds in `std` and use it in tests. +/// +/// This is the static method (or associated function) version, should be made obsolete over time, +/// since it has state, but hides it in a global state. Makes it difficult to mock. +pub trait StaticSealedIO: Sized { + type Error: From + std::fmt::Debug + 'static; + + /// Type that is unsealed. + type Unsealed; + + fn unseal_from_static_file() -> Result; + fn seal_to_static_file(unsealed: Self::Unsealed) -> Result<(), Self::Error>; +} + +/// Abstraction around IO that is supposed to use `SgxFile`. We expose it also in `std` to +/// be able to put it as trait bounds in `std` and use it in tests. +/// pub trait SealedIO: Sized { type Error: From + std::fmt::Debug + 'static; /// Type that is unsealed. type Unsealed; - fn unseal() -> Result; - fn seal(unsealed: Self::Unsealed) -> Result<(), Self::Error>; + fn unseal(&self) -> Result; + fn seal(&self, unsealed: Self::Unsealed) -> Result<(), Self::Error>; } -pub fn read(path: &str) -> IOResult> { +pub fn read>(path: P) -> IOResult> { let mut buf = Vec::new(); fs::File::open(path).map(|mut f| f.read_to_end(&mut buf))??; Ok(buf) } -pub fn write(bytes: &[u8], path: &str) -> IOResult<()> { +pub fn write>(bytes: &[u8], path: P) -> IOResult<()> { fs::File::create(path).map(|mut f| f.write_all(bytes))? } -pub fn read_to_string(filepath: &str) -> IOResult { +pub fn read_to_string>(filepath: P) -> IOResult { let mut contents = String::new(); fs::File::open(filepath).map(|mut f| f.read_to_string(&mut contents))??; Ok(contents) @@ -57,18 +75,20 @@ pub fn read_to_string(filepath: &str) -> IOResult { #[cfg(feature = "sgx")] mod sgx { use std::{ + convert::AsRef, io::{Read, Result, Write}, + path::Path, sgxfs::SgxFile, vec::Vec, }; - pub fn unseal(path: &str) -> Result> { + pub fn unseal>(path: P) -> Result> { let mut buf = Vec::new(); SgxFile::open(path).map(|mut f| f.read_to_end(&mut buf))??; Ok(buf) } - pub fn seal(bytes: &[u8], path: &str) -> Result<()> { + pub fn seal>(bytes: &[u8], path: P) -> Result<()> { SgxFile::create(path).map(|mut f| f.write_all(bytes))? } } diff --git a/core-primitives/stf-executor/src/executor.rs b/core-primitives/stf-executor/src/executor.rs index 75f119c73b..2044be8c82 100644 --- a/core-primitives/stf-executor/src/executor.rs +++ b/core-primitives/stf-executor/src/executor.rs @@ -54,7 +54,7 @@ pub struct StfExecutor { impl StfExecutor where OCallApi: EnclaveAttestationOCallApi + GetStorageVerified, - StateHandler: HandleState, + StateHandler: HandleState, ExternalitiesT: SgxExternalitiesTrait + Encode, { pub fn new(ocall_api: Arc, state_handler: Arc) -> Self { @@ -124,7 +124,7 @@ impl StfExecuteTrustedCall for StfExecutor where OCallApi: EnclaveAttestationOCallApi + GetStorageVerified, - StateHandler: HandleState, + StateHandler: HandleState, ExternalitiesT: SgxExternalitiesTrait + Encode, { fn execute_trusted_call( @@ -157,7 +157,7 @@ where calls.append(&mut extrinsic_callbacks); trace!("Updating state of shard {:?}", shard); - self.state_handler.write(state, state_lock, shard)?; + self.state_handler.write_after_mutation(state, state_lock, shard)?; Ok(maybe_call_hash) } @@ -167,7 +167,7 @@ impl StfExecuteShieldFunds for StfExecutor where OCallApi: EnclaveAttestationOCallApi + GetStorageVerified, - StateHandler: HandleState, + StateHandler: HandleState, ExternalitiesT: SgxExternalitiesTrait + Encode, { fn execute_shield_funds( @@ -190,7 +190,9 @@ where Stf::execute(&mut state, trusted_call, &mut Vec::::new()) .map_err::(|e| e.into())?; - self.state_handler.write(state, state_lock, shard).map_err(|e| e.into()) + self.state_handler + .write_after_mutation(state, state_lock, shard) + .map_err(|e| e.into()) } } @@ -198,7 +200,7 @@ impl StfUpdateState for StfExecutor where OCallApi: EnclaveAttestationOCallApi + GetStorageVerified, - StateHandler: HandleState + QueryShardState, + StateHandler: HandleState + QueryShardState, ExternalitiesT: SgxExternalitiesTrait + Encode, { fn update_states(&self, header: &ParentchainHeader) -> Result<()> { @@ -222,7 +224,7 @@ where let (state_lock, mut state) = self.state_handler.load_for_mutation(&shard_id)?; match Stf::update_parentchain_block(&mut state, header.clone()) { Ok(_) => { - self.state_handler.write(state, state_lock, &shard_id)?; + self.state_handler.write_after_mutation(state, state_lock, &shard_id)?; }, Err(e) => error!("Could not update parentchain block. {:?}: {:?}", shard_id, e), } @@ -252,7 +254,7 @@ where error!("Could not update parentchain block. {:?}: {:?}", shard_id, e) } - self.state_handler.write(state, state_lock, &shard_id)?; + self.state_handler.write_after_mutation(state, state_lock, &shard_id)?; } }, None => debug!("No shards are on the chain yet"), @@ -266,7 +268,7 @@ impl StateUpdateProposer for StfExecutor where OCallApi: EnclaveAttestationOCallApi + GetStorageVerified, - StateHandler: HandleState, + StateHandler: HandleState, ExternalitiesT: SgxExternalitiesTrait + Encode, { type Externalities = ExternalitiesT; @@ -285,7 +287,7 @@ where { let ends_at = duration_now() + max_exec_duration; - let state = self.state_handler.load_initialized(shard)?; + let state = self.state_handler.load(shard)?; let state_hash_before_execution = state_hash(&state); // Execute any pre-processing steps. @@ -327,7 +329,7 @@ impl StfExecuteTimedGettersBatch for StfExecutor where OCallApi: EnclaveAttestationOCallApi + GetStorageVerified, - StateHandler: HandleState, + StateHandler: HandleState, ExternalitiesT: SgxExternalitiesTrait + Encode, { type Externalities = ExternalitiesT; @@ -350,7 +352,7 @@ where } // load state once per shard - let mut state = self.state_handler.load_initialized(&shard)?; + let mut state = self.state_handler.load(&shard)?; for trusted_getter_signed in trusted_getters.into_iter() { // get state @@ -371,7 +373,7 @@ where impl StfExecuteGenericUpdate for StfExecutor where - StateHandler: HandleState, + StateHandler: HandleState, ExternalitiesT: SgxExternalitiesTrait + Encode, { type Externalities = ExternalitiesT; @@ -393,7 +395,7 @@ where let new_state_hash = self .state_handler - .write(new_state, state_lock, shard) + .write_after_mutation(new_state, state_lock, shard) .map_err(|e| Error::StateHandler(e))?; Ok((result, new_state_hash)) } diff --git a/core-primitives/stf-executor/src/executor_tests.rs b/core-primitives/stf-executor/src/executor_tests.rs index b6b671c64c..960fe51ebc 100644 --- a/core-primitives/stf-executor/src/executor_tests.rs +++ b/core-primitives/stf-executor/src/executor_tests.rs @@ -63,7 +63,7 @@ pub fn propose_state_update_executes_all_calls_given_enough_time() { let call_operation_hash_two: H256 = blake2_256(&signed_call_two.clone().into_trusted_operation(true).encode()).into(); - let old_state_hash = state_hash(&state_handler.load_initialized(&shard).unwrap()); + let old_state_hash = state_hash(&state_handler.load(&shard).unwrap()); // when let batch_execution_result = stf_executor @@ -84,10 +84,7 @@ pub fn propose_state_update_executes_all_calls_given_enough_time() { vec![call_operation_hash, call_operation_hash_two] ); // Ensure that state has been updated and not actually written. - assert_ne!( - state_handler.load_initialized(&shard).unwrap(), - batch_execution_result.state_after_execution - ); + assert_ne!(state_handler.load(&shard).unwrap(), batch_execution_result.state_after_execution); } pub fn propose_state_update_executes_only_one_trusted_call_given_not_enough_time() { @@ -111,7 +108,7 @@ pub fn propose_state_update_executes_only_one_trusted_call_given_not_enough_time ) .sign(&sender.clone().into(), 0, &mrenclave, &shard); - let old_state_hash = state_hash(&state_handler.load_initialized(&shard).unwrap()); + let old_state_hash = state_hash(&state_handler.load(&shard).unwrap()); // when let batch_execution_result = stf_executor @@ -129,19 +126,17 @@ pub fn propose_state_update_executes_only_one_trusted_call_given_not_enough_time assert_eq!(batch_execution_result.executed_operations.len(), 1); assert_eq!(batch_execution_result.get_executed_operation_hashes(), vec![call_operation_hash]); // Ensure that state has been updated and not actually written. - assert_ne!( - state_handler.load_initialized(&shard).unwrap(), - batch_execution_result.state_after_execution - ); + assert_ne!(state_handler.load(&shard).unwrap(), batch_execution_result.state_after_execution); } pub fn propose_state_update_always_executes_preprocessing_step() { // given let shard = ShardIdentifier::default(); let (stf_executor, _, state_handler) = stf_executor(); + let _init_hash = state_handler.initialize_shard(shard).unwrap(); let key = "my_key".encode(); let value = "my_value".encode(); - let old_state_hash = state_hash(&state_handler.load_initialized(&shard).unwrap()); + let old_state_hash = state_hash(&state_handler.load(&shard).unwrap()); // when let batch_execution_result = stf_executor @@ -161,7 +156,7 @@ pub fn propose_state_update_always_executes_preprocessing_step() { assert_eq!(old_state_hash, batch_execution_result.state_hash_before_execution); // Ensure that state has been updated. - let old_state = state_handler.load_initialized(&shard).unwrap(); + let old_state = state_handler.load(&shard).unwrap(); let retrieved_value = batch_execution_result.state_after_execution.get(key.as_slice()).unwrap(); assert_eq!(*retrieved_value, value); // Ensure that state has not been actually written. @@ -244,9 +239,10 @@ pub fn execute_update_works() { // given let shard = ShardIdentifier::default(); let (stf_executor, _ocall_api, state_handler) = stf_executor(); + let _init_hash = state_handler.initialize_shard(shard).unwrap(); let key = "my_key".encode(); let value = "my_value".encode(); - let old_state_hash = state_hash(&state_handler.load_initialized(&shard).unwrap()); + let old_state_hash = state_hash(&state_handler.load(&shard).unwrap()); // when let (result, updated_state_hash) = stf_executor @@ -261,7 +257,7 @@ pub fn execute_update_works() { assert_ne!(updated_state_hash, old_state_hash); // Ensure that state has been written. - let updated_state = state_handler.load_initialized(&shard).unwrap(); + let updated_state = state_handler.load(&shard).unwrap(); let retrieved_value = updated_state.get(key.as_slice()).unwrap(); assert_eq!(*retrieved_value, value); } @@ -307,11 +303,12 @@ fn init_state_and_shard_with_state_handler>( state_handler: &S, ) -> (State, ShardIdentifier) { let shard = ShardIdentifier::default(); + let _hash = state_handler.initialize_shard(shard).unwrap(); let (lock, mut state) = state_handler.load_for_mutation(&shard).unwrap(); test_genesis_setup(&mut state); - state_handler.write(state.clone(), lock, &shard).unwrap(); + state_handler.write_after_mutation(state.clone(), lock, &shard).unwrap(); (state, shard) } diff --git a/core-primitives/stf-state-handler/Cargo.toml b/core-primitives/stf-state-handler/Cargo.toml index 7effa399b3..f36a1e4f2a 100644 --- a/core-primitives/stf-state-handler/Cargo.toml +++ b/core-primitives/stf-state-handler/Cargo.toml @@ -14,6 +14,7 @@ std = [ "ita-stf/std", "itp-sgx-crypto/std", "itp-sgx-io/std", + "itp-time-utils/std", "itp-types/std", "sgx-externalities/std", "thiserror", @@ -25,6 +26,7 @@ sgx = [ "ita-stf/sgx", "itp-sgx-crypto/sgx", "itp-sgx-io/sgx", + "itp-time-utils/sgx", "sgx-externalities/sgx", "thiserror_sgx", ] @@ -40,6 +42,7 @@ ita-stf = { path = "../../app-libs/stf", default-features = false } itp-settings = { path = "../../core-primitives/settings" } itp-sgx-crypto = { path = "../../core-primitives/sgx/crypto", default-features = false } itp-sgx-io = { path = "../../core-primitives/sgx/io", default-features = false } +itp-time-utils = { path = "../../core-primitives/time-utils", default-features = false } itp-types = { path = "../../core-primitives/types", default-features = false } # sgx enabled external libraries @@ -58,3 +61,6 @@ codec = { package = "parity-scale-codec", version = "3.0.0", default-features = log = { version = "0.4", default-features = false } lazy_static = { version = "1.1.0", features = ["spin_no_std"] } sp-core = { version = "6.0.0", default-features = false, features = ["full_crypto"], git = "https://github.com/paritytech/substrate.git", branch = "master" } + +[dev-dependencies] +itp-sgx-crypto = { path = "../../core-primitives/sgx/crypto", features = ["mocks"] } \ No newline at end of file diff --git a/core-primitives/stf-state-handler/src/error.rs b/core-primitives/stf-state-handler/src/error.rs index bba2db2d1b..c990238552 100644 --- a/core-primitives/stf-state-handler/src/error.rs +++ b/core-primitives/stf-state-handler/src/error.rs @@ -24,13 +24,25 @@ use rust_base58::base58::FromBase58Error; #[cfg(feature = "sgx")] use base58::FromBase58Error; +use crate::state_snapshot_primitives::StateId; +use itp_types::ShardIdentifier; use sgx_types::sgx_status_t; -use std::{boxed::Box, format}; +use std::{boxed::Box, format, string::String}; pub type Result = core::result::Result; #[derive(Debug, thiserror::Error)] pub enum Error { + #[error("Empty state repository")] + EmptyRepository, + #[error("State ID is invalid and does not exist: {0}")] + InvalidStateId(StateId), + #[error("Shard is invalid and does not exist: {0}")] + InvalidShard(ShardIdentifier), + #[error("State with hash {0} could not be found in the state repository")] + StateNotFoundInRepository(String), + #[error("Cache size for registry is zero")] + ZeroCacheSize, #[error("Could not acquire lock, lock is poisoned")] LockPoisoning, #[error("OsString conversion error")] diff --git a/core-primitives/stf-state-handler/src/file_io.rs b/core-primitives/stf-state-handler/src/file_io.rs index 58f3923bb8..bfeac8e290 100644 --- a/core-primitives/stf-state-handler/src/file_io.rs +++ b/core-primitives/stf-state-handler/src/file_io.rs @@ -18,134 +18,373 @@ #[cfg(all(not(feature = "std"), feature = "sgx"))] use crate::sgx_reexport_prelude::*; -use crate::error::{Error, Result}; -use base58::{FromBase58, ToBase58}; -use codec::{Decode, Encode}; -use ita_stf::{State as StfState, StateType as StfStateType, Stf}; -use itp_settings::files::{ENCRYPTED_STATE_FILE, SHARDS_PATH}; -use itp_sgx_crypto::{AesSeal, StateCrypto}; -use itp_sgx_io::{read as io_read, write as io_write, SealedIO}; -use itp_types::{ShardIdentifier, H256}; -use log::*; -use sgx_tcrypto::rsgx_sha256_slice; -use sgx_types::sgx_status_t; -use std::{format, fs, io::Write, path::Path, vec::Vec}; - -pub(crate) fn load_initialized_state(shard: &ShardIdentifier) -> Result { - trace!("Loading state from shard {:?}", shard); - let state = if exists(&shard) { - load(&shard)? - } else { - trace!("Initialize new shard: {:?}", shard); - init_shard(&shard)?; - Stf::init_state() - }; - trace!("Successfully loaded or initialized state from shard {:?}", shard); - Ok(state) -} +#[cfg(any(test, feature = "std"))] +use rust_base58::base58::ToBase58; -pub(crate) fn load(shard: &ShardIdentifier) -> Result { - // load last state - let state_path = - format!("{}/{}/{}", SHARDS_PATH, shard.encode().to_base58(), ENCRYPTED_STATE_FILE); - trace!("loading state from: {}", state_path); - let state_vec = read(&state_path)?; - - // state is now decrypted! - let state: StfStateType = match state_vec.len() { - 0 => { - debug!("state at {} is empty. will initialize it.", state_path); - Stf::init_state().state - }, - n => { - debug!("State loaded from {} with size {}B, deserializing...", state_path, n); - StfStateType::decode(&mut state_vec.as_slice())? - }, - }; - trace!("state decoded successfully"); - // add empty state-diff - let state_with_diff = StfState { state, state_diff: Default::default() }; - trace!("New state created: {:?}", state_with_diff); - Ok(state_with_diff) -} +#[cfg(feature = "sgx")] +use base58::ToBase58; -/// Writes the state (without the state diff) encrypted into the enclave storage -/// Returns the hash of the saved state (independent of the diff!) -pub(crate) fn write(state: StfState, shard: &ShardIdentifier) -> Result { - let state_path = - format!("{}/{}/{}", SHARDS_PATH, shard.encode().to_base58(), ENCRYPTED_STATE_FILE); - trace!("writing state to: {}", state_path); +#[cfg(any(test, feature = "sgx"))] +use itp_settings::files::ENCRYPTED_STATE_FILE; - // only save the state, the state diff is pruned - let cyphertext = encrypt(state.state.encode())?; +#[cfg(any(test, feature = "sgx"))] +use std::string::String; - let state_hash = rsgx_sha256_slice(&cyphertext)?; +use crate::{error::Result, state_snapshot_primitives::StateId}; +use codec::Encode; +use itp_settings::files::SHARDS_PATH; +use itp_types::ShardIdentifier; +use log::error; +use std::{format, path::PathBuf, vec::Vec}; - debug!("new encrypted state with hash={:?} written to {}", state_hash, state_path); +/// Trait to abstract file I/O for state. +pub trait StateFileIo { + type StateType; + type HashType; - io_write(&cyphertext, &state_path)?; - Ok(state_hash.into()) -} + /// Load a state (returns error if it does not exist). + fn load( + &self, + shard_identifier: &ShardIdentifier, + state_id: StateId, + ) -> Result; -pub(crate) fn exists(shard: &ShardIdentifier) -> bool { - Path::new(&format!("{}/{}/{}", SHARDS_PATH, shard.encode().to_base58(), ENCRYPTED_STATE_FILE)) - .exists() -} + /// Compute the state hash of a specific state (returns error if it does not exist). + fn compute_hash( + &self, + shard_identifier: &ShardIdentifier, + state_id: StateId, + ) -> Result; + + /// Create an empty (default initialized) state. + fn create_initialized( + &self, + shard_identifier: &ShardIdentifier, + state_id: StateId, + ) -> Result; + + /// Write the state. + fn write( + &self, + shard_identifier: &ShardIdentifier, + state_id: StateId, + state: Self::StateType, + ) -> Result; + + /// Remove a state. + fn remove(&self, shard_identifier: &ShardIdentifier, state_id: StateId) -> Result<()>; + + /// Checks if a given shard directory exists and contains at least one state instance. + fn shard_exists(&self, shard_identifier: &ShardIdentifier) -> bool; -pub(crate) fn init_shard(shard: &ShardIdentifier) -> Result<()> { - let path = format!("{}/{}", SHARDS_PATH, shard.encode().to_base58()); - fs::create_dir_all(path.clone())?; - let mut file = fs::File::create(format!("{}/{}", path, ENCRYPTED_STATE_FILE))?; - Ok(file.write_all(b"")?) + /// Lists all shards. + fn list_shards(&self) -> Result>; + + /// List all states for a shard. + fn list_state_ids_for_shard(&self, shard_identifier: &ShardIdentifier) -> Result>; } -pub(crate) fn read(path: &str) -> Result> { - let mut bytes = io_read(path)?; +#[cfg(feature = "sgx")] +pub mod sgx { + + use super::*; + use crate::{error::Error, state_key_repository::AccessStateKey}; + use base58::FromBase58; + use codec::Decode; + use ita_stf::{State as StfState, StateType as StfStateType, Stf}; + use itp_sgx_crypto::StateCrypto; + use itp_sgx_io::{read as io_read, write as io_write}; + use itp_types::H256; + use log::*; + use sgx_tcrypto::rsgx_sha256_slice; + use std::{fs, path::Path, sync::Arc}; - if bytes.is_empty() { - return Ok(bytes) + /// SGX state file I/O. + pub struct SgxStateFileIo { + state_key_repository: Arc, } - let state_hash = rsgx_sha256_slice(&bytes)?; - debug!( - "read encrypted state with hash {:?} from {}", - H256::from_slice(state_hash.as_ref()), - path - ); + impl SgxStateFileIo + where + StateKeyRepository: AccessStateKey, + { + pub fn new(state_key_repository: Arc) -> Self { + SgxStateFileIo { state_key_repository } + } + + fn read(&self, path: &Path) -> Result> { + let mut bytes = io_read(path)?; + + if bytes.is_empty() { + return Ok(bytes) + } + + let state_hash = rsgx_sha256_slice(&bytes)?; + debug!( + "read encrypted state with hash {:?} from {:?}", + H256::from_slice(state_hash.as_ref()), + path + ); + + let state_key = self.state_key_repository.retrieve_key()?; + + state_key + .decrypt(&mut bytes) + .map_err(|e| Error::Other(format!("{:?}", e).into()))?; + trace!("buffer decrypted = {:?}", bytes); + + Ok(bytes) + } + + fn encrypt(&self, mut state: Vec) -> Result> { + let state_key = self.state_key_repository.retrieve_key()?; + + state_key + .encrypt(&mut state) + .map_err(|e| Error::Other(format!("{:?}", e).into()))?; + Ok(state) + } + } + + impl StateFileIo for SgxStateFileIo + where + StateKey: AccessStateKey, + { + type StateType = StfState; + type HashType = H256; + + fn load( + &self, + shard_identifier: &ShardIdentifier, + state_id: StateId, + ) -> Result { + if !file_for_state_exists(shard_identifier, state_id) { + return Err(Error::InvalidStateId(state_id)) + } + + let state_path = state_file_path(shard_identifier, state_id); + trace!("loading state from: {:?}", state_path); + let state_encoded = self.read(&state_path)?; + + // State is now decrypted. + debug!( + "State loaded from {:?} with size {}B, deserializing...", + state_path, + state_encoded.len() + ); + let state = StfStateType::decode(&mut state_encoded.as_slice())?; + + trace!("state decoded successfully"); + // Add empty state-diff. + let state_with_diff = StfState { state, state_diff: Default::default() }; + trace!("New state created: {:?}", state_with_diff); + Ok(state_with_diff) + } + + fn compute_hash( + &self, + shard_identifier: &ShardIdentifier, + state_id: StateId, + ) -> Result { + if !file_for_state_exists(shard_identifier, state_id) { + return Err(Error::InvalidStateId(state_id)) + } + + let state_file_path = state_file_path(shard_identifier, state_id); + let bytes = io_read(state_file_path)?; + let state_hash = rsgx_sha256_slice(&bytes)?; + Ok(H256::from_slice(state_hash.as_ref())) + } + + fn create_initialized( + &self, + shard_identifier: &ShardIdentifier, + state_id: StateId, + ) -> Result { + init_shard(&shard_identifier)?; + let state = Stf::init_state(); + self.write(shard_identifier, state_id, state) + } + + /// Writes the state (without the state diff) encrypted into the enclave storage. + /// Returns the hash of the saved state (independent of the diff!). + fn write( + &self, + shard_identifier: &ShardIdentifier, + state_id: StateId, + state: Self::StateType, + ) -> Result { + let state_path = state_file_path(shard_identifier, state_id); + trace!("writing state to: {:?}", state_path); + + // Only save the state, the state diff is pruned. + let cyphertext = self.encrypt(state.state.encode())?; + + let state_hash = rsgx_sha256_slice(&cyphertext)?; + + debug!("new encrypted state with hash={:?} written to {:?}", state_hash, state_path); - AesSeal::unseal().map(|key| key.decrypt(&mut bytes))??; - trace!("buffer decrypted = {:?}", bytes); + io_write(&cyphertext, &state_path)?; + Ok(state_hash.into()) + } - Ok(bytes) + fn remove(&self, shard_identifier: &ShardIdentifier, state_id: StateId) -> Result<()> { + fs::remove_file(state_file_path(shard_identifier, state_id)) + .map_err(|e| Error::Other(e.into())) + } + + fn shard_exists(&self, shard_identifier: &ShardIdentifier) -> bool { + shard_exists(shard_identifier) + } + + fn list_shards(&self) -> Result> { + list_shards() + } + + fn list_state_ids_for_shard( + &self, + shard_identifier: &ShardIdentifier, + ) -> Result> { + let shard_path = shard_path(shard_identifier); + let directory_items = list_items_in_directory(&shard_path); + + Ok(directory_items + .iter() + .flat_map(|item| { + let maybe_state_id = extract_state_id_from_file_name(item.as_str()); + if maybe_state_id.is_none() { + warn!("Found item ({}) that does not match state snapshot naming pattern, ignoring it", item) + } + maybe_state_id + }) + .collect()) + } + } + + fn state_file_path(shard: &ShardIdentifier, state_id: StateId) -> PathBuf { + let mut shard_file_path = shard_path(shard); + shard_file_path.push(to_file_name(state_id)); + shard_file_path + } + + fn file_for_state_exists(shard: &ShardIdentifier, state_id: StateId) -> bool { + state_file_path(shard, state_id).exists() + } + + /// Returns true if a shard directory for a given identifier exists AND contains at least one state file. + pub(crate) fn shard_exists(shard: &ShardIdentifier) -> bool { + let shard_path = shard_path(shard); + if !shard_path.exists() { + return false + } + + shard_path + .read_dir() + // When the iterator over all files in the directory returns none, the directory is empty. + .map(|mut d| d.next().is_some()) + .unwrap_or(false) + } + + pub(crate) fn init_shard(shard: &ShardIdentifier) -> Result<()> { + let path = shard_path(shard); + fs::create_dir_all(path).map_err(|e| Error::Other(e.into())) + } + + /// List any valid shards that are found in the shard path. + /// Ignore any items (files, directories) that are not valid shard identifiers. + pub(crate) fn list_shards() -> Result> { + let directory_items = list_items_in_directory(&PathBuf::from(SHARDS_PATH)); + Ok(directory_items + .iter() + .flat_map(|item| { + item.from_base58() + .ok() + .map(|encoded_shard_id| { + ShardIdentifier::decode(&mut encoded_shard_id.as_slice()).ok() + }) + .flatten() + }) + .collect()) + } + + fn list_items_in_directory(directory: &Path) -> Vec { + let items = match directory.read_dir() { + Ok(rd) => rd, + Err(_) => return Vec::new(), + }; + + items + .flat_map(|fr| fr.map(|de| de.file_name().into_string().ok()).ok().flatten()) + .collect() + } } -#[allow(unused)] -fn write_encrypted(bytes: &mut Vec, path: &str) -> Result { - debug!("plaintext data to be written: {:?}", bytes); - AesSeal::unseal().map(|key| key.encrypt(bytes))?; - io_write(&bytes, path)?; - Ok(sgx_status_t::SGX_SUCCESS) +/// Remove a shard directory with all of its content. +pub fn purge_shard_dir(shard: &ShardIdentifier) { + let shard_dir_path = shard_path(shard); + if let Err(e) = std::fs::remove_dir_all(&shard_dir_path) { + error!("Failed to remove shard directory {:?}: {:?}", shard_dir_path, e); + } +} + +pub(crate) fn shard_path(shard: &ShardIdentifier) -> PathBuf { + PathBuf::from(format!("{}/{}", SHARDS_PATH, shard.encode().to_base58())) } -pub(crate) fn encrypt(mut state: Vec) -> Result> { - AesSeal::unseal().map(|key| key.encrypt(&mut state))??; - Ok(state) +#[cfg(any(test, feature = "sgx"))] +fn to_file_name(state_id: StateId) -> String { + format!("{}_{}", state_id, ENCRYPTED_STATE_FILE) } -pub(crate) fn list_shards() -> Result> { - let files = match fs::read_dir(SHARDS_PATH) { - Ok(f) => f, - Err(_) => return Ok(Vec::new()), - }; - let mut shards = Vec::new(); - for file_result in files { - let s = file_result? - .file_name() - .into_string() - .map_err(|_| Error::OsStringConversion)? - .from_base58()?; - - shards.push(ShardIdentifier::decode(&mut s.as_slice())?); +#[cfg(any(test, feature = "sgx"))] +fn extract_state_id_from_file_name(file_name: &str) -> Option { + let state_id_str = file_name.strip_suffix(format!("_{}", ENCRYPTED_STATE_FILE).as_str())?; + state_id_str.parse::().ok() +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::state_snapshot_primitives::generate_current_timestamp_state_id; + + #[test] + fn state_id_to_file_name_works() { + assert!(to_file_name(generate_current_timestamp_state_id()).ends_with(ENCRYPTED_STATE_FILE)); + assert!(to_file_name(generate_current_timestamp_state_id()) + .strip_suffix(format!("_{}", ENCRYPTED_STATE_FILE).as_str()) + .is_some()); + + let now_time_stamp = generate_current_timestamp_state_id(); + assert_eq!( + extract_state_id_from_file_name(to_file_name(now_time_stamp).as_str()).unwrap(), + now_time_stamp + ); + } + + #[test] + fn extract_timestamp_from_file_name_works() { + assert_eq!( + 123456u128, + extract_state_id_from_file_name(format!("123456_{}", ENCRYPTED_STATE_FILE).as_str()) + .unwrap() + ); + assert_eq!( + 0u128, + extract_state_id_from_file_name(format!("0_{}", ENCRYPTED_STATE_FILE).as_str()) + .unwrap() + ); + + assert!(extract_state_id_from_file_name( + format!("987345{}", ENCRYPTED_STATE_FILE).as_str() + ) + .is_none()); + assert!( + extract_state_id_from_file_name(format!("{}", ENCRYPTED_STATE_FILE).as_str()).is_none() + ); + assert!(extract_state_id_from_file_name( + format!("1234_{}-other", ENCRYPTED_STATE_FILE).as_str() + ) + .is_none()); } - Ok(shards) } diff --git a/core-primitives/stf-state-handler/src/global_file_state_handler.rs b/core-primitives/stf-state-handler/src/global_file_state_handler.rs deleted file mode 100644 index 03933e6037..0000000000 --- a/core-primitives/stf-state-handler/src/global_file_state_handler.rs +++ /dev/null @@ -1,83 +0,0 @@ -/* - Copyright 2021 Integritee AG and Supercomputing Systems AG - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ - -#[cfg(feature = "sgx")] -use std::sync::{SgxRwLock as RwLock, SgxRwLockWriteGuard as RwLockWriteGuard}; - -#[cfg(feature = "std")] -use std::sync::{RwLock, RwLockWriteGuard}; - -use crate::{ - error::{Error, Result}, - file_io::{exists, list_shards, load_initialized_state, write as state_write}, - handle_state::HandleState, - query_shard_state::QueryShardState, -}; -use ita_stf::State as StfState; -use itp_types::{ShardIdentifier, H256}; -use lazy_static::lazy_static; -use std::vec::Vec; - -lazy_static! { - // as long as we have a file backend, we use this 'dummy' lock, - // which guards against concurrent read/write access - pub static ref STF_STATE_LOCK: RwLock<()> = Default::default(); -} - -/// Implementation of the `HandleState` trait using global files and locks. -/// -/// For each call it will make a file access and encrypt/decrypt the state from file I/O. -/// The lock it uses is therefore an 'empty' dummy lock, that guards against concurrent file access. -pub struct GlobalFileStateHandler; - -impl HandleState for GlobalFileStateHandler { - type WriteLockPayload = (); - type StateT = StfState; - - fn load_initialized(&self, shard: &ShardIdentifier) -> Result { - let _state_read_lock = STF_STATE_LOCK.read().map_err(|_| Error::LockPoisoning)?; - load_initialized_state(shard) - } - - fn load_for_mutation( - &self, - shard: &ShardIdentifier, - ) -> Result<(RwLockWriteGuard<'_, Self::WriteLockPayload>, Self::StateT)> { - let state_write_lock = STF_STATE_LOCK.write().map_err(|_| Error::LockPoisoning)?; - let loaded_state = load_initialized_state(shard)?; - Ok((state_write_lock, loaded_state)) - } - - fn write( - &self, - state: Self::StateT, - _state_lock: RwLockWriteGuard<'_, Self::WriteLockPayload>, - shard: &ShardIdentifier, - ) -> Result { - state_write(state, shard) - } -} - -impl QueryShardState for GlobalFileStateHandler { - fn exists(&self, shard: &ShardIdentifier) -> bool { - exists(shard) - } - - fn list_shards(&self) -> Result> { - list_shards() - } -} diff --git a/core-primitives/stf-state-handler/src/handle_state.rs b/core-primitives/stf-state-handler/src/handle_state.rs index 419849758f..c6b8702610 100644 --- a/core-primitives/stf-state-handler/src/handle_state.rs +++ b/core-primitives/stf-state-handler/src/handle_state.rs @@ -22,20 +22,25 @@ use std::sync::SgxRwLockWriteGuard as RwLockWriteGuard; use std::sync::RwLockWriteGuard; use crate::error::Result; -use itp_types::{ShardIdentifier, H256}; +use itp_types::ShardIdentifier; -/// Facade for handling STF state loading and storing (e.g. from file) +/// Facade for handling STF state loading and storing (e.g. from file). pub trait HandleState { type WriteLockPayload; type StateT; + type HashType; - /// Load the state for a given shard + /// Initialize a new shard. /// - /// Initializes the shard and state if necessary, so this is guaranteed to - /// return a state - fn load_initialized(&self, shard: &ShardIdentifier) -> Result; + /// Initializes a default state for the shard and returns its hash. + fn initialize_shard(&self, shard: ShardIdentifier) -> Result; - /// Load the state in order to mutate it + /// Load the state for a given shard. + /// + /// Requires the shard to exist and be initialized, otherwise returns an error. + fn load(&self, shard: &ShardIdentifier) -> Result; + + /// Load the state in order to mutate it. /// /// Returns a write lock to protect against any concurrent access as long as /// the lock is held. Finalize the operation by calling `write` and returning @@ -45,13 +50,18 @@ pub trait HandleState { shard: &ShardIdentifier, ) -> Result<(RwLockWriteGuard<'_, Self::WriteLockPayload>, Self::StateT)>; - /// Writes the state (without the state diff) encrypted into the enclave + /// Writes the state (without the state diff) encrypted into the enclave. /// - /// Returns the hash of the saved state (independent of the diff!) - fn write( + /// Returns the hash of the saved state (independent of the diff!). + fn write_after_mutation( &self, state: Self::StateT, state_lock: RwLockWriteGuard<'_, Self::WriteLockPayload>, shard: &ShardIdentifier, - ) -> Result; + ) -> Result; + + /// Reset (or override) a state. + /// + /// Use in cases where the previous state is of no interest. Otherwise use `load_for_mutation` and `write_after_mutation`. + fn reset(&self, state: Self::StateT, shard: &ShardIdentifier) -> Result; } diff --git a/core-primitives/stf-state-handler/src/in_memory_state_file_io.rs b/core-primitives/stf-state-handler/src/in_memory_state_file_io.rs new file mode 100644 index 0000000000..f9dfb921ed --- /dev/null +++ b/core-primitives/stf-state-handler/src/in_memory_state_file_io.rs @@ -0,0 +1,380 @@ +/* + Copyright 2021 Integritee AG and Supercomputing Systems AG + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +#[cfg(feature = "sgx")] +use std::sync::SgxRwLock as RwLock; + +#[cfg(feature = "std")] +use std::sync::RwLock; + +use crate::{ + error::{Error, Result}, + file_io::StateFileIo, + state_snapshot_primitives::StateId, +}; +use codec::Encode; +use itp_types::ShardIdentifier; +use std::{collections::HashMap, hash::Hasher as HasherTrait, vec::Vec}; + +type StateHash = u64; +type ShardDirectory = HashMap; +type ShardsRootDirectory = HashMap>; + +/// State file I/O using (unencrypted) in-memory representation of the state files. +/// Uses u64 hash type. Can be used as mock for testing. +#[derive(Default)] +pub struct InMemoryStateFileIo +where + State: Clone + Default + Encode, + Hasher: HasherTrait + Clone + Default, +{ + emulated_shard_directory: RwLock>, + hasher: Hasher, +} + +impl InMemoryStateFileIo +where + State: Clone + Default + Encode, + Hasher: HasherTrait + Clone + Default, +{ + #[allow(unused)] + pub fn new(hash_function: Hasher, shards: &[ShardIdentifier]) -> Self { + let shard_hash_map: HashMap<_, _> = + shards.iter().map(|s| (*s, ShardDirectory::::default())).collect(); + + InMemoryStateFileIo { + emulated_shard_directory: RwLock::new(shard_hash_map), + hasher: hash_function, + } + } + + #[cfg(test)] + pub fn get_states_for_shard( + &self, + shard_identifier: &ShardIdentifier, + ) -> Result> { + let files_lock = self.emulated_shard_directory.read().map_err(|_| Error::LockPoisoning)?; + files_lock + .get(shard_identifier) + .cloned() + .ok_or_else(|| Error::InvalidShard(*shard_identifier)) + } + + fn compute_state_hash(&self, state: &State) -> StateHash { + let encoded_state = state.encode(); + let mut hasher = self.hasher.clone(); + hasher.write(encoded_state.as_slice()); + hasher.finish() + } + + fn default_states_map(&self, state_id: StateId) -> ShardDirectory { + self.initialize_states_map(state_id, State::default()) + } + + fn initialize_states_map(&self, state_id: StateId, state: State) -> ShardDirectory { + HashMap::from([(state_id, self.generate_state_entry(state))]) + } + + fn generate_default_state_entry(&self) -> (StateHash, State) { + self.generate_state_entry(State::default()) + } + + fn generate_state_entry(&self, state: State) -> (StateHash, State) { + let state_hash = self.compute_state_hash(&state); + (state_hash, state) + } +} + +impl StateFileIo for InMemoryStateFileIo +where + State: Clone + Default + Encode, + Hasher: HasherTrait + Clone + Default, +{ + type StateType = State; + type HashType = StateHash; + + fn load( + &self, + shard_identifier: &ShardIdentifier, + state_id: StateId, + ) -> Result { + let directory_lock = + self.emulated_shard_directory.read().map_err(|_| Error::LockPoisoning)?; + let states_for_shard = directory_lock + .get(shard_identifier) + .ok_or_else(|| Error::InvalidShard(*shard_identifier))?; + states_for_shard + .get(&state_id) + .map(|(_, s)| -> State { s.clone() }) + .ok_or(Error::InvalidStateId(state_id)) + } + + fn compute_hash( + &self, + shard_identifier: &ShardIdentifier, + state_id: StateId, + ) -> Result { + let state = self.load(shard_identifier, state_id)?; + Ok(self.compute_state_hash(&state)) + } + + fn create_initialized( + &self, + shard_identifier: &ShardIdentifier, + state_id: StateId, + ) -> Result { + let mut directory_lock = + self.emulated_shard_directory.write().map_err(|_| Error::LockPoisoning)?; + let states_for_shard = directory_lock + .entry(*shard_identifier) + .or_insert_with(|| self.default_states_map(state_id)); + let state_entry = states_for_shard + .entry(state_id) + .or_insert_with(|| self.generate_state_entry(State::default())); + Ok(state_entry.0) + } + + fn write( + &self, + shard_identifier: &ShardIdentifier, + state_id: StateId, + state: Self::StateType, + ) -> Result { + let mut directory_lock = + self.emulated_shard_directory.write().map_err(|_| Error::LockPoisoning)?; + + let states_for_shard = directory_lock + .entry(*shard_identifier) + .or_insert_with(|| self.default_states_map(state_id)); + + let state_hash = self.compute_state_hash(&state); + *states_for_shard + .entry(state_id) + .or_insert_with(|| self.generate_default_state_entry()) = (state_hash, state); + + Ok(state_hash) + } + + fn remove(&self, shard_identifier: &ShardIdentifier, state_id: StateId) -> Result<()> { + let mut directory_lock = + self.emulated_shard_directory.write().map_err(|_| Error::LockPoisoning)?; + + let states_for_shard = directory_lock + .get_mut(shard_identifier) + .ok_or_else(|| Error::InvalidShard(*shard_identifier))?; + + states_for_shard + .remove(&state_id) + .ok_or(Error::InvalidStateId(state_id)) + .map(|_| {}) + } + + fn shard_exists(&self, shard_identifier: &ShardIdentifier) -> bool { + let directory_lock = self.emulated_shard_directory.read().unwrap(); + directory_lock.contains_key(shard_identifier) + } + + fn list_shards(&self) -> Result> { + let directory_lock = + self.emulated_shard_directory.read().map_err(|_| Error::LockPoisoning)?; + Ok(directory_lock.keys().copied().collect()) + } + + fn list_state_ids_for_shard(&self, shard_identifier: &ShardIdentifier) -> Result> { + let directory_lock = + self.emulated_shard_directory.read().map_err(|_| Error::LockPoisoning)?; + let shard_directory = directory_lock + .get(shard_identifier) + .ok_or_else(|| Error::InvalidShard(*shard_identifier))?; + Ok(shard_directory.keys().cloned().collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::{assert_matches::assert_matches, collections::hash_map::DefaultHasher}; + + type TestState = u64; + type TestStateFileIo = InMemoryStateFileIo; + + #[test] + fn shard_directory_is_empty_after_initialization() { + let state_file_io = create_empty_in_memory_state_file_io(); + assert!(state_file_io.list_shards().unwrap().is_empty()); + } + + #[test] + fn load_on_empty_directory_and_shard_returns_error() { + let state_file_io = create_empty_in_memory_state_file_io(); + + assert_matches!( + state_file_io.load(&ShardIdentifier::random(), 1234), + Err(Error::InvalidShard(_)) + ); + } + + #[test] + fn initialize_with_shard_creates_empty_directory() { + let shard = ShardIdentifier::from([2u8; 32]); + let state_file_io = create_in_memory_state_file_io(&[shard]); + + assert!(state_file_io.list_state_ids_for_shard(&shard).unwrap().is_empty()); + assert!(state_file_io + .list_state_ids_for_shard(&ShardIdentifier::from([3u8; 32])) + .is_err()); + } + + #[test] + fn load_when_state_does_not_exist_returns_error() { + let state_file_io = create_empty_in_memory_state_file_io(); + let shard_id = ShardIdentifier::random(); + let _ = state_file_io.create_initialized(&shard_id, 1234).unwrap(); + + assert_matches!(state_file_io.load(&shard_id, 12345), Err(Error::InvalidStateId(12345))); + } + + #[test] + fn create_initialized_when_shard_already_exists_works() { + let shard = ShardIdentifier::random(); + let state_file_io = create_in_memory_state_file_io(&[shard]); + + assert!(state_file_io.create_initialized(&shard, 1245).is_ok()); + } + + #[test] + fn create_initialized_adds_default_state() { + let state_file_io = create_empty_in_memory_state_file_io(); + let shard_id = ShardIdentifier::random(); + let state_id = 31081984u128; + let state_hash = state_file_io.create_initialized(&shard_id, state_id).unwrap(); + + assert_eq!(1, state_file_io.list_shards().unwrap().len()); + assert_eq!(TestState::default(), state_file_io.load(&shard_id, state_id).unwrap()); + assert_eq!(1, state_file_io.list_state_ids_for_shard(&shard_id).unwrap().len()); + + assert_entry(&state_file_io, &shard_id, state_id, &StateHash::default(), &state_hash); + } + + #[test] + fn write_works_when_no_previous_shard_or_file_exists() { + let state_file_io = create_empty_in_memory_state_file_io(); + let shard_id = ShardIdentifier::random(); + let state_id = 23u128; + let test_state = 42u64; + + let state_hash = state_file_io.write(&shard_id, state_id, test_state).unwrap(); + + assert_eq!(1, state_file_io.list_shards().unwrap().len()); + assert_eq!(test_state, state_file_io.load(&shard_id, state_id).unwrap()); + assert_eq!(1, state_file_io.list_state_ids_for_shard(&shard_id).unwrap().len()); + assert_entry(&state_file_io, &shard_id, state_id, &test_state, &state_hash); + } + + #[test] + fn write_overwrites_existing_state() { + let state_file_io = create_empty_in_memory_state_file_io(); + let shard_id = ShardIdentifier::random(); + let state_id = 123456u128; + let _ = state_file_io.create_initialized(&shard_id, state_id).unwrap(); + + let test_state = 4256u64; + let state_hash = state_file_io.write(&shard_id, state_id, test_state).unwrap(); + + assert_eq!(1, state_file_io.list_shards().unwrap().len()); + assert_eq!(test_state, state_file_io.load(&shard_id, state_id).unwrap()); + assert_eq!(1, state_file_io.list_state_ids_for_shard(&shard_id).unwrap().len()); + assert_entry(&state_file_io, &shard_id, state_id, &test_state, &state_hash); + } + + #[test] + fn remove_files_works() { + let state_file_io = create_empty_in_memory_state_file_io(); + let shard_id = ShardIdentifier::random(); + let initial_state_id = 42u128; + let _ = state_file_io.create_initialized(&shard_id, initial_state_id).unwrap(); + + let state_ids = vec![1u128, 2u128, 3u128]; + + for state_id in state_ids.iter() { + let _ = state_file_io.write(&shard_id, *state_id, 987345).unwrap(); + } + + let mut expected_size = state_ids.len() + 1; + assert_eq!(expected_size, state_file_io.list_state_ids_for_shard(&shard_id).unwrap().len()); + expected_size -= 1; + + for state_id in state_ids.iter() { + state_file_io.remove(&shard_id, *state_id).unwrap(); + assert_matches!( + state_file_io.load(&shard_id, *state_id), + Err(Error::InvalidStateId(_)) + ); + assert_eq!( + expected_size, + state_file_io.list_state_ids_for_shard(&shard_id).unwrap().len() + ); + expected_size -= 1; + } + } + + #[test] + fn initialize_with_shards_creates_empty_maps() { + let shards = vec![ShardIdentifier::random(), ShardIdentifier::random()]; + let state_file_io = create_in_memory_state_file_io(shards.as_slice()); + + assert_eq!(shards.len(), state_file_io.list_shards().unwrap().len()); + for shard in shards { + assert!(state_file_io.list_state_ids_for_shard(&shard).unwrap().is_empty()); + } + } + + fn assert_entry( + state_file_io: &TestStateFileIo, + shard_id: &ShardIdentifier, + state_id: StateId, + state: &TestState, + state_hash: &StateHash, + ) { + let (retrieved_hash, retrieved_state) = + get_state_entry(&state_file_io, &shard_id, state_id); + assert!(state_file_io.shard_exists(shard_id)); + assert_eq!(state_hash, &retrieved_hash); + assert_eq!(state, &retrieved_state); + } + + fn get_state_entry( + state_file_io: &TestStateFileIo, + shard_id: &ShardIdentifier, + state_id: StateId, + ) -> (StateHash, TestState) { + state_file_io + .get_states_for_shard(shard_id) + .unwrap() + .get(&state_id) + .unwrap() + .clone() + } + + fn create_in_memory_state_file_io(shards: &[ShardIdentifier]) -> TestStateFileIo { + InMemoryStateFileIo::new(DefaultHasher::default(), shards) + } + + fn create_empty_in_memory_state_file_io() -> TestStateFileIo { + create_in_memory_state_file_io(&[]) + } +} diff --git a/core-primitives/stf-state-handler/src/lib.rs b/core-primitives/stf-state-handler/src/lib.rs index 8b44cf689b..492522ef24 100644 --- a/core-primitives/stf-state-handler/src/lib.rs +++ b/core-primitives/stf-state-handler/src/lib.rs @@ -16,6 +16,7 @@ */ #![cfg_attr(not(feature = "std"), no_std)] +#![feature(assert_matches)] #[cfg(all(feature = "std", feature = "sgx"))] compile_error!("feature \"std\" and feature \"sgx\" cannot be enabled at the same time"); @@ -31,17 +32,15 @@ pub mod sgx_reexport_prelude { } pub mod error; +pub mod file_io; pub mod handle_state; +mod in_memory_state_file_io; pub mod query_shard_state; - -#[cfg(feature = "sgx")] -pub mod global_file_state_handler; - -#[cfg(feature = "sgx")] -pub use global_file_state_handler::GlobalFileStateHandler; - -#[cfg(feature = "sgx")] -mod file_io; - -#[cfg(all(feature = "test", feature = "sgx"))] -pub mod tests; +pub mod state_handler; +pub mod state_key_repository; +mod state_snapshot_primitives; +pub mod state_snapshot_repository; +pub mod state_snapshot_repository_loader; +pub mod test; + +pub use state_handler::StateHandler; diff --git a/core-primitives/stf-state-handler/src/query_shard_state.rs b/core-primitives/stf-state-handler/src/query_shard_state.rs index 6d5d449a2c..11ff46d044 100644 --- a/core-primitives/stf-state-handler/src/query_shard_state.rs +++ b/core-primitives/stf-state-handler/src/query_shard_state.rs @@ -25,7 +25,7 @@ use std::vec::Vec; /// SGX exclusive data structures (feature sgx) pub trait QueryShardState { /// Query whether a given shard exists - fn exists(&self, shard: &ShardIdentifier) -> bool; + fn shard_exists(&self, shard: &ShardIdentifier) -> Result; /// List all available shards fn list_shards(&self) -> Result>; diff --git a/core-primitives/stf-state-handler/src/state_handler.rs b/core-primitives/stf-state-handler/src/state_handler.rs new file mode 100644 index 0000000000..b4b8010237 --- /dev/null +++ b/core-primitives/stf-state-handler/src/state_handler.rs @@ -0,0 +1,180 @@ +/* + Copyright 2021 Integritee AG and Supercomputing Systems AG + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +#[cfg(feature = "sgx")] +use std::sync::{SgxRwLock as RwLock, SgxRwLockWriteGuard as RwLockWriteGuard}; + +#[cfg(feature = "std")] +use std::sync::{RwLock, RwLockWriteGuard}; + +use crate::{ + error::{Error, Result}, + handle_state::HandleState, + query_shard_state::QueryShardState, + state_snapshot_repository::VersionedStateAccess, +}; +use itp_types::ShardIdentifier; +use std::vec::Vec; + +/// Implementation of the `HandleState` trait. +/// +/// It's a concurrency wrapper around a state snapshot repository, which handles +/// access to any shards and state files. The state handler ensures we have thread-safe +/// concurrent access to that repository. +pub struct StateHandler { + state_snapshot_repository: RwLock, +} + +impl StateHandler { + pub fn new(state_snapshot_repository: Repository) -> Self { + StateHandler { state_snapshot_repository: RwLock::new(state_snapshot_repository) } + } +} + +impl HandleState for StateHandler +where + Repository: VersionedStateAccess, +{ + type WriteLockPayload = Repository; + type StateT = Repository::StateType; + type HashType = Repository::HashType; + + fn initialize_shard(&self, shard: ShardIdentifier) -> Result { + let mut state_write_lock = + self.state_snapshot_repository.write().map_err(|_| Error::LockPoisoning)?; + state_write_lock.initialize_new_shard(shard) + } + + fn load(&self, shard: &ShardIdentifier) -> Result { + self.state_snapshot_repository + .read() + .map_err(|_| Error::LockPoisoning)? + .load_latest(shard) + } + + fn load_for_mutation( + &self, + shard: &ShardIdentifier, + ) -> Result<(RwLockWriteGuard<'_, Self::WriteLockPayload>, Self::StateT)> { + let state_write_lock = + self.state_snapshot_repository.write().map_err(|_| Error::LockPoisoning)?; + let loaded_state = state_write_lock.load_latest(shard)?; + Ok((state_write_lock, loaded_state)) + } + + fn write_after_mutation( + &self, + state: Self::StateT, + mut state_lock: RwLockWriteGuard<'_, Self::WriteLockPayload>, + shard: &ShardIdentifier, + ) -> Result { + state_lock.update(shard, state) + } + + fn reset(&self, state: Self::StateT, shard: &ShardIdentifier) -> Result { + let mut state_write_lock = + self.state_snapshot_repository.write().map_err(|_| Error::LockPoisoning)?; + + state_write_lock.update(shard, state) + } +} + +impl QueryShardState for StateHandler +where + Repository: VersionedStateAccess, +{ + fn shard_exists(&self, shard: &ShardIdentifier) -> Result { + let registry_lock = + self.state_snapshot_repository.read().map_err(|_| Error::LockPoisoning)?; + + Ok(registry_lock.shard_exists(shard)) + } + + fn list_shards(&self) -> Result> { + self.state_snapshot_repository + .read() + .map_err(|_| Error::LockPoisoning)? + .list_shards() + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::test::mocks::versioned_state_access_mock::VersionedStateAccessMock; + use std::{ + collections::{HashMap, VecDeque}, + sync::Arc, + thread, + }; + + type TestState = u64; + type TestHash = u64; + type TestStateRepository = VersionedStateAccessMock; + type TestStateHandler = StateHandler; + + #[test] + fn load_for_mutation_blocks_any_concurrent_access() { + let shard_id = ShardIdentifier::random(); + let state_handler = default_state_handler(&shard_id); + + let (lock, _s) = state_handler.load_for_mutation(&shard_id).unwrap(); + let new_state = 4u64; + + let state_handler_clone = state_handler.clone(); + let join_handle = thread::spawn(move || { + let latest_state = state_handler_clone.load(&shard_id).unwrap(); + assert_eq!(new_state, latest_state); + }); + + let _hash = state_handler.write_after_mutation(new_state, lock, &shard_id).unwrap(); + + join_handle.join().unwrap(); + } + + #[test] + fn load_initialized_works() { + let shard_id = ShardIdentifier::random(); + let state_handler = default_state_handler(&shard_id); + assert!(state_handler.load(&shard_id).is_ok()); + assert!(state_handler.load(&ShardIdentifier::random()).is_err()); + } + + #[test] + fn list_shards_works() { + let shard_id = ShardIdentifier::random(); + let state_handler = default_state_handler(&shard_id); + assert!(state_handler.list_shards().is_ok()); + } + + #[test] + fn shard_exists_works() { + let shard_id = ShardIdentifier::random(); + let state_handler = default_state_handler(&shard_id); + assert!(state_handler.shard_exists(&shard_id).unwrap()); + assert!(!state_handler.shard_exists(&ShardIdentifier::random()).unwrap()); + } + + fn default_state_handler(shard: &ShardIdentifier) -> Arc { + Arc::new(TestStateHandler::new(default_repository(shard))) + } + + fn default_repository(shard: &ShardIdentifier) -> TestStateRepository { + TestStateRepository::new(HashMap::from([(*shard, VecDeque::from([1, 2, 3]))])) + } +} diff --git a/core-primitives/stf-state-handler/src/state_key_repository.rs b/core-primitives/stf-state-handler/src/state_key_repository.rs new file mode 100644 index 0000000000..4ee6b8c46f --- /dev/null +++ b/core-primitives/stf-state-handler/src/state_key_repository.rs @@ -0,0 +1,96 @@ +/* + Copyright 2021 Integritee AG and Supercomputing Systems AG + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +#[cfg(feature = "sgx")] +use std::sync::SgxRwLock as RwLock; + +#[cfg(feature = "std")] +use std::sync::RwLock; + +use crate::error::{Error, Result}; +use itp_sgx_crypto::StateCrypto; +use itp_sgx_io::SealedIO; +use std::sync::Arc; + +pub trait AccessStateKey { + type KeyType: StateCrypto; + + fn retrieve_key(&self) -> Result; +} + +pub trait MutateStateKey { + fn update_key(&self, key: KeyType) -> Result<()>; +} + +pub struct StateKeyRepository { + key_lock: RwLock, + sealed_io: Arc, +} + +impl StateKeyRepository { + pub fn new(key: KeyType, sealed_io: Arc) -> Self { + StateKeyRepository { key_lock: RwLock::new(key), sealed_io } + } +} + +impl AccessStateKey for StateKeyRepository +where + KeyType: StateCrypto + Clone, +{ + type KeyType = KeyType; + + fn retrieve_key(&self) -> Result { + self.key_lock.read().map_err(|_| Error::LockPoisoning).map(|l| l.clone()) + } +} + +impl MutateStateKey for StateKeyRepository +where + KeyType: StateCrypto, + SealedIo: SealedIO, +{ + fn update_key(&self, key: KeyType) -> Result<()> { + let mut key_lock = self.key_lock.write().map_err(|_| Error::LockPoisoning)?; + + self.sealed_io.seal(key)?; + *key_lock = self.sealed_io.unseal()?; + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use itp_sgx_crypto::{aes::Aes, mocks::AesSealMock}; + + type TestKeyRepository = StateKeyRepository; + + #[test] + fn update_and_retrieve_key_works() { + let seal_mock = Arc::new(AesSealMock::default()); + let key_repository = TestKeyRepository::new(seal_mock.unseal().unwrap(), seal_mock.clone()); + + assert_eq!(seal_mock.unseal().unwrap(), key_repository.retrieve_key().unwrap()); + + let updated_key = Aes::new([2u8; 16], [0u8; 16]); + key_repository.update_key(updated_key).unwrap(); + + assert_eq!(updated_key, key_repository.retrieve_key().unwrap()); + assert_eq!(updated_key, seal_mock.unseal().unwrap()); + } +} diff --git a/core-primitives/stf-state-handler/src/state_snapshot_primitives.rs b/core-primitives/stf-state-handler/src/state_snapshot_primitives.rs new file mode 100644 index 0000000000..cd464b7201 --- /dev/null +++ b/core-primitives/stf-state-handler/src/state_snapshot_primitives.rs @@ -0,0 +1,55 @@ +/* + Copyright 2021 Integritee AG and Supercomputing Systems AG + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +use crate::{error::Result, file_io::StateFileIo}; +use itp_time_utils::now_as_nanos; +use itp_types::ShardIdentifier; +use std::collections::{HashMap, VecDeque}; + +pub type StateId = u128; + +pub(crate) type SnapshotHistory = + HashMap>>; + +/// Internal wrapper for a state hash and state ID. +#[derive(Clone)] +pub(crate) struct StateSnapshotMetaData { + pub(crate) state_hash: HashType, + pub(crate) state_id: StateId, +} + +impl StateSnapshotMetaData { + pub fn new(state_hash: HashType, state_id: StateId) -> Self { + StateSnapshotMetaData { state_hash, state_id } + } +} + +pub(crate) fn initialize_shard_with_snapshot( + shard_identifier: &ShardIdentifier, + file_io: &FileIo, +) -> Result> +where + FileIo: StateFileIo, +{ + let state_id = generate_current_timestamp_state_id(); + let state_hash = file_io.create_initialized(shard_identifier, state_id)?; + Ok(StateSnapshotMetaData::new(state_hash, state_id)) +} + +pub(crate) fn generate_current_timestamp_state_id() -> StateId { + now_as_nanos() +} diff --git a/core-primitives/stf-state-handler/src/state_snapshot_repository.rs b/core-primitives/stf-state-handler/src/state_snapshot_repository.rs new file mode 100644 index 0000000000..6239aed50d --- /dev/null +++ b/core-primitives/stf-state-handler/src/state_snapshot_repository.rs @@ -0,0 +1,443 @@ +/* + Copyright 2021 Integritee AG and Supercomputing Systems AG + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +use crate::{ + error::{Error, Result}, + file_io::StateFileIo, + state_snapshot_primitives::{ + generate_current_timestamp_state_id, initialize_shard_with_snapshot, SnapshotHistory, + StateId, StateSnapshotMetaData, + }, +}; +use core::ops::RangeBounds; +use itp_types::ShardIdentifier; +use log::*; +use std::{collections::VecDeque, fmt::Debug, format, marker::PhantomData, sync::Arc, vec::Vec}; + +/// Trait for versioned state access. Manages history of state snapshots. +pub trait VersionedStateAccess { + type StateType; + type HashType; + + /// Load the latest version of the state. + fn load_latest(&self, shard_identifier: &ShardIdentifier) -> Result; + + /// Update the state, returning the hash of the state. + fn update( + &mut self, + shard_identifier: &ShardIdentifier, + state: Self::StateType, + ) -> Result; + + /// Reverts the state of a given shard to a state version identified by a state hash. + fn revert_to( + &mut self, + shard_identifier: &ShardIdentifier, + state_hash: &Self::HashType, + ) -> Result; + + /// Initialize a new shard. + fn initialize_new_shard(&mut self, shard_identifier: ShardIdentifier) + -> Result; + + /// Checks if a shard for a given identifier exists. + fn shard_exists(&self, shard_identifier: &ShardIdentifier) -> bool; + + /// Lists all shards. + fn list_shards(&self) -> Result>; +} + +/// State snapshot repository. +/// +/// Keeps versions of state snapshots, cycles them in a fixed-size circular buffer. +/// Creates a state snapshot for each write/update operation. Allows reverting to a specific snapshot, +/// identified by a state hash. Snapshot files names includes a timestamp to be unique. +pub struct StateSnapshotRepository { + file_io: Arc, + snapshot_history_cache_size: usize, + snapshot_history: SnapshotHistory, + phantom_data: PhantomData, +} + +impl StateSnapshotRepository +where + FileIo: StateFileIo, + HashType: Copy + Eq + Debug, +{ + /// Constructor, initialized with no shards or snapshot history. + pub fn empty(file_io: Arc, snapshot_history_cache_size: usize) -> Result { + Self::new(file_io, snapshot_history_cache_size, SnapshotHistory::default()) + } + + /// Constructor to initialize the repository with shards and snapshot history. + /// + /// Crate private, to be used by the loader. + pub(crate) fn new( + file_io: Arc, + snapshot_history_cache_size: usize, + snapshot_history: SnapshotHistory, + ) -> Result { + if snapshot_history_cache_size == 0usize { + return Err(Error::ZeroCacheSize) + } + + Ok(StateSnapshotRepository { + file_io, + snapshot_history_cache_size, + snapshot_history, + phantom_data: Default::default(), + }) + } + + fn get_snapshot_history_mut( + &mut self, + shard_identifier: &ShardIdentifier, + ) -> Result<&mut VecDeque>> { + self.snapshot_history + .get_mut(shard_identifier) + .ok_or_else(|| Error::InvalidShard(*shard_identifier)) + } + + fn get_snapshot_history( + &self, + shard_identifier: &ShardIdentifier, + ) -> Result<&VecDeque>> { + self.snapshot_history + .get(shard_identifier) + .ok_or_else(|| Error::InvalidShard(*shard_identifier)) + } + + fn get_latest_snapshot_metadata( + &self, + shard_identifier: &ShardIdentifier, + ) -> Result<&StateSnapshotMetaData> { + let snapshot_history = self.get_snapshot_history(shard_identifier)?; + snapshot_history.front().ok_or(Error::EmptyRepository) + } + + fn prune_snapshot_history_by_range>( + &mut self, + shard_identifier: &ShardIdentifier, + range: R, + ) -> Result<()> { + let state_snapshots_to_remove = self + .get_snapshot_history_mut(shard_identifier)? + .drain(range) + .collect::>(); + + self.remove_snapshots(shard_identifier, state_snapshots_to_remove.as_slice()); + Ok(()) + } + + /// Remove snapshots referenced by metadata. + /// Does not stop on error, it's guaranteed to call `remove` on all elements. + /// Logs any errors that occur. + fn remove_snapshots( + &self, + shard_identifier: &ShardIdentifier, + snapshots_metadata: &[StateSnapshotMetaData], + ) { + for snapshot_metadata in snapshots_metadata { + if let Err(e) = self.file_io.remove(shard_identifier, snapshot_metadata.state_id) { + // We just log an error, don't want to return the error here, because the operation + // in general was successful, just a side-effect that failed. + error!("Failed to remove state, with id '{}': {:?}", snapshot_metadata.state_id, e); + } + } + } + + fn write_new_state( + &self, + shard_identifier: &ShardIdentifier, + state: State, + ) -> Result<(HashType, StateId)> { + let state_id = generate_current_timestamp_state_id(); + let state_hash = self.file_io.write(shard_identifier, state_id, state)?; + Ok((state_hash, state_id)) + } + + fn load_state( + &self, + shard_identifier: &ShardIdentifier, + snapshot_metadata: &StateSnapshotMetaData, + ) -> Result { + self.file_io.load(shard_identifier, snapshot_metadata.state_id) + } +} + +impl VersionedStateAccess + for StateSnapshotRepository +where + FileIo: StateFileIo, + HashType: Copy + Eq + Debug, +{ + type StateType = State; + type HashType = HashType; + + fn load_latest(&self, shard_identifier: &ShardIdentifier) -> Result { + let latest_snapshot_metadata = self.get_latest_snapshot_metadata(shard_identifier)?; + self.file_io.load(shard_identifier, latest_snapshot_metadata.state_id) + } + + fn update( + &mut self, + shard_identifier: &ShardIdentifier, + state: Self::StateType, + ) -> Result { + if !self.shard_exists(shard_identifier) { + return Err(Error::InvalidShard(*shard_identifier)) + } + + let (state_hash, state_id) = self.write_new_state(shard_identifier, state)?; + let cache_size = self.snapshot_history_cache_size; + + let snapshot_history = self.get_snapshot_history_mut(shard_identifier)?; + snapshot_history.push_front(StateSnapshotMetaData::new(state_hash, state_id)); + + // In case we're above max queue size we remove the oldest entries and corresponding files + if snapshot_history.len() > cache_size { + self.prune_snapshot_history_by_range(shard_identifier, cache_size..)?; + } + + Ok(state_hash) + } + + fn revert_to( + &mut self, + shard_identifier: &ShardIdentifier, + state_hash: &Self::HashType, + ) -> Result { + let snapshot_history = self.get_snapshot_history(shard_identifier)?; + + // We use `position()` instead of `find()`, because it then allows us to easily drain + // all the newer states. + let snapshot_metadata_index = snapshot_history + .iter() + .position(|fmd| fmd.state_hash == *state_hash) + .ok_or_else(|| Error::StateNotFoundInRepository(format!("{:?}", state_hash)))?; + + // Should never fail, since we got the index from above, with `position()`. + let snapshot_metadata = snapshot_history + .get(snapshot_metadata_index) + .ok_or_else(|| Error::StateNotFoundInRepository(format!("{:?}", state_hash)))?; + + let state = self.load_state(shard_identifier, snapshot_metadata)?; + + // Remove any state versions newer than the one we're resetting to + // (do this irreversible operation last, to ensure the loading has succeeded) + self.prune_snapshot_history_by_range(shard_identifier, ..snapshot_metadata_index)?; + + Ok(state) + } + + fn initialize_new_shard( + &mut self, + shard_identifier: ShardIdentifier, + ) -> Result { + if let Some(state_snapshots) = self.snapshot_history.get(&shard_identifier) { + warn!("Shard ({:?}) already exists, will not initialize again", shard_identifier); + return state_snapshots.front().map(|s| s.state_hash).ok_or(Error::EmptyRepository) + } + + let snapshot_metadata = + initialize_shard_with_snapshot(&shard_identifier, self.file_io.as_ref())?; + + let state_hash = snapshot_metadata.state_hash; + self.snapshot_history + .insert(shard_identifier, VecDeque::from([snapshot_metadata])); + Ok(state_hash) + } + + fn shard_exists(&self, shard_identifier: &ShardIdentifier) -> bool { + self.snapshot_history.get(shard_identifier).is_some() + } + + fn list_shards(&self) -> Result> { + Ok(self.snapshot_history.keys().cloned().collect()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ + in_memory_state_file_io::InMemoryStateFileIo, + state_snapshot_repository_loader::StateSnapshotRepositoryLoader, + }; + use std::{collections::hash_map::DefaultHasher, vec}; + + type TestState = u64; + type TestStateHash = u64; + type TestFileIo = InMemoryStateFileIo; + type TestSnapshotRepository = StateSnapshotRepository; + + const TEST_SNAPSHOT_REPOSITORY_CACHE_SIZE: usize = 3; + + #[test] + fn new_with_zero_cache_size_returns_error() { + let shards = + vec![ShardIdentifier::random(), ShardIdentifier::random(), ShardIdentifier::random()]; + let file_io = create_test_file_io(shards.as_slice()); + + assert!(TestSnapshotRepository::empty(file_io.clone(), 0usize).is_err()); + } + + #[test] + fn upon_new_all_shards_are_initialized() { + let shards = + vec![ShardIdentifier::random(), ShardIdentifier::random(), ShardIdentifier::random()]; + let (file_io, state_snapshot_repository) = create_state_snapshot_repository( + shards.as_slice(), + TEST_SNAPSHOT_REPOSITORY_CACHE_SIZE, + ); + + assert_eq!(shards.len(), file_io.list_shards().unwrap().len()); + assert_eq!(shards.len(), state_snapshot_repository.snapshot_history.len()); + assert_eq!(shards.len(), state_snapshot_repository.list_shards().unwrap().len()); + for states_per_shard in state_snapshot_repository.snapshot_history.values() { + assert_eq!(1, states_per_shard.len()); + } + for shard in shards { + assert!(state_snapshot_repository.load_latest(&shard).is_ok()); + assert!(state_snapshot_repository.shard_exists(&shard)); + } + } + + #[test] + fn update_latest_creates_new_state_file() { + let shards = + vec![ShardIdentifier::random(), ShardIdentifier::random(), ShardIdentifier::random()]; + let (file_io, mut state_snapshot_repository) = create_state_snapshot_repository( + shards.as_slice(), + TEST_SNAPSHOT_REPOSITORY_CACHE_SIZE, + ); + + let shard_to_update = shards.get(1).unwrap(); + assert_eq!(1, file_io.get_states_for_shard(shard_to_update).unwrap().len()); + + let new_state = 1234u64; + + let _ = state_snapshot_repository.update(shard_to_update, new_state).unwrap(); + + let snapshot_history = + state_snapshot_repository.snapshot_history.get(shard_to_update).unwrap(); + assert_eq!(2, snapshot_history.len()); + assert_eq!(new_state, state_snapshot_repository.load_latest(shard_to_update).unwrap()); + assert_eq!(2, file_io.get_states_for_shard(shard_to_update).unwrap().len()); + } + + #[test] + fn update_latest_prunes_states_when_above_cache_size() { + let shard_id = ShardIdentifier::random(); + let (file_io, mut state_snapshot_repository) = + create_state_snapshot_repository(&[shard_id], TEST_SNAPSHOT_REPOSITORY_CACHE_SIZE); + + let states = vec![1u64, 2u64, 3u64, 4u64, 5u64, 6u64]; + assert!(states.len() > TEST_SNAPSHOT_REPOSITORY_CACHE_SIZE); // ensures we have pruning + + states.iter().for_each(|state| { + let _ = state_snapshot_repository.update(&shard_id, *state).unwrap(); + }); + + let snapshot_history = state_snapshot_repository.snapshot_history.get(&shard_id).unwrap(); + assert_eq!(TEST_SNAPSHOT_REPOSITORY_CACHE_SIZE, snapshot_history.len()); + assert_eq!( + *states.last().unwrap(), + state_snapshot_repository.load_latest(&shard_id).unwrap() + ); + assert_eq!( + TEST_SNAPSHOT_REPOSITORY_CACHE_SIZE, + file_io.get_states_for_shard(&shard_id).unwrap().len() + ); + } + + #[test] + fn update_latest_with_invalid_shard_returns_error_without_modification() { + let shard_id = ShardIdentifier::random(); + let (file_io, mut state_snapshot_repository) = + create_state_snapshot_repository(&[shard_id], TEST_SNAPSHOT_REPOSITORY_CACHE_SIZE); + + assert!(state_snapshot_repository.update(&ShardIdentifier::random(), 45).is_err()); + + let snapshot_history = state_snapshot_repository.snapshot_history.get(&shard_id).unwrap(); + assert_eq!(1, snapshot_history.len()); + assert_eq!(0u64, state_snapshot_repository.load_latest(&shard_id).unwrap()); + assert_eq!(1, file_io.get_states_for_shard(&shard_id).unwrap().len()); + } + + #[test] + fn revert_to_removes_version_newer_than_target_hash() { + let shard_id = ShardIdentifier::random(); + let (file_io, mut state_snapshot_repository) = + create_state_snapshot_repository(&[shard_id], 6); + + let states = vec![1u64, 2u64, 3u64, 4u64, 5u64]; + + let state_hashes = states + .iter() + .map(|state| state_snapshot_repository.update(&shard_id, *state).unwrap()) + .collect::>(); + let revert_target_hash = state_hashes.get(1).unwrap(); + + let reverted_state = + state_snapshot_repository.revert_to(&shard_id, revert_target_hash).unwrap(); + + assert_eq!(2u64, reverted_state); + assert_eq!(3, state_snapshot_repository.snapshot_history.get(&shard_id).unwrap().len()); // because we have initialized version '0' as well + assert_eq!(2u64, state_snapshot_repository.load_latest(&shard_id).unwrap()); + assert_eq!(3, file_io.get_states_for_shard(&shard_id).unwrap().len()); + } + + #[test] + fn initializing_new_shard_works() { + let (_, mut state_snapshot_repository) = create_state_snapshot_repository(&[], 2); + + let shard_id = ShardIdentifier::random(); + + assert!(state_snapshot_repository.load_latest(&shard_id).is_err()); + assert!(state_snapshot_repository.list_shards().unwrap().is_empty()); + + let _hash = state_snapshot_repository.initialize_new_shard(shard_id).unwrap(); + + assert!(state_snapshot_repository.load_latest(&shard_id).is_ok()); + assert_eq!(1, state_snapshot_repository.list_shards().unwrap().len()); + } + + #[test] + fn initialize_new_state_when_shard_already_exists_returns_ok() { + let shard_id = ShardIdentifier::random(); + let (_, mut state_snapshot_repository) = create_state_snapshot_repository(&[shard_id], 2); + + let _hash = state_snapshot_repository.initialize_new_shard(shard_id).unwrap(); + + assert!(state_snapshot_repository.load_latest(&shard_id).is_ok()); + assert_eq!(1, state_snapshot_repository.list_shards().unwrap().len()); + } + + fn create_state_snapshot_repository( + shards: &[ShardIdentifier], + snapshot_history_size: usize, + ) -> (Arc, TestSnapshotRepository) { + let file_io = create_test_file_io(shards); + let repository_loader = StateSnapshotRepositoryLoader::new(file_io.clone()); + (file_io, repository_loader.load_snapshot_repository(snapshot_history_size).unwrap()) + } + + fn create_test_file_io(shards: &[ShardIdentifier]) -> Arc { + Arc::new(TestFileIo::new(DefaultHasher::default(), shards)) + } +} diff --git a/core-primitives/stf-state-handler/src/state_snapshot_repository_loader.rs b/core-primitives/stf-state-handler/src/state_snapshot_repository_loader.rs new file mode 100644 index 0000000000..729dbbc2f8 --- /dev/null +++ b/core-primitives/stf-state-handler/src/state_snapshot_repository_loader.rs @@ -0,0 +1,202 @@ +/* + Copyright 2021 Integritee AG and Supercomputing Systems AG + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +use crate::{ + error::Result, + file_io::StateFileIo, + state_snapshot_primitives::{ + initialize_shard_with_snapshot, SnapshotHistory, StateId, StateSnapshotMetaData, + }, + state_snapshot_repository::StateSnapshotRepository, +}; +use itp_types::ShardIdentifier; +use log::*; +use std::{ + collections::VecDeque, fmt::Debug, iter::FromIterator, marker::PhantomData, sync::Arc, vec::Vec, +}; + +/// Loads a state snapshot repository from existing shards directory with state files. +pub struct StateSnapshotRepositoryLoader { + file_io: Arc, + phantom_data: PhantomData<(State, HashType)>, +} + +impl StateSnapshotRepositoryLoader +where + FileIo: StateFileIo, + HashType: Copy + Eq + Debug, +{ + pub fn new(file_io: Arc) -> Self { + StateSnapshotRepositoryLoader { file_io, phantom_data: Default::default() } + } + + /// Load a state snapshot repository from an existing set of files and directories. + pub fn load_snapshot_repository( + &self, + snapshot_history_cache_size: usize, + ) -> Result> { + let snapshot_history = self.load_and_initialize_state_snapshot_history()?; + + StateSnapshotRepository::new( + self.file_io.clone(), + snapshot_history_cache_size, + snapshot_history, + ) + } + + fn load_and_initialize_state_snapshot_history(&self) -> Result> { + let mut repository = SnapshotHistory::new(); + + let shards = self.file_io.list_shards()?; + debug!("Found {} shard(s) to load state from", shards.len()); + + for shard in shards { + let mut state_ids = self.file_io.list_state_ids_for_shard(&shard)?; + // Sort by id (which are timestamp), highest, i.e. newest, first + state_ids.sort_unstable(); + state_ids.reverse(); + + let mut snapshot_metadata: Vec<_> = self.map_to_snapshot_metadata(&shard, state_ids); + + if snapshot_metadata.is_empty() { + warn!( + "No (valid) states found for shard {:?}, initializing empty shard state", + shard + ); + let initial_snapshot_metadata = + initialize_shard_with_snapshot(&shard, self.file_io.as_ref())?; + snapshot_metadata.push(initial_snapshot_metadata); + } else { + debug!( + "Found {} state snapshot(s) for shard {}, latest snapshot is {}", + snapshot_metadata.len(), + &shard, + snapshot_metadata.first().map(|f| f.state_id).unwrap_or_default() + ); + } + + let snapshot_history = VecDeque::from_iter(snapshot_metadata); + + repository.insert(shard, snapshot_history); + } + Ok(repository) + } + + fn map_to_snapshot_metadata( + &self, + shard: &ShardIdentifier, + state_ids: Vec, + ) -> Vec> { + state_ids + .into_iter() + .flat_map(|state_id| match self.file_io.compute_hash(shard, state_id) { + Ok(hash) => Some(StateSnapshotMetaData::new(hash, state_id)), + Err(e) => { + warn!( + "Failed to compute hash for state snapshot with id {}: {:?}, ignoring snapshot as a result", + state_id, e + ); + None + }, + }) + .collect() + } +} + +#[cfg(test)] +mod tests { + + use super::*; + use crate::in_memory_state_file_io::InMemoryStateFileIo; + use std::collections::hash_map::DefaultHasher; + + type TestState = u64; + type TestStateHash = u64; + type TestFileIo = InMemoryStateFileIo; + type TestLoader = StateSnapshotRepositoryLoader; + + #[test] + fn loading_from_empty_shard_directories_initializes_files() { + let shards = + vec![ShardIdentifier::random(), ShardIdentifier::random(), ShardIdentifier::random()]; + let (_, loader) = create_test_fixtures(shards.as_slice()); + + let snapshot_history = loader.load_and_initialize_state_snapshot_history().unwrap(); + assert_eq!(shards.len(), snapshot_history.len()); + for snapshots in snapshot_history.values() { + assert_eq!(1, snapshots.len()); + } + } + + #[test] + fn loading_without_shards_returns_empty_directory() { + let (_, loader) = create_test_fixtures(&[]); + + let snapshot_history = loader.load_and_initialize_state_snapshot_history().unwrap(); + assert!(snapshot_history.is_empty()); + } + + #[test] + fn loading_from_files_orders_by_timestamp() { + let shards = + vec![ShardIdentifier::random(), ShardIdentifier::random(), ShardIdentifier::random()]; + let (file_io, loader) = create_test_fixtures(shards.as_slice()); + + add_state_snapshots( + file_io.as_ref(), + &shards[0], + &[1_000_000, 2_000_000, 3_000_000, 4_000_000], + ); + add_state_snapshots(file_io.as_ref(), &shards[1], &[10_000_000, 9_000_000]); + add_state_snapshots(file_io.as_ref(), &shards[2], &[14_000_000, 11_000_000, 12_000_000]); + + let snapshot_history = loader.load_and_initialize_state_snapshot_history().unwrap(); + + assert_eq!(shards.len(), snapshot_history.len()); + assert_latest_state_id(&snapshot_history, &shards[0], 4_000_000); + assert_latest_state_id(&snapshot_history, &shards[1], 10_000_000); + assert_latest_state_id(&snapshot_history, &shards[2], 14_000_000); + } + + fn add_state_snapshots(file_io: &TestFileIo, shard: &ShardIdentifier, state_ids: &[StateId]) { + for state_id in state_ids { + add_snapshot_with_state_ids(file_io, shard, *state_id); + } + } + + fn add_snapshot_with_state_ids( + file_io: &TestFileIo, + shard: &ShardIdentifier, + state_id: StateId, + ) { + file_io.create_initialized(shard, state_id).unwrap(); + } + + fn assert_latest_state_id( + snapshot_history: &SnapshotHistory, + shard: &ShardIdentifier, + state_id: StateId, + ) { + assert_eq!(snapshot_history.get(shard).unwrap().front().unwrap().state_id, state_id) + } + + fn create_test_fixtures(shards: &[ShardIdentifier]) -> (Arc, TestLoader) { + let file_io = Arc::new(TestFileIo::new(DefaultHasher::default(), shards)); + let loader = StateSnapshotRepositoryLoader::new(file_io.clone()); + (file_io, loader) + } +} diff --git a/core-primitives/stf-state-handler/src/test/mocks/mod.rs b/core-primitives/stf-state-handler/src/test/mocks/mod.rs new file mode 100644 index 0000000000..400f2530c7 --- /dev/null +++ b/core-primitives/stf-state-handler/src/test/mocks/mod.rs @@ -0,0 +1,19 @@ +/* + Copyright 2021 Integritee AG and Supercomputing Systems AG + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +pub mod state_key_repository_mock; +pub mod versioned_state_access_mock; diff --git a/core-primitives/stf-state-handler/src/test/mocks/state_key_repository_mock.rs b/core-primitives/stf-state-handler/src/test/mocks/state_key_repository_mock.rs new file mode 100644 index 0000000000..d1b958c8ac --- /dev/null +++ b/core-primitives/stf-state-handler/src/test/mocks/state_key_repository_mock.rs @@ -0,0 +1,68 @@ +/* + Copyright 2021 Integritee AG and Supercomputing Systems AG + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +#[cfg(feature = "sgx")] +use std::sync::SgxRwLock as RwLock; + +#[cfg(feature = "std")] +use std::sync::RwLock; + +use crate::{ + error::Result, + state_key_repository::{AccessStateKey, MutateStateKey}, +}; +use itp_sgx_crypto::StateCrypto; + +#[derive(Default)] +pub struct StateKeyRepositoryMock +where + KeyType: StateCrypto + Clone + Default, +{ + key: RwLock, +} + +impl StateKeyRepositoryMock +where + KeyType: StateCrypto + Clone + Default, +{ + #[cfg(all(feature = "test", feature = "sgx"))] + pub fn new(key: KeyType) -> Self { + StateKeyRepositoryMock { key: RwLock::new(key) } + } +} + +impl AccessStateKey for StateKeyRepositoryMock +where + KeyType: StateCrypto + Clone + Default, +{ + type KeyType = KeyType; + + fn retrieve_key(&self) -> Result { + Ok(self.key.read().unwrap().clone()) + } +} + +impl MutateStateKey for StateKeyRepositoryMock +where + KeyType: StateCrypto + Clone + Default, +{ + fn update_key(&self, key: KeyType) -> Result<()> { + let mut lock = self.key.write().unwrap(); + *lock = key; + Ok(()) + } +} diff --git a/core-primitives/stf-state-handler/src/test/mocks/versioned_state_access_mock.rs b/core-primitives/stf-state-handler/src/test/mocks/versioned_state_access_mock.rs new file mode 100644 index 0000000000..65294db6e6 --- /dev/null +++ b/core-primitives/stf-state-handler/src/test/mocks/versioned_state_access_mock.rs @@ -0,0 +1,100 @@ +/* + Copyright 2021 Integritee AG and Supercomputing Systems AG + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +use crate::{ + error::{Error, Result}, + state_snapshot_repository::VersionedStateAccess, +}; +use itp_types::ShardIdentifier; +use std::{ + collections::{HashMap, VecDeque}, + marker::PhantomData, + string::ToString, + vec::Vec, +}; + +#[derive(Default, Clone)] +pub struct VersionedStateAccessMock { + state_history: HashMap>, + phantom_data: PhantomData, +} + +impl VersionedStateAccessMock { + #[cfg(test)] + pub fn new(state_history: HashMap>) -> Self { + VersionedStateAccessMock { state_history, phantom_data: Default::default() } + } +} + +impl VersionedStateAccess for VersionedStateAccessMock +where + State: Default + Clone, + Hash: Default, +{ + type StateType = State; + type HashType = Hash; + + fn load_latest(&self, shard_identifier: &ShardIdentifier) -> Result { + self.state_history + .get(shard_identifier) + .ok_or(Error::InvalidShard(*shard_identifier))? + .front() + .cloned() + .ok_or(Error::StateNotFoundInRepository("".to_string())) + } + + fn update( + &mut self, + shard_identifier: &ShardIdentifier, + state: Self::StateType, + ) -> Result { + let state_history = self + .state_history + .entry(*shard_identifier) + .or_insert_with(|| VecDeque::default()); + state_history.push_front(state); + Ok(Hash::default()) + } + + fn revert_to( + &mut self, + shard_identifier: &ShardIdentifier, + _state_hash: &Self::HashType, + ) -> Result { + let state_history = self + .state_history + .get_mut(shard_identifier) + .ok_or_else(|| Error::InvalidShard(*shard_identifier))?; + state_history.drain(..).last().ok_or(Error::EmptyRepository) + } + + fn initialize_new_shard( + &mut self, + shard_identifier: ShardIdentifier, + ) -> Result { + self.state_history.insert(shard_identifier, VecDeque::from([State::default()])); + Ok(Hash::default()) + } + + fn shard_exists(&self, shard_identifier: &ShardIdentifier) -> bool { + self.state_history.get(shard_identifier).is_some() + } + + fn list_shards(&self) -> Result> { + Ok(self.state_history.keys().copied().collect()) + } +} diff --git a/core-primitives/stf-state-handler/src/test/mod.rs b/core-primitives/stf-state-handler/src/test/mod.rs new file mode 100644 index 0000000000..e3552cd37f --- /dev/null +++ b/core-primitives/stf-state-handler/src/test/mod.rs @@ -0,0 +1,25 @@ +/* + Copyright 2021 Integritee AG and Supercomputing Systems AG + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +#[cfg(test)] +pub(crate) mod mocks; + +#[cfg(all(feature = "test", feature = "sgx"))] +pub mod mocks; + +#[cfg(all(feature = "test", feature = "sgx"))] +pub mod sgx_tests; diff --git a/core-primitives/stf-state-handler/src/test/sgx_tests.rs b/core-primitives/stf-state-handler/src/test/sgx_tests.rs new file mode 100644 index 0000000000..fb9c9c3ab5 --- /dev/null +++ b/core-primitives/stf-state-handler/src/test/sgx_tests.rs @@ -0,0 +1,340 @@ +/* + Copyright 2021 Integritee AG and Supercomputing Systems AG + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +*/ + +use crate::{ + error::{Error, Result}, + file_io::{ + purge_shard_dir, + sgx::{init_shard, shard_exists, SgxStateFileIo}, + shard_path, StateFileIo, + }, + handle_state::HandleState, + query_shard_state::QueryShardState, + state_handler::StateHandler, + state_snapshot_repository::StateSnapshotRepository, + state_snapshot_repository_loader::StateSnapshotRepositoryLoader, + test::mocks::state_key_repository_mock::StateKeyRepositoryMock, +}; +use codec::{Decode, Encode}; +use ita_stf::{State as StfState, StateType as StfStateType}; +use itp_sgx_crypto::{Aes, AesSeal, StateCrypto}; +use itp_sgx_io::{write, StaticSealedIO}; +use itp_types::{ShardIdentifier, H256}; +use sgx_externalities::SgxExternalitiesTrait; +use sp_core::hashing::blake2_256; +use std::{sync::Arc, thread, vec::Vec}; + +const STATE_SNAPSHOTS_CACHE_SIZE: usize = 3; + +type TestStateFileIo = SgxStateFileIo>; +type TestStateRepository = StateSnapshotRepository; +type TestStateRepositoryLoader = StateSnapshotRepositoryLoader; +type TestStateHandler = StateHandler; + +/// Directory handle to automatically initialize a directory +/// and upon dropping the reference, removing it again. +struct ShardDirectoryHandle { + shard: ShardIdentifier, +} + +impl ShardDirectoryHandle { + pub fn new(shard: ShardIdentifier) -> Result { + given_initialized_shard(&shard)?; + Ok(ShardDirectoryHandle { shard }) + } +} + +impl Drop for ShardDirectoryHandle { + fn drop(&mut self) { + purge_shard_dir(&self.shard) + } +} + +// Fixme: Move this test to sgx-runtime: +// +// https://github.com/integritee-network/sgx-runtime/issues/23 +pub fn test_sgx_state_decode_encode_works() { + // given + let state = given_hello_world_state(); + + // when + let encoded_state = state.state.encode(); + let state2 = StfStateType::decode(&mut encoded_state.as_slice()).unwrap(); + + // then + assert_eq!(state.state, state2); +} + +pub fn test_encrypt_decrypt_state_type_works() { + // given + let state = given_hello_world_state(); + let state_key = AesSeal::unseal_from_static_file().unwrap(); + + // when + let mut state_buffer = state.state.encode(); + state_key.encrypt(&mut state_buffer).unwrap(); + + state_key.decrypt(&mut state_buffer).unwrap(); + let decoded = StfStateType::decode(&mut state_buffer.as_slice()).unwrap(); + + // then + assert_eq!(state.state, decoded); +} + +pub fn test_write_and_load_state_works() { + // given + let shard: ShardIdentifier = [94u8; 32].into(); + let (state_handler, shard_dir_handle) = initialize_state_handler_with_directory_handle(&shard); + + let state = given_hello_world_state(); + + // when + let (lock, _s) = state_handler.load_for_mutation(&shard).unwrap(); + let _hash = state_handler.write_after_mutation(state.clone(), lock, &shard).unwrap(); + + let result = state_handler.load(&shard).unwrap(); + + // then + assert_eq!(state.state, result.state); + + // clean up + std::mem::drop(shard_dir_handle); +} + +pub fn test_ensure_subsequent_state_loads_have_same_hash() { + // given + let shard: ShardIdentifier = [49u8; 32].into(); + let (state_handler, shard_dir_handle) = initialize_state_handler_with_directory_handle(&shard); + + let (lock, initial_state) = state_handler.load_for_mutation(&shard).unwrap(); + state_handler.write_after_mutation(initial_state.clone(), lock, &shard).unwrap(); + + let state_loaded = state_handler.load(&shard).unwrap(); + + assert_eq!(hash_of(&initial_state.state), hash_of(&state_loaded.state)); + + // clean up + std::mem::drop(shard_dir_handle); +} + +fn hash_of(encodable: &T) -> H256 { + encodable.using_encoded(blake2_256).into() +} + +pub fn test_write_access_locks_read_until_finished() { + // here we want to test that a lock we obtain for + // mutating state locks out any read attempt that happens during that time + + // given + let shard: ShardIdentifier = [47u8; 32].into(); + let (state_handler, shard_dir_handle) = initialize_state_handler_with_directory_handle(&shard); + + let new_state_key = "my_new_state".encode(); + let (lock, mut state_to_mutate) = state_handler.load_for_mutation(&shard).unwrap(); + + // spawn a new thread that reads state + // this thread should be blocked until the write lock is released, i.e. until + // the new state is written. We can verify this, by trying to read that state variable + // that will be inserted further down below + let new_state_key_for_read = new_state_key.clone(); + let state_handler_clone = state_handler.clone(); + let shard_for_read = shard.clone(); + let join_handle = thread::spawn(move || { + let state_to_read = state_handler_clone.load(&shard_for_read).unwrap(); + assert!(state_to_read.get(new_state_key_for_read.as_slice()).is_some()); + }); + + assert!(state_to_mutate.get(new_state_key.clone().as_slice()).is_none()); + state_to_mutate.insert(new_state_key, "mega_secret_value".encode()); + + let _hash = state_handler.write_after_mutation(state_to_mutate, lock, &shard).unwrap(); + + join_handle.join().unwrap(); + + // clean up + std::mem::drop(shard_dir_handle); +} + +pub fn test_state_handler_file_backend_is_initialized() { + let shard: ShardIdentifier = [11u8; 32].into(); + let (state_handler, shard_dir_handle) = initialize_state_handler_with_directory_handle(&shard); + + assert!(state_handler.shard_exists(&shard).unwrap()); + assert!(1 <= state_handler.list_shards().unwrap().len()); // only greater equal, because there might be other (non-test) shards present + assert_eq!(1, number_of_files_in_shard_dir(&shard).unwrap()); // creates a first initialized file + + let _state = state_handler.load(&shard).unwrap(); + + assert_eq!(1, number_of_files_in_shard_dir(&shard).unwrap()); + + // clean up + std::mem::drop(shard_dir_handle); +} + +pub fn test_multiple_state_updates_create_snapshots_up_to_cache_size() { + let shard: ShardIdentifier = [17u8; 32].into(); + let (state_handler, _shard_dir_handle) = initialize_state_handler_with_directory_handle(&shard); + + assert_eq!(1, number_of_files_in_shard_dir(&shard).unwrap()); + + let hash_1 = update_state( + state_handler.as_ref(), + &shard, + ("my_key_1".encode(), "mega_secret_value".encode()), + ); + assert_eq!(2, number_of_files_in_shard_dir(&shard).unwrap()); + + let hash_2 = update_state( + state_handler.as_ref(), + &shard, + ("my_key_2".encode(), "mega_secret_value222".encode()), + ); + assert_eq!(3, number_of_files_in_shard_dir(&shard).unwrap()); + + let hash_3 = update_state( + state_handler.as_ref(), + &shard, + ("my_key_3".encode(), "mega_secret_value3".encode()), + ); + assert_eq!(3, number_of_files_in_shard_dir(&shard).unwrap()); + + let hash_4 = update_state( + state_handler.as_ref(), + &shard, + ("my_key_3".encode(), "mega_secret_valuenot3".encode()), + ); + assert_eq!(3, number_of_files_in_shard_dir(&shard).unwrap()); + + assert_ne!(hash_1, hash_2); + assert_ne!(hash_1, hash_3); + assert_ne!(hash_1, hash_4); + assert_ne!(hash_2, hash_3); + assert_ne!(hash_2, hash_4); + assert_ne!(hash_3, hash_4); + + assert_eq!(STATE_SNAPSHOTS_CACHE_SIZE, number_of_files_in_shard_dir(&shard).unwrap()); +} + +pub fn test_file_io_get_state_hash_works() { + let shard: ShardIdentifier = [21u8; 32].into(); + let _shard_dir_handle = ShardDirectoryHandle::new(shard).unwrap(); + let state_key_access = + Arc::new(StateKeyRepositoryMock::new(AesSeal::unseal_from_static_file().unwrap())); + + let file_io = TestStateFileIo::new(state_key_access); + + let state_id = 1234u128; + let state_hash = file_io.create_initialized(&shard, state_id).unwrap(); + assert_eq!(state_hash, file_io.compute_hash(&shard, state_id).unwrap()); + + let state_hash = file_io.write(&shard, state_id, given_hello_world_state()).unwrap(); + assert_eq!(state_hash, file_io.compute_hash(&shard, state_id).unwrap()); +} + +pub fn test_state_files_from_handler_can_be_loaded_again() { + let shard: ShardIdentifier = [15u8; 32].into(); + let (state_handler, _shard_dir_handle) = initialize_state_handler_with_directory_handle(&shard); + + update_state(state_handler.as_ref(), &shard, ("test_key_1".encode(), "value1".encode())); + update_state(state_handler.as_ref(), &shard, ("test_key_2".encode(), "value2".encode())); + update_state( + state_handler.as_ref(), + &shard, + ("test_key_2".encode(), "value2_updated".encode()), + ); + update_state(state_handler.as_ref(), &shard, ("test_key_3".encode(), "value3".encode())); + + // We initialize another state handler to load the state from the changes we just made. + let updated_state_handler = initialize_state_handler(); + + assert_eq!(STATE_SNAPSHOTS_CACHE_SIZE, number_of_files_in_shard_dir(&shard).unwrap()); + assert_eq!( + &"value3".encode(), + updated_state_handler + .load(&shard) + .unwrap() + .state() + .get("test_key_3".encode().as_slice()) + .unwrap() + ); +} + +pub fn test_list_state_ids_ignores_files_not_matching_the_pattern() { + let shard: ShardIdentifier = [21u8; 32].into(); + let _shard_dir_handle = ShardDirectoryHandle::new(shard).unwrap(); + let state_key_access = + Arc::new(StateKeyRepositoryMock::new(AesSeal::unseal_from_static_file().unwrap())); + + let file_io = TestStateFileIo::new(state_key_access); + + let mut invalid_state_file_path = shard_path(&shard); + invalid_state_file_path.push("invalid-state.bin"); + write(&[0, 1, 2, 3, 4, 5], invalid_state_file_path).unwrap(); + + file_io.create_initialized(&shard, 1234).unwrap(); + + assert_eq!(1, file_io.list_state_ids_for_shard(&shard).unwrap().len()); +} + +fn initialize_state_handler_with_directory_handle( + shard: &ShardIdentifier, +) -> (Arc, ShardDirectoryHandle) { + let shard_dir_handle = ShardDirectoryHandle::new(*shard).unwrap(); + (initialize_state_handler(), shard_dir_handle) +} + +fn initialize_state_handler() -> Arc { + let state_key_access = + Arc::new(StateKeyRepositoryMock::new(AesSeal::unseal_from_static_file().unwrap())); + let file_io = Arc::new(TestStateFileIo::new(state_key_access)); + let state_repository_loader = TestStateRepositoryLoader::new(file_io); + let state_snapshot_repository = state_repository_loader + .load_snapshot_repository(STATE_SNAPSHOTS_CACHE_SIZE) + .unwrap(); + Arc::new(TestStateHandler::new(state_snapshot_repository)) +} + +fn update_state( + state_handler: &TestStateHandler, + shard: &ShardIdentifier, + kv_pair: (Vec, Vec), +) -> H256 { + let (lock, mut state_to_mutate) = state_handler.load_for_mutation(shard).unwrap(); + state_to_mutate.insert(kv_pair.0, kv_pair.1); + state_handler.write_after_mutation(state_to_mutate, lock, shard).unwrap() +} + +fn given_hello_world_state() -> StfState { + let key: Vec = "hello".encode(); + let value: Vec = "world".encode(); + let mut state = StfState::new(); + state.insert(key, value); + state +} + +fn given_initialized_shard(shard: &ShardIdentifier) -> Result<()> { + if shard_exists(&shard) { + purge_shard_dir(shard); + } + init_shard(&shard) +} + +fn number_of_files_in_shard_dir(shard: &ShardIdentifier) -> Result { + let shard_dir_path = shard_path(shard); + let files_in_dir = std::fs::read_dir(shard_dir_path).map_err(|e| Error::Other(e.into()))?; + Ok(files_in_dir.count()) +} diff --git a/core-primitives/stf-state-handler/src/tests.rs b/core-primitives/stf-state-handler/src/tests.rs deleted file mode 100644 index dc7bbd588b..0000000000 --- a/core-primitives/stf-state-handler/src/tests.rs +++ /dev/null @@ -1,175 +0,0 @@ -/* - Copyright 2021 Integritee AG and Supercomputing Systems AG - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ - -#[cfg(all(not(feature = "std"), feature = "sgx"))] -use crate::sgx_reexport_prelude::*; - -use crate::{ - file_io::{encrypt, exists, init_shard, list_shards, load, write as state_write}, - global_file_state_handler::GlobalFileStateHandler, - handle_state::HandleState, -}; -use base58::ToBase58; -use codec::{Decode, Encode}; -use ita_stf::{State as StfState, StateType as StfStateType}; -use itp_settings::files::SHARDS_PATH; -use itp_types::{ShardIdentifier, H256}; -use sgx_externalities::SgxExternalitiesTrait; -use sp_core::hashing::blake2_256; -use std::{format, thread, vec::Vec}; - -// Fixme: Move this test to sgx-runtime: -// -// https://github.com/integritee-network/sgx-runtime/issues/23 -pub fn test_sgx_state_decode_encode_works() { - // given - let state = given_hello_world_state(); - - // when - let encoded_state = state.state.encode(); - let state2 = StfStateType::decode(&mut encoded_state.as_slice()).unwrap(); - - // then - assert_eq!(state.state, state2); -} - -pub fn test_encrypt_decrypt_state_type_works() { - // given - let state = given_hello_world_state(); - - // when - let encrypted = encrypt(state.state.encode()).unwrap(); - - let decrypted = encrypt(encrypted).unwrap(); - let decoded = StfStateType::decode(&mut decrypted.as_slice()).unwrap(); - - // then - assert_eq!(state.state, decoded); -} - -pub fn test_write_and_load_state_works() { - // given - ensure_no_empty_shard_directory_exists(); - - let state = given_hello_world_state(); - - let shard: ShardIdentifier = [94u8; 32].into(); - given_initialized_shard(&shard); - - // when - let _hash = state_write(state.clone(), &shard).unwrap(); - let result = load(&shard).unwrap(); - - // then - assert_eq!(state.state, result.state); - - // clean up - remove_shard_dir(&shard); -} - -pub fn test_ensure_subsequent_state_loads_have_same_hash() { - // given - ensure_no_empty_shard_directory_exists(); - - let shard: ShardIdentifier = [49u8; 32].into(); - given_initialized_shard(&shard); - - let state_handler = GlobalFileStateHandler; - - let (lock, initial_state) = state_handler.load_for_mutation(&shard).unwrap(); - state_handler.write(initial_state.clone(), lock, &shard).unwrap(); - - let state_loaded = state_handler.load_initialized(&shard).unwrap(); - - assert_eq!(hash_of(&initial_state.state), hash_of(&state_loaded.state)); - - // clean up - remove_shard_dir(&shard); -} - -fn hash_of(encodable: &T) -> H256 { - encodable.using_encoded(blake2_256).into() -} - -pub fn test_write_access_locks_read_until_finished() { - // here we want to test that a lock we obtain for - // mutating state locks out any read attempt that happens during that time - - // given - ensure_no_empty_shard_directory_exists(); - - let shard: ShardIdentifier = [47u8; 32].into(); - given_initialized_shard(&shard); - - let state_handler = GlobalFileStateHandler; - - let new_state_key = "my_new_state".encode(); - let (lock, mut state_to_mutate) = state_handler.load_for_mutation(&shard).unwrap(); - - // spawn a new thread that reads state - // this thread should be blocked until the write lock is released, i.e. until - // the new state is written. We can verify this, by trying to read that state variable - // that will be inserted further down below - let new_state_key_for_read = new_state_key.clone(); - let shard_for_read = shard.clone(); - let join_handle = thread::spawn(move || { - let state_handler = GlobalFileStateHandler; - let state_to_read = state_handler.load_initialized(&shard_for_read).unwrap(); - assert!(state_to_read.get(new_state_key_for_read.as_slice()).is_some()); - }); - - assert!(state_to_mutate.get(new_state_key.clone().as_slice()).is_none()); - state_to_mutate.insert(new_state_key, "mega_secret_value".encode()); - - let _hash = state_handler.write(state_to_mutate, lock, &shard).unwrap(); - - join_handle.join().unwrap(); - - // clean up - remove_shard_dir(&shard); -} - -fn ensure_no_empty_shard_directory_exists() { - // ensure no empty states are within directory (created with init-shard) - // otherwise an 'index out of bounds: the len is x but the index is x' - // error will be thrown - let shards = list_shards().unwrap(); - for shard in shards { - if !exists(&shard) { - init_shard(&shard).unwrap(); - } - } -} - -fn given_hello_world_state() -> StfState { - let key: Vec = "hello".encode(); - let value: Vec = "world".encode(); - let mut state = StfState::new(); - state.insert(key, value); - state -} - -fn given_initialized_shard(shard: &ShardIdentifier) { - if exists(&shard) { - remove_shard_dir(shard); - } - init_shard(&shard).unwrap(); -} - -fn remove_shard_dir(shard: &ShardIdentifier) { - std::fs::remove_dir_all(&format!("{}/{}", SHARDS_PATH, shard.encode().to_base58())).unwrap(); -} diff --git a/core-primitives/test/src/mock/handle_state_mock.rs b/core-primitives/test/src/mock/handle_state_mock.rs index b514bbb8f4..c33ea7f4d7 100644 --- a/core-primitives/test/src/mock/handle_state_mock.rs +++ b/core-primitives/test/src/mock/handle_state_mock.rs @@ -22,13 +22,13 @@ use std::sync::{SgxRwLock as RwLock, SgxRwLockWriteGuard as RwLockWriteGuard}; use std::sync::{RwLock, RwLockWriteGuard}; use codec::Encode; -use ita_stf::{ShardIdentifier, State as StfState}; +use ita_stf::State as StfState; use itp_stf_state_handler::{ error::{Error, Result}, handle_state::HandleState, query_shard_state::QueryShardState, }; -use itp_types::H256; +use itp_types::{ShardIdentifier, H256}; use sp_core::blake2_256; use std::{collections::HashMap, format, vec::Vec}; @@ -40,52 +40,72 @@ pub struct HandleStateMock { state_map: RwLock>, } +impl HandleStateMock { + pub fn from_shard(shard: ShardIdentifier) -> Result { + let state_handler = HandleStateMock { state_map: Default::default() }; + state_handler.initialize_shard(shard)?; + Ok(state_handler) + } +} + impl HandleState for HandleStateMock { type WriteLockPayload = HashMap; type StateT = StfState; + type HashType = H256; - fn load_initialized(&self, shard: &ShardIdentifier) -> Result { - let maybe_state = self.state_map.read().unwrap().get(shard).cloned(); + fn initialize_shard(&self, shard: ShardIdentifier) -> Result { + let maybe_state = self.state_map.read().unwrap().get(&shard).cloned(); return match maybe_state { // Initialize with default state, if it doesn't exist yet. None => { - self.state_map.write().unwrap().insert(*shard, StfState::default()); - - self.state_map.read().unwrap().get(shard).cloned().ok_or_else(|| { - Error::Other( - format!("state does not exist after inserting it, shard {:?}", shard) - .into(), - ) - }) + let state = StfState::default(); + let state_hash = state.using_encoded(blake2_256).into(); + self.state_map.write().unwrap().insert(shard, state); + Ok(state_hash) }, - Some(s) => Ok(s), + Some(s) => Ok(s.using_encoded(blake2_256).into()), } } + fn load(&self, shard: &ShardIdentifier) -> Result { + self.state_map + .read() + .unwrap() + .get(shard) + .cloned() + .ok_or_else(|| Error::Other(format!("shard is not initialized {:?}", shard).into())) + } + fn load_for_mutation( &self, shard: &ShardIdentifier, ) -> Result<(RwLockWriteGuard<'_, Self::WriteLockPayload>, StfState)> { - let initialized_state = self.load_initialized(shard)?; + let initialized_state = self.load(shard)?; let write_lock = self.state_map.write().unwrap(); Ok((write_lock, initialized_state)) } - fn write( + fn write_after_mutation( &self, state: StfState, mut state_lock: RwLockWriteGuard<'_, Self::WriteLockPayload>, shard: &ShardIdentifier, - ) -> Result { + ) -> Result { state_lock.insert(*shard, state.clone()); Ok(state.using_encoded(blake2_256).into()) } + + fn reset(&self, state: Self::StateT, shard: &ShardIdentifier) -> Result { + let write_lock = self.state_map.write().unwrap(); + self.write_after_mutation(state, write_lock, shard) + } } impl QueryShardState for HandleStateMock { - fn exists(&self, shard: &ShardIdentifier) -> bool { - self.state_map.read().unwrap().get(shard).is_some() + fn shard_exists(&self, shard: &ShardIdentifier) -> Result { + let state_map_lock = self.state_map.read().map_err(|_| Error::LockPoisoning)?; + Ok(state_map_lock.get(shard).is_some()) } fn list_shards(&self) -> Result> { @@ -112,15 +132,18 @@ pub mod tests { pub fn shard_exists_after_inserting() { let state_handler = HandleStateMock::default(); let shard = ShardIdentifier::default(); - let _loaded_state_result = state_handler.load_initialized(&shard); - assert!(state_handler.exists(&shard)); + state_handler.initialize_shard(shard).unwrap(); + + assert!(state_handler.load(&shard).is_ok()); + assert!(state_handler.shard_exists(&shard).unwrap()); } - pub fn load_initialized_inserts_default_state() { + pub fn initialize_creates_default_state() { let state_handler = HandleStateMock::default(); let shard = ShardIdentifier::default(); + state_handler.initialize_shard(shard).unwrap(); - let loaded_state_result = state_handler.load_initialized(&shard); + let loaded_state_result = state_handler.load(&shard); assert!(loaded_state_result.is_ok()); } @@ -128,33 +151,33 @@ pub mod tests { pub fn load_mutate_and_write_works() { let state_handler = HandleStateMock::default(); let shard = ShardIdentifier::default(); + state_handler.initialize_shard(shard).unwrap(); let (lock, mut state) = state_handler.load_for_mutation(&shard).unwrap(); let (key, value) = ("my_key", "my_value"); state.insert(key.encode(), value.encode()); - state_handler.write(state, lock, &shard).unwrap(); + state_handler.write_after_mutation(state, lock, &shard).unwrap(); - let updated_state = state_handler.load_initialized(&shard).unwrap(); + let updated_state = state_handler.load(&shard).unwrap(); let inserted_value = updated_state.get(key.encode().as_slice()).expect("value for key should exist"); assert_eq!(*inserted_value, value.encode()); } - // This is the same test as for the `GlobalFileStateHandler` to ensure we don't have any effects - // from having the state in-memory (as here) vs. in file (`GlobalFileStateHandler`). pub fn ensure_subsequent_state_loads_have_same_hash() { let state_handler = HandleStateMock::default(); let shard = ShardIdentifier::default(); + state_handler.initialize_shard(shard).unwrap(); let (lock, _) = state_handler.load_for_mutation(&shard).unwrap(); let initial_state = Stf::init_state(); let state_hash_before_execution = hash_of(&initial_state.state); - state_handler.write(initial_state, lock, &shard).unwrap(); + state_handler.write_after_mutation(initial_state, lock, &shard).unwrap(); - let state_loaded = state_handler.load_initialized(&shard).unwrap(); + let state_loaded = state_handler.load(&shard).unwrap(); let loaded_state_hash = hash_of(&state_loaded.state); assert_eq!(state_hash_before_execution, loaded_state_hash); diff --git a/core-primitives/time-utils/src/lib.rs b/core-primitives/time-utils/src/lib.rs index 0244ed6041..6ae6f348af 100644 --- a/core-primitives/time-utils/src/lib.rs +++ b/core-primitives/time-utils/src/lib.rs @@ -31,6 +31,11 @@ pub fn now_as_u64() -> u64 { duration_now().as_millis() as u64 } +/// Returns the current timestamp based on the unix epoch in nanoseconds. +pub fn now_as_nanos() -> u128 { + duration_now().as_nanos() +} + /// Calculates the remaining time `until`. pub fn remaining_time(until: Duration) -> Option { until.checked_sub(duration_now()) diff --git a/core-primitives/top-pool-author/src/author.rs b/core-primitives/top-pool-author/src/author.rs index 0b224c9761..cdb714d9a4 100644 --- a/core-primitives/top-pool-author/src/author.rs +++ b/core-primitives/top-pool-author/src/author.rs @@ -114,11 +114,14 @@ where shard: ShardIdentifier, submission_mode: TopSubmissionMode, ) -> PoolFuture, RpcError> { - // check if shard already exists - if !self.state_facade.exists(&shard) { - //FIXME: Should this be an error? -> Issue error handling - return Box::pin(ready(Err(ClientError::InvalidShard.into()))) - } + // check if shard exists + match self.state_facade.shard_exists(&shard) { + Err(_) => return Box::pin(ready(Err(ClientError::InvalidShard.into()))), + Ok(shard_exists) => + if !shard_exists { + return Box::pin(ready(Err(ClientError::InvalidShard.into()))) + }, + }; // decrypt call let request_vec = match self.encryption_key.decrypt(ext.as_slice()) { diff --git a/core-primitives/top-pool-author/src/author_tests.rs b/core-primitives/top-pool-author/src/author_tests.rs index c3fa22b8af..59b176a4e9 100644 --- a/core-primitives/top-pool-author/src/author_tests.rs +++ b/core-primitives/top-pool-author/src/author_tests.rs @@ -104,8 +104,8 @@ fn create_author_with_filter>( let top_pool = Arc::new(TrustedOperationPoolMock::default()); let shard_id = shard_id(); - let state_facade = HandleStateMock::default(); - let _ = state_facade.load_initialized(&shard_id).unwrap(); + let state_facade = HandleStateMock::from_shard(shard_id).unwrap(); + let _ = state_facade.load(&shard_id).unwrap(); let encryption_key = ShieldingCryptoMock::default(); let ocall_mock = Arc::new(MetricsOCallMock {}); diff --git a/core/parentchain/block-import-dispatcher/src/triggered_dispatcher.rs b/core/parentchain/block-import-dispatcher/src/triggered_dispatcher.rs index 3185315905..3f1f5d34c2 100644 --- a/core/parentchain/block-import-dispatcher/src/triggered_dispatcher.rs +++ b/core/parentchain/block-import-dispatcher/src/triggered_dispatcher.rs @@ -103,6 +103,8 @@ where let latest_imported_block = blocks_to_import.last().map(|b| (*b).clone()); + debug!("Trigger import of all parentchain blocks in queue ({})", blocks_to_import.len()); + self.block_importer .import_parentchain_blocks(blocks_to_import) .map_err(Error::BlockImport)?; @@ -114,6 +116,11 @@ where let blocks_to_import = self.import_queue.pop_all_but_last().map_err(Error::BlockImportQueue)?; + debug!( + "Trigger import of all parentchain blocks, except the latest, from queue ({})", + blocks_to_import.len() + ); + self.block_importer .import_parentchain_blocks(blocks_to_import) .map_err(Error::BlockImport) diff --git a/core/parentchain/light-client/src/concurrent_access.rs b/core/parentchain/light-client/src/concurrent_access.rs index 8b6eb7b187..6efa925a6d 100644 --- a/core/parentchain/light-client/src/concurrent_access.rs +++ b/core/parentchain/light-client/src/concurrent_access.rs @@ -29,7 +29,7 @@ use crate::{ LightClientState, Validator as ValidatorTrait, }; use finality_grandpa::BlockNumberOps; -use itp_sgx_io::SealedIO; +use itp_sgx_io::StaticSealedIO; use lazy_static::lazy_static; use sp_runtime::traits::{Block as ParentchainBlockTrait, NumberFor}; use std::marker::PhantomData; @@ -69,7 +69,7 @@ where pub struct GlobalValidatorAccessor where Validator: ValidatorTrait + LightClientState, - Seal: SealedIO, + Seal: StaticSealedIO, ParentchainBlock: ParentchainBlockTrait, NumberFor: BlockNumberOps, { @@ -80,7 +80,7 @@ impl Default for GlobalValidatorAccessor where Validator: ValidatorTrait + LightClientState, - Seal: SealedIO, + Seal: StaticSealedIO, ParentchainBlock: ParentchainBlockTrait, NumberFor: BlockNumberOps, { @@ -92,7 +92,7 @@ where impl GlobalValidatorAccessor where Validator: ValidatorTrait + LightClientState, - Seal: SealedIO, + Seal: StaticSealedIO, ParentchainBlock: ParentchainBlockTrait, NumberFor: BlockNumberOps, { @@ -105,7 +105,7 @@ impl ValidatorAccess for GlobalValidatorAccessor where Validator: ValidatorTrait + LightClientState, - Seal: SealedIO, + Seal: StaticSealedIO, ParentchainBlock: ParentchainBlockTrait, NumberFor: BlockNumberOps, { @@ -116,7 +116,7 @@ where F: FnOnce(&Self::ValidatorType) -> Result, { let _read_lock = VALIDATOR_LOCK.read().map_err(|_| Error::PoisonedLock)?; - let validator = Seal::unseal()?; + let validator = Seal::unseal_from_static_file()?; getter_function(&validator) } @@ -125,9 +125,9 @@ where F: FnOnce(&mut Self::ValidatorType) -> Result, { let _write_lock = VALIDATOR_LOCK.write().map_err(|_| Error::PoisonedLock)?; - let mut validator = Seal::unseal()?; + let mut validator = Seal::unseal_from_static_file()?; let result = mutating_function(&mut validator); - Seal::seal(validator)?; + Seal::seal_to_static_file(validator)?; result } } diff --git a/core/parentchain/light-client/src/io.rs b/core/parentchain/light-client/src/io.rs index b8c2052cee..74850758d2 100644 --- a/core/parentchain/light-client/src/io.rs +++ b/core/parentchain/light-client/src/io.rs @@ -19,7 +19,7 @@ use crate::{error::Result, Error, LightClientState, LightValidation, NumberFor, use codec::{Decode, Encode}; use derive_more::Display; use itp_settings::files::LIGHT_CLIENT_DB; -use itp_sgx_io::{seal, unseal, SealedIO}; +use itp_sgx_io::{seal, unseal, StaticSealedIO}; use itp_storage::StorageProof; use log::*; use sp_finality_grandpa::VersionedAuthorityList; @@ -31,15 +31,15 @@ pub struct LightClientSeal { _phantom: B, } -impl SealedIO for LightClientSeal { +impl StaticSealedIO for LightClientSeal { type Error = Error; type Unsealed = LightValidation; - fn unseal() -> Result { + fn unseal_from_static_file() -> Result { Ok(unseal(LIGHT_CLIENT_DB).map(|b| Decode::decode(&mut b.as_slice()))??) } - fn seal(unsealed: Self::Unsealed) -> Result<()> { + fn seal_to_static_file(unsealed: Self::Unsealed) -> Result<()> { debug!("backup light client state"); if fs::copy(LIGHT_CLIENT_DB, format!("{}.1", LIGHT_CLIENT_DB)).is_err() { warn!("could not backup previous light client state"); @@ -62,7 +62,7 @@ where return init_validator::(header, auth, proof) } - let validator = LightClientSeal::::unseal()?; + let validator = LightClientSeal::::unseal_from_static_file()?; let genesis = validator.genesis_hash(validator.num_relays()).unwrap(); if genesis == header.hash() { @@ -85,7 +85,7 @@ where let mut validator = LightValidation::::new(); validator.initialize_relay(header, auth.into(), proof)?; - LightClientSeal::::seal(validator.clone())?; + LightClientSeal::::seal_to_static_file(validator.clone())?; Ok(validator.latest_finalized_header(validator.num_relays()).unwrap()) } diff --git a/core/parentchain/light-client/src/mocks/validator_mock_seal.rs b/core/parentchain/light-client/src/mocks/validator_mock_seal.rs index 03c7c27d82..7bf142cbfd 100644 --- a/core/parentchain/light-client/src/mocks/validator_mock_seal.rs +++ b/core/parentchain/light-client/src/mocks/validator_mock_seal.rs @@ -16,21 +16,21 @@ */ use crate::{error::Error, mocks::validator_mock::ValidatorMock}; -use itp_sgx_io::SealedIO; +use itp_sgx_io::StaticSealedIO; /// A seal that returns a mock validator. #[derive(Clone)] pub struct ValidatorMockSeal; -impl SealedIO for ValidatorMockSeal { +impl StaticSealedIO for ValidatorMockSeal { type Error = Error; type Unsealed = ValidatorMock; - fn unseal() -> Result { + fn unseal_from_static_file() -> Result { Ok(ValidatorMock) } - fn seal(_unsealed: Self::Unsealed) -> Result<(), Self::Error> { + fn seal_to_static_file(_unsealed: Self::Unsealed) -> Result<(), Self::Error> { Ok(()) } } diff --git a/enclave-runtime/Cargo.lock b/enclave-runtime/Cargo.lock index 5d513ca666..4f58cdc6aa 100644 --- a/enclave-runtime/Cargo.lock +++ b/enclave-runtime/Cargo.lock @@ -1415,6 +1415,7 @@ dependencies = [ "itp-settings", "itp-sgx-crypto", "itp-sgx-io", + "itp-time-utils", "itp-types", "lazy_static", "log 0.4.14 (git+https://github.com/mesalock-linux/log-sgx)", diff --git a/enclave-runtime/Enclave.edl b/enclave-runtime/Enclave.edl index 0230e493c6..f712094d13 100644 --- a/enclave-runtime/Enclave.edl +++ b/enclave-runtime/Enclave.edl @@ -56,6 +56,10 @@ enclave { [out, size=latest_header_size] uint8_t* latest_header, size_t latest_header_size ); + public sgx_status_t init_shard( + [in, size=shard_size] uint8_t* shard, uint32_t shard_size + ); + public sgx_status_t trigger_parentchain_block_import(); public sgx_status_t execute_trusted_getters(); diff --git a/enclave-runtime/src/attestation.rs b/enclave-runtime/src/attestation.rs index 09a7020547..b68c140d4c 100644 --- a/enclave-runtime/src/attestation.rs +++ b/enclave-runtime/src/attestation.rs @@ -41,7 +41,7 @@ use itp_settings::{ node::{REGISTER_ENCLAVE, RUNTIME_SPEC_VERSION, RUNTIME_TRANSACTION_VERSION, TEEREX_MODULE}, }; use itp_sgx_crypto::Ed25519Seal; -use itp_sgx_io::SealedIO; +use itp_sgx_io::StaticSealedIO; use log::*; use sgx_rand::*; use sgx_tcrypto::*; @@ -449,7 +449,7 @@ pub fn create_ra_report_and_signature( ocall_api: &A, skip_ra: bool, ) -> EnclaveResult<(Vec, Vec)> { - let chain_signer = Ed25519Seal::unseal()?; + let chain_signer = Ed25519Seal::unseal_from_static_file()?; info!("[Enclave Attestation] Ed25519 pub raw : {:?}", chain_signer.public().0); info!(" [Enclave] Generate keypair"); @@ -520,7 +520,7 @@ pub unsafe extern "C" fn perform_ra( let url_slice = slice::from_raw_parts(w_url, w_url_size as usize); let extrinsic_slice = slice::from_raw_parts_mut(unchecked_extrinsic, unchecked_extrinsic_size as usize); - let signer = match Ed25519Seal::unseal() { + let signer = match Ed25519Seal::unseal_from_static_file() { Ok(pair) => pair, Err(e) => return e.into(), }; diff --git a/enclave-runtime/src/global_components.rs b/enclave-runtime/src/global_components.rs index 91ef2ab06c..5818ad51db 100644 --- a/enclave-runtime/src/global_components.rs +++ b/enclave-runtime/src/global_components.rs @@ -21,7 +21,7 @@ //! and ensures that the global instances are initialized once. use crate::ocall::OcallApi; -use ita_stf::Hash; +use ita_stf::{Hash, State as StfState}; use itc_direct_rpc_server::{ rpc_connection_registry::ConnectionRegistry, rpc_watch_extractor::RpcWatchExtractor, rpc_ws_handler::RpcWsHandler, @@ -36,9 +36,12 @@ use itp_block_import_queue::BlockImportQueue; use itp_component_container::ComponentContainer; use itp_extrinsics_factory::ExtrinsicsFactory; use itp_nonce_cache::NonceCache; -use itp_sgx_crypto::Aes; +use itp_sgx_crypto::{Aes, AesSeal}; use itp_stf_executor::executor::StfExecutor; -use itp_stf_state_handler::GlobalFileStateHandler; +use itp_stf_state_handler::{ + file_io::sgx::SgxStateFileIo, state_key_repository::StateKeyRepository, + state_snapshot_repository::StateSnapshotRepository, StateHandler, +}; use itp_top_pool_author::{ author::{Author, AuthorTopFilter}, pool_types::BPool, @@ -55,11 +58,16 @@ use its_sidechain::{ state::SidechainDB, top_pool_executor::TopPoolOperationHandler, }; +use primitive_types::H256; use sgx_crypto_helper::rsa3072::Rsa3072KeyPair; use sgx_externalities::SgxExternalities; use sp_core::ed25519::Pair; -pub type EnclaveStateHandler = GlobalFileStateHandler; +pub type EnclaveStateKeyRepository = StateKeyRepository; +pub type EnclaveStateFileIo = SgxStateFileIo; +pub type EnclaveStateSnapshotRepository = + StateSnapshotRepository; +pub type EnclaveStateHandler = StateHandler; pub type EnclaveOCallApi = OcallApi; pub type EnclaveStfExecutor = StfExecutor; pub type EnclaveExtrinsicsFactory = ExtrinsicsFactory; @@ -124,6 +132,10 @@ pub type EnclaveSidechainBlockImportQueueWorker = BlockImportQueueWorker< /// Base component instances ///------------------------------------------------------------------------------------------------- +/// State key repository +pub static GLOBAL_STATE_KEY_REPOSITORY_COMPONENT: ComponentContainer = + ComponentContainer::new("State key repository"); + /// STF executor. pub static GLOBAL_STF_EXECUTOR_COMPONENT: ComponentContainer = ComponentContainer::new("STF executor"); diff --git a/enclave-runtime/src/initialization.rs b/enclave-runtime/src/initialization.rs index b6df0827ef..f07a223296 100644 --- a/enclave-runtime/src/initialization.rs +++ b/enclave-runtime/src/initialization.rs @@ -19,14 +19,15 @@ use crate::{ error::{Error, Result as EnclaveResult}, global_components::{ EnclaveSidechainBlockImportQueue, EnclaveSidechainBlockImportQueueWorker, - EnclaveSidechainBlockImporter, EnclaveSidechainBlockSyncer, EnclaveStfExecutor, - EnclaveTopPoolOperationHandler, EnclaveValidatorAccessor, - GLOBAL_EXTRINSICS_FACTORY_COMPONENT, GLOBAL_OCALL_API_COMPONENT, + EnclaveSidechainBlockImporter, EnclaveSidechainBlockSyncer, EnclaveStateFileIo, + EnclaveStateKeyRepository, EnclaveStfExecutor, EnclaveTopPoolOperationHandler, + EnclaveValidatorAccessor, GLOBAL_EXTRINSICS_FACTORY_COMPONENT, GLOBAL_OCALL_API_COMPONENT, GLOBAL_PARENTCHAIN_IMPORT_DISPATCHER_COMPONENT, GLOBAL_RPC_WS_HANDLER_COMPONENT, GLOBAL_SIDECHAIN_BLOCK_COMPOSER_COMPONENT, GLOBAL_SIDECHAIN_BLOCK_SYNCER_COMPONENT, GLOBAL_SIDECHAIN_IMPORT_QUEUE_COMPONENT, GLOBAL_SIDECHAIN_IMPORT_QUEUE_WORKER_COMPONENT, - GLOBAL_STATE_HANDLER_COMPONENT, GLOBAL_STF_EXECUTOR_COMPONENT, - GLOBAL_TOP_POOL_AUTHOR_COMPONENT, GLOBAL_TOP_POOL_OPERATION_HANDLER_COMPONENT, + GLOBAL_STATE_HANDLER_COMPONENT, GLOBAL_STATE_KEY_REPOSITORY_COMPONENT, + GLOBAL_STF_EXECUTOR_COMPONENT, GLOBAL_TOP_POOL_AUTHOR_COMPONENT, + GLOBAL_TOP_POOL_OPERATION_HANDLER_COMPONENT, }, ocall::OcallApi, rpc::worker_api_direct::public_api_rpc_handler, @@ -34,6 +35,7 @@ use crate::{ }; use base58::ToBase58; use codec::Encode; +use ita_stf::State as StfState; use itc_direct_rpc_server::{ create_determine_watch, rpc_connection_registry::ConnectionRegistry, rpc_ws_handler::RpcWsHandler, @@ -50,16 +52,21 @@ use itp_component_container::{ComponentGetter, ComponentInitializer}; use itp_extrinsics_factory::ExtrinsicsFactory; use itp_nonce_cache::GLOBAL_NONCE_CACHE; use itp_primitives_cache::GLOBAL_PRIMITIVES_CACHE; +use itp_settings::files::STATE_SNAPSHOTS_CACHE_SIZE; use itp_sgx_crypto::{aes, ed25519, rsa3072, AesSeal, Ed25519Seal, Rsa3072Seal}; -use itp_sgx_io::SealedIO; -use itp_stf_state_handler::{query_shard_state::QueryShardState, GlobalFileStateHandler}; +use itp_sgx_io::StaticSealedIO; +use itp_stf_state_handler::{ + handle_state::HandleState, query_shard_state::QueryShardState, + state_snapshot_repository_loader::StateSnapshotRepositoryLoader, StateHandler, +}; use itp_storage::StorageProof; -use itp_types::{Block, Header, SignedBlock}; +use itp_types::{Block, Header, ShardIdentifier, SignedBlock}; use its_sidechain::{ aura::block_importer::BlockImporter, block_composer::BlockComposer, top_pool_executor::TopPoolOperationHandler, }; use log::*; +use primitive_types::H256; use sp_core::crypto::Pair; use sp_finality_grandpa::VersionedAuthorityList; use std::{string::String, sync::Arc}; @@ -69,16 +76,27 @@ pub(crate) fn init_enclave(mu_ra_url: String, untrusted_worker_url: String) -> E env_logger::init(); ed25519::create_sealed_if_absent().map_err(Error::Crypto)?; - let signer = Ed25519Seal::unseal().map_err(Error::Crypto)?; + let signer = Ed25519Seal::unseal_from_static_file().map_err(Error::Crypto)?; info!("[Enclave initialized] Ed25519 prim raw : {:?}", signer.public().0); rsa3072::create_sealed_if_absent()?; // Create the aes key that is used for state encryption such that a key is always present in tests. - // It will be overwritten anyway if mutual remote attastation is performed with the primary worker. + // It will be overwritten anyway if mutual remote attestation is performed with the primary worker. aes::create_sealed_if_absent().map_err(Error::Crypto)?; - let state_handler = Arc::new(GlobalFileStateHandler); + let state_key = AesSeal::unseal_from_static_file()?; + let state_key_repository = + Arc::new(EnclaveStateKeyRepository::new(state_key, Arc::new(AesSeal))); + GLOBAL_STATE_KEY_REPOSITORY_COMPONENT.initialize(state_key_repository.clone()); + + let state_file_io = Arc::new(EnclaveStateFileIo::new(state_key_repository)); + let state_snapshot_repository_loader = + StateSnapshotRepositoryLoader::::new(state_file_io); + let state_snapshot_repository = + state_snapshot_repository_loader.load_snapshot_repository(STATE_SNAPSHOTS_CACHE_SIZE)?; + + let state_handler = Arc::new(StateHandler::new(state_snapshot_repository)); GLOBAL_STATE_HANDLER_COMPONENT.initialize(state_handler.clone()); let ocall_api = Arc::new(OcallApi); @@ -101,7 +119,7 @@ pub(crate) fn init_enclave(mu_ra_url: String, untrusted_worker_url: String) -> E ) .map_err(Error::PrimitivesAccess)?; - let shielding_key = Rsa3072Seal::unseal()?; + let shielding_key = Rsa3072Seal::unseal_from_static_file()?; let watch_extractor = Arc::new(create_determine_watch::()); let connection_registry = Arc::new(ConnectionRegistry::::new()); @@ -150,8 +168,8 @@ pub(crate) fn init_enclave_sidechain_components() -> EnclaveResult<()> { let parentchain_block_import_dispatcher = GLOBAL_PARENTCHAIN_IMPORT_DISPATCHER_COMPONENT.get()?; - let signer = Ed25519Seal::unseal()?; - let state_key = AesSeal::unseal()?; + let signer = Ed25519Seal::unseal_from_static_file()?; + let state_key = AesSeal::unseal_from_static_file()?; let sidechain_block_importer = Arc::::new(BlockImporter::new( state_handler, @@ -193,8 +211,8 @@ pub(crate) fn init_light_client( )?; // Initialize the global parentchain block import dispatcher instance. - let signer = Ed25519Seal::unseal()?; - let shielding_key = Rsa3072Seal::unseal()?; + let signer = Ed25519Seal::unseal_from_static_file()?; + let shielding_key = Rsa3072Seal::unseal_from_static_file()?; let stf_executor = GLOBAL_STF_EXECUTOR_COMPONENT.get()?; let ocall_api = GLOBAL_OCALL_API_COMPONENT.get()?; @@ -234,3 +252,9 @@ pub(crate) fn init_direct_invocation_server(server_addr: String) -> EnclaveResul Ok(()) } + +pub(crate) fn init_shard(shard: ShardIdentifier) -> EnclaveResult<()> { + let state_handler = GLOBAL_STATE_HANDLER_COMPONENT.get()?; + let _ = state_handler.initialize_shard(shard)?; + Ok(()) +} diff --git a/enclave-runtime/src/lib.rs b/enclave-runtime/src/lib.rs index fcbae08431..4a07868ed0 100644 --- a/enclave-runtime/src/lib.rs +++ b/enclave-runtime/src/lib.rs @@ -57,7 +57,7 @@ use itp_settings::node::{ }; use itp_sgx_crypto::{ed25519, Ed25519Seal, Rsa3072Seal}; use itp_sgx_io as io; -use itp_sgx_io::SealedIO; +use itp_sgx_io::StaticSealedIO; use itp_stf_state_handler::handle_state::HandleState; use itp_storage::StorageProof; use itp_types::{Header, SignedBlock}; @@ -161,7 +161,7 @@ pub unsafe extern "C" fn get_ecc_signing_pubkey(pubkey: *mut u8, pubkey_size: u3 return e.into() } - let signer = match Ed25519Seal::unseal().map_err(Error::Crypto) { + let signer = match Ed25519Seal::unseal_from_static_file().map_err(Error::Crypto) { Ok(pair) => pair, Err(e) => return e.into(), }; @@ -212,7 +212,7 @@ pub unsafe extern "C" fn mock_register_enclave_xt( .get_mrenclave_of_self() .map_or_else(|_| Vec::::new(), |m| m.m.encode()); - let signer = Ed25519Seal::unseal().unwrap(); + let signer = Ed25519Seal::unseal_from_static_file().unwrap(); let call = ([TEEREX_MODULE, REGISTER_ENCLAVE], mre, url); let nonce_cache = GLOBAL_NONCE_CACHE.clone(); @@ -318,7 +318,7 @@ pub unsafe extern "C" fn get_state( }, }; - let mut state = match state_handler.load_initialized(&shard) { + let mut state = match state_handler.load(&shard) { Ok(s) => s, Err(e) => return Error::StfStateHandler(e).into(), }; @@ -428,6 +428,19 @@ pub unsafe extern "C" fn init_light_client( sgx_status_t::SGX_SUCCESS } +#[no_mangle] +pub unsafe extern "C" fn init_shard(shard: *const u8, shard_size: u32) -> sgx_status_t { + let shard_identifier = + ShardIdentifier::from_slice(slice::from_raw_parts(shard, shard_size as usize)); + + if let Err(e) = initialization::init_shard(shard_identifier) { + error!("Failed to initialize shard ({:?}): {:?}", shard_identifier, e); + return sgx_status_t::SGX_ERROR_UNEXPECTED + } + + sgx_status_t::SGX_SUCCESS +} + #[no_mangle] pub unsafe extern "C" fn sync_parentchain( blocks_to_sync: *const u8, diff --git a/enclave-runtime/src/test/fixtures/initialize_test_state.rs b/enclave-runtime/src/test/fixtures/initialize_test_state.rs index 87f0907734..456caba0d9 100644 --- a/enclave-runtime/src/test/fixtures/initialize_test_state.rs +++ b/enclave-runtime/src/test/fixtures/initialize_test_state.rs @@ -27,12 +27,13 @@ pub fn init_state>( ) -> (State, ShardIdentifier) { let shard = ShardIdentifier::default(); + let _hash = state_handler.initialize_shard(shard).unwrap(); let (lock, _) = state_handler.load_for_mutation(&shard).unwrap(); let mut state = Stf::init_state(); state.prune_state_diff(); - state_handler.write(state.clone(), lock, &shard).unwrap(); + state_handler.write_after_mutation(state.clone(), lock, &shard).unwrap(); (state, shard) } diff --git a/enclave-runtime/src/test/sidechain_aura_tests.rs b/enclave-runtime/src/test/sidechain_aura_tests.rs index d24c9ae636..303ce27056 100644 --- a/enclave-runtime/src/test/sidechain_aura_tests.rs +++ b/enclave-runtime/src/test/sidechain_aura_tests.rs @@ -194,7 +194,7 @@ pub fn produce_sidechain_block_and_import_it() { get_state_hash(state_handler.as_ref(), &shard_id) ); - let mut state = state_handler.load_initialized(&shard_id).unwrap(); + let mut state = state_handler.load(&shard_id).unwrap(); let free_balance = Stf::account_data(&mut state, &receiver.public().into()).unwrap().free; assert_eq!(free_balance, transfered_amount); } @@ -236,7 +236,7 @@ fn get_state_hashes_from_block( } fn get_state_hash(state_handler: &HandleStateMock, shard_id: &ShardIdentifier) -> H256 { - let state = state_handler.load_initialized(shard_id).unwrap(); + let state = state_handler.load(shard_id).unwrap(); let sidechain_state = TestSidechainDb::new(state); sidechain_state.state_hash() } diff --git a/enclave-runtime/src/tests.rs b/enclave-runtime/src/tests.rs index b3b88900a1..b7c00062e6 100644 --- a/enclave-runtime/src/tests.rs +++ b/enclave-runtime/src/tests.rs @@ -85,11 +85,16 @@ type TestTopPoolAuthor = pub extern "C" fn test_main_entrance() -> size_t { rsgx_unit_tests!( attestation::tests::decode_spid_works, - itp_stf_state_handler::tests::test_write_and_load_state_works, - itp_stf_state_handler::tests::test_sgx_state_decode_encode_works, - itp_stf_state_handler::tests::test_encrypt_decrypt_state_type_works, - itp_stf_state_handler::tests::test_write_access_locks_read_until_finished, - itp_stf_state_handler::tests::test_ensure_subsequent_state_loads_have_same_hash, + itp_stf_state_handler::test::sgx_tests::test_write_and_load_state_works, + itp_stf_state_handler::test::sgx_tests::test_sgx_state_decode_encode_works, + itp_stf_state_handler::test::sgx_tests::test_encrypt_decrypt_state_type_works, + itp_stf_state_handler::test::sgx_tests::test_write_access_locks_read_until_finished, + itp_stf_state_handler::test::sgx_tests::test_ensure_subsequent_state_loads_have_same_hash, + itp_stf_state_handler::test::sgx_tests::test_state_handler_file_backend_is_initialized, + itp_stf_state_handler::test::sgx_tests::test_multiple_state_updates_create_snapshots_up_to_cache_size, + itp_stf_state_handler::test::sgx_tests::test_state_files_from_handler_can_be_loaded_again, + itp_stf_state_handler::test::sgx_tests::test_file_io_get_state_hash_works, + itp_stf_state_handler::test::sgx_tests::test_list_state_ids_ignores_files_not_matching_the_pattern, test_compose_block_and_confirmation, test_submit_trusted_call_to_top_pool, test_submit_trusted_getter_to_top_pool, @@ -109,7 +114,7 @@ pub extern "C" fn test_main_entrance() -> size_t { author_tests::submitting_getter_to_author_when_top_is_filtered_inserts_in_pool, handle_state_mock::tests::initialized_shards_list_is_empty, handle_state_mock::tests::shard_exists_after_inserting, - handle_state_mock::tests::load_initialized_inserts_default_state, + handle_state_mock::tests::initialize_creates_default_state, handle_state_mock::tests::load_mutate_and_write_works, handle_state_mock::tests::ensure_subsequent_state_loads_have_same_hash, handle_state_mock::tests::ensure_encode_and_encrypt_does_not_affect_state_hash, @@ -164,7 +169,7 @@ fn test_compose_block_and_confirmation() { let mut db = SidechainDB::::new(state.clone()); db.set_block_number(&1); let state_hash_before_execution = db.state_hash(); - state_handler.write(db.ext.clone(), lock, &shard).unwrap(); + state_handler.write_after_mutation(db.ext.clone(), lock, &shard).unwrap(); // when let (opaque_call, signed_block) = block_composer @@ -452,14 +457,14 @@ fn test_executing_call_updates_account_nonce() { } // then - let mut state = state_handler.load_initialized(&shard).unwrap(); + let mut state = state_handler.load(&shard).unwrap(); let nonce = Stf::account_nonce(&mut state, &sender.public().into()); assert_eq!(nonce, 1); } fn test_call_set_update_parentchain_block() { let (_, _, shard, _, _, state_handler) = test_setup(); - let mut state = state_handler.load_initialized(&shard).unwrap(); + let mut state = state_handler.load(&shard).unwrap(); let block_number = 3; let parent_hash = H256::from([1; 32]); diff --git a/enclave-runtime/src/tls_ra/seal_handler.rs b/enclave-runtime/src/tls_ra/seal_handler.rs index 009547900c..4303b8efec 100644 --- a/enclave-runtime/src/tls_ra/seal_handler.rs +++ b/enclave-runtime/src/tls_ra/seal_handler.rs @@ -21,40 +21,46 @@ use crate::error::{Error as EnclaveError, Result as EnclaveResult}; use codec::{Decode, Encode}; use ita_stf::{State as StfState, StateType as StfStateType}; -use itp_sgx_crypto::{Aes, AesSeal, Error as CryptoError}; -use itp_sgx_io::SealedIO; -use itp_stf_state_handler::handle_state::HandleState; +use itp_sgx_crypto::{Aes, Error as CryptoError}; +use itp_sgx_io::StaticSealedIO; +use itp_stf_state_handler::{ + handle_state::HandleState, + state_key_repository::{AccessStateKey, MutateStateKey}, +}; use itp_types::ShardIdentifier; use log::*; use sgx_crypto_helper::rsa3072::Rsa3072KeyPair; use std::{marker::PhantomData, sync::Arc, vec::Vec}; -pub trait SealedIOForShieldingKey = SealedIO; -pub trait SealedIOForStateKey = SealedIO; +pub trait SealedIOForShieldingKey = StaticSealedIO; /// Handles the sealing and unsealing of the shielding key, state key and the state. #[derive(Default)] -pub struct SealHandler +pub struct SealHandler where ShieldingKeyHandler: SealedIOForShieldingKey, - StateKeyHandler: SealedIOForStateKey, + StateKeyRepository: AccessStateKey + MutateStateKey, // Constraint StateT = StfState currently necessary because SgxExternalities Encode/Decode does not work. // See https://github.com/integritee-network/sgx-runtime/issues/46. StateHandler: HandleState, { state_handler: Arc, - _phantom_key_handler: PhantomData<(ShieldingKeyHandler, StateKeyHandler)>, + state_key_repository: Arc, + _phantom_key_handler: PhantomData, } -impl - SealHandler +impl + SealHandler where ShieldingKeyHandler: SealedIOForShieldingKey, - StateKeyHandler: SealedIOForStateKey, + StateKeyRepository: AccessStateKey + MutateStateKey, StateHandler: HandleState, { - pub fn new(state_handler: Arc) -> Self { - Self { state_handler, _phantom_key_handler: Default::default() } + pub fn new( + state_handler: Arc, + state_key_repository: Arc, + ) -> Self { + Self { state_handler, state_key_repository, _phantom_key_handler: Default::default() } } } pub trait SealStateAndKeys { @@ -69,11 +75,11 @@ pub trait UnsealStateAndKeys { fn unseal_state(&self, shard: &ShardIdentifier) -> EnclaveResult>; } -impl SealStateAndKeys - for SealHandler +impl SealStateAndKeys + for SealHandler where ShieldingKeyHandler: SealedIOForShieldingKey, - StateKeyHandler: SealedIOForStateKey, + StateKeyRepository: AccessStateKey + MutateStateKey, StateHandler: HandleState, { fn seal_shielding_key(&self, bytes: &[u8]) -> EnclaveResult<()> { @@ -81,14 +87,14 @@ where error!(" [Enclave] Received Invalid RSA key"); EnclaveError::Other(e.into()) })?; - ShieldingKeyHandler::seal(key)?; + ShieldingKeyHandler::seal_to_static_file(key)?; info!("Successfully stored a new shielding key"); Ok(()) } fn seal_state_key(&self, mut bytes: &[u8]) -> EnclaveResult<()> { let aes = Aes::decode(&mut bytes)?; - AesSeal::seal(Aes::new(aes.key, aes.init_vec))?; + self.state_key_repository.update_key(aes)?; info!("Successfully stored a new state key"); Ok(()) } @@ -96,32 +102,34 @@ where fn seal_state(&self, mut bytes: &[u8], shard: &ShardIdentifier) -> EnclaveResult<()> { let state = StfStateType::decode(&mut bytes)?; let state_with_empty_diff = StfState { state, state_diff: Default::default() }; - let (state_lock, _) = self.state_handler.load_for_mutation(shard)?; - self.state_handler.write(state_with_empty_diff, state_lock, shard)?; + self.state_handler.reset(state_with_empty_diff, shard)?; info!("Successfully updated shard {:?} with provisioned state", shard); Ok(()) } } -impl UnsealStateAndKeys - for SealHandler +impl UnsealStateAndKeys + for SealHandler where ShieldingKeyHandler: SealedIOForShieldingKey, - StateKeyHandler: SealedIOForStateKey, + StateKeyRepository: AccessStateKey + MutateStateKey, StateHandler: HandleState, { fn unseal_shielding_key(&self) -> EnclaveResult> { - let shielding_key = ShieldingKeyHandler::unseal()?; + let shielding_key = ShieldingKeyHandler::unseal_from_static_file()?; serde_json::to_vec(&shielding_key).map_err(|e| EnclaveError::Other(e.into())) } fn unseal_state_key(&self) -> EnclaveResult> { - Ok(AesSeal::unseal()?.encode()) + self.state_key_repository + .retrieve_key() + .map(|k| k.encode()) + .map_err(|e| EnclaveError::Other(e.into())) } fn unseal_state(&self, shard: &ShardIdentifier) -> EnclaveResult> { - let state = self.state_handler.load_initialized(shard)?; + let state = self.state_handler.load(shard)?; Ok(state.state.encode()) } } @@ -129,11 +137,13 @@ where #[cfg(feature = "test")] pub mod test { use super::*; - use itp_sgx_crypto::mocks::{AesSealMock, Rsa3072SealMock}; + use itp_sgx_crypto::mocks::sgx::Rsa3072SealMock; + use itp_stf_state_handler::test::mocks::state_key_repository_mock::StateKeyRepositoryMock; use itp_test::mock::handle_state_mock::HandleStateMock; use sgx_externalities::SgxExternalitiesTrait; - type SealHandlerMock = SealHandler; + type SealHandlerMock = + SealHandler, HandleStateMock>; pub fn seal_shielding_key_works() { let seal_handler = SealHandlerMock::default(); @@ -192,6 +202,7 @@ pub mod test { let seal_handler = SealHandlerMock::default(); let state = ::StateT::default(); let shard = ShardIdentifier::default(); + let _init_hash = seal_handler.state_handler.initialize_shard(shard).unwrap(); let result = seal_handler.seal_state(&state.encode(), &shard); @@ -210,11 +221,12 @@ pub mod test { pub fn unseal_seal_state_works() { let seal_handler = SealHandlerMock::default(); let shard = ShardIdentifier::default(); + seal_handler.state_handler.initialize_shard(shard).unwrap(); // Fill our mock state: let (lock, mut state) = seal_handler.state_handler.load_for_mutation(&shard).unwrap(); let (key, value) = ("my_key", "my_value"); state.insert(key.encode(), value.encode()); - seal_handler.state_handler.write(state, lock, &shard).unwrap(); + seal_handler.state_handler.write_after_mutation(state, lock, &shard).unwrap(); let state_in_bytes = seal_handler.unseal_state(&shard).unwrap(); diff --git a/enclave-runtime/src/tls_ra/tls_ra_client.rs b/enclave-runtime/src/tls_ra/tls_ra_client.rs index a7e3acda04..3b6d7537db 100644 --- a/enclave-runtime/src/tls_ra/tls_ra_client.rs +++ b/enclave-runtime/src/tls_ra/tls_ra_client.rs @@ -21,12 +21,14 @@ use super::{authentication::ServerAuth, Opcode, TcpHeader}; use crate::{ attestation::{create_ra_report_and_signature, DEV_HOSTNAME}, error::{Error as EnclaveError, Result as EnclaveResult}, + global_components::GLOBAL_STATE_KEY_REPOSITORY_COMPONENT, ocall::OcallApi, tls_ra::seal_handler::{SealHandler, SealStateAndKeys}, + GLOBAL_STATE_HANDLER_COMPONENT, }; +use itp_component_container::ComponentGetter; use itp_ocall_api::EnclaveAttestationOCallApi; -use itp_sgx_crypto::{AesSeal, Rsa3072Seal}; -use itp_stf_state_handler::GlobalFileStateHandler; +use itp_sgx_crypto::Rsa3072Seal; use itp_types::ShardIdentifier; use log::*; use rustls::{ClientConfig, ClientSession, Stream}; @@ -139,8 +141,23 @@ pub unsafe extern "C" fn request_state_provisioning( let _ = backtrace::enable_backtrace("enclave.signed.so", PrintFormat::Short); let shard = ShardIdentifier::from_slice(slice::from_raw_parts(shard, shard_size as usize)); - let state_handler = Arc::new(GlobalFileStateHandler); - let seal_handler = SealHandler::::new(state_handler); + let state_handler = match GLOBAL_STATE_HANDLER_COMPONENT.get() { + Ok(s) => s, + Err(e) => { + error!("{:?}", e); + return sgx_status_t::SGX_ERROR_UNEXPECTED + }, + }; + + let state_key_repository = match GLOBAL_STATE_KEY_REPOSITORY_COMPONENT.get() { + Ok(s) => s, + Err(e) => { + error!("{:?}", e); + return sgx_status_t::SGX_ERROR_UNEXPECTED + }, + }; + + let seal_handler = SealHandler::::new(state_handler, state_key_repository); if let Err(e) = request_state_provisioning_internal(socket_fd, sign_type, shard, skip_ra, seal_handler) diff --git a/enclave-runtime/src/tls_ra/tls_ra_server.rs b/enclave-runtime/src/tls_ra/tls_ra_server.rs index 9bba9c6805..b7800c10c1 100644 --- a/enclave-runtime/src/tls_ra/tls_ra_server.rs +++ b/enclave-runtime/src/tls_ra/tls_ra_server.rs @@ -21,12 +21,14 @@ use super::{authentication::ClientAuth, Opcode, TcpHeader}; use crate::{ attestation::create_ra_report_and_signature, error::{Error as EnclaveError, Result as EnclaveResult}, + global_components::GLOBAL_STATE_KEY_REPOSITORY_COMPONENT, ocall::OcallApi, tls_ra::seal_handler::{SealHandler, UnsealStateAndKeys}, + GLOBAL_STATE_HANDLER_COMPONENT, }; +use itp_component_container::ComponentGetter; use itp_ocall_api::EnclaveAttestationOCallApi; -use itp_sgx_crypto::{AesSeal, Rsa3072Seal}; -use itp_stf_state_handler::GlobalFileStateHandler; +use itp_sgx_crypto::Rsa3072Seal; use itp_types::ShardIdentifier; use log::*; use rustls::{ServerConfig, ServerSession, Stream}; @@ -37,6 +39,7 @@ use std::{ net::TcpStream, sync::Arc, }; + /// Server part of the TCP-level connection and the underlying TLS-level session. /// /// Includes a seal handler, which handles the reading part of the data to be sent. @@ -104,8 +107,23 @@ pub unsafe extern "C" fn run_state_provisioning_server( ) -> sgx_status_t { let _ = backtrace::enable_backtrace("enclave.signed.so", PrintFormat::Short); - let state_handler = Arc::new(GlobalFileStateHandler); - let seal_handler = SealHandler::::new(state_handler); + let state_handler = match GLOBAL_STATE_HANDLER_COMPONENT.get() { + Ok(s) => s, + Err(e) => { + error!("{:?}", e); + return sgx_status_t::SGX_ERROR_UNEXPECTED + }, + }; + + let state_key_repository = match GLOBAL_STATE_KEY_REPOSITORY_COMPONENT.get() { + Ok(s) => s, + Err(e) => { + error!("{:?}", e); + return sgx_status_t::SGX_ERROR_UNEXPECTED + }, + }; + + let seal_handler = SealHandler::::new(state_handler, state_key_repository); if let Err(e) = run_state_provisioning_server_internal(socket_fd, sign_type, skip_ra, seal_handler) diff --git a/enclave-runtime/src/top_pool_execution.rs b/enclave-runtime/src/top_pool_execution.rs index 899b0f978c..06b17582db 100644 --- a/enclave-runtime/src/top_pool_execution.rs +++ b/enclave-runtime/src/top_pool_execution.rs @@ -41,8 +41,8 @@ use itp_extrinsics_factory::CreateExtrinsics; use itp_ocall_api::{EnclaveOnChainOCallApi, EnclaveSidechainOCallApi}; use itp_settings::sidechain::SLOT_DURATION; use itp_sgx_crypto::Ed25519Seal; -use itp_sgx_io::SealedIO; -use itp_stf_state_handler::{query_shard_state::QueryShardState, GlobalFileStateHandler}; +use itp_sgx_io::StaticSealedIO; +use itp_stf_state_handler::query_shard_state::QueryShardState; use itp_storage_verifier::GetStorageVerified; use itp_time_utils::{duration_now, remaining_time}; use itp_types::{Block, OpaqueCall, H256}; @@ -81,9 +81,9 @@ fn execute_top_pool_trusted_getters_on_all_shards() -> Result<()> { use itp_settings::enclave::MAX_TRUSTED_GETTERS_EXEC_DURATION; let top_pool_executor = GLOBAL_TOP_POOL_OPERATION_HANDLER_COMPONENT.get()?; - - let state_handler = Arc::new(GlobalFileStateHandler); + let state_handler = GLOBAL_STATE_HANDLER_COMPONENT.get()?; let shards = state_handler.list_shards()?; + let mut remaining_shards = shards.len() as u32; let ends_at = duration_now() + MAX_TRUSTED_GETTERS_EXEC_DURATION; @@ -166,7 +166,7 @@ fn execute_top_pool_trusted_calls_internal() -> Result<()> { let extrinsics_factory = GLOBAL_EXTRINSICS_FACTORY_COMPONENT.get()?; let state_handler = GLOBAL_STATE_HANDLER_COMPONENT.get()?; - let authority = Ed25519Seal::unseal()?; + let authority = Ed25519Seal::unseal_from_static_file()?; match yield_next_slot( slot_beginning_timestamp, diff --git a/service/Cargo.toml b/service/Cargo.toml index 00c45d1e54..c2b9a30f3f 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -49,6 +49,7 @@ itp-node-api-extensions = { path = "../core-primitives/node-api-extensions" } itp-enclave-api = { path = "../core-primitives/enclave-api" } itp-enclave-metrics = { path = "../core-primitives/enclave-metrics" } itp-settings = { path = "../core-primitives/settings" } +itp-stf-state-handler = { path = "../core-primitives/stf-state-handler" } itp-test = { path = "../core-primitives/test" } itp-types = { path = "../core-primitives/types" } its-consensus-slots = { path = "../sidechain/consensus/slots" } diff --git a/service/src/main.rs b/service/src/main.rs index 6774d2a1e3..45cacb5d52 100644 --- a/service/src/main.rs +++ b/service/src/main.rs @@ -55,8 +55,8 @@ use itp_node_api_extensions::{ }; use itp_settings::{ files::{ - ENCRYPTED_STATE_FILE, SHARDS_PATH, SHIELDING_KEY_FILE, SIDECHAIN_PURGE_INTERVAL, - SIDECHAIN_PURGE_LIMIT, SIDECHAIN_STORAGE_PATH, SIGNING_KEY_FILE, + SHIELDING_KEY_FILE, SIDECHAIN_PURGE_INTERVAL, SIDECHAIN_PURGE_LIMIT, + SIDECHAIN_STORAGE_PATH, SIGNING_KEY_FILE, }, sidechain::SLOT_DURATION, }; @@ -79,8 +79,7 @@ use sp_finality_grandpa::VersionedAuthorityList; use sp_keyring::AccountKeyring; use std::{ fs::{self, File}, - io::{stdin, Write}, - path::{Path, PathBuf}, + path::PathBuf, str, sync::{ mpsc::{channel, Sender}, @@ -229,7 +228,14 @@ fn main() { println!("{}", enclave.get_mrenclave().unwrap().encode().to_base58()); } else if let Some(_matches) = matches.subcommand_matches("init-shard") { let shard = extract_shard(_matches, enclave.as_ref()); - init_shard(&shard); + match enclave.init_shard(shard.encode()) { + Err(e) => { + println!("Failed to initialize shard {:?}: {:?}", shard, e); + }, + Ok(_) => { + println!("Successfully initialized shard {:?}", shard); + }, + } } else if let Some(_matches) = matches.subcommand_matches("test") { if _matches.is_present("provisioning-server") { println!("*** Running Enclave MU-RA TLS server\n"); @@ -700,25 +706,6 @@ fn execute_trusted_calls(enclave_api: &E) { }; } -fn init_shard(shard: &ShardIdentifier) { - let path = format!("{}/{}", SHARDS_PATH, shard.encode().to_base58()); - println!("initializing shard at {}", path); - fs::create_dir_all(path.clone()).expect("could not create dir"); - - let path = format!("{}/{}", path, ENCRYPTED_STATE_FILE); - if Path::new(&path).exists() { - println!("shard state exists. Overwrite? [y/N]"); - let buffer = &mut String::new(); - stdin().read_line(buffer).unwrap(); - match buffer.trim() { - "y" | "Y" => (), - _ => return, - } - } - let mut file = fs::File::create(path).unwrap(); - file.write_all(b"").unwrap(); -} - /// Get the public signing key of the TEE. fn enclave_account(enclave_api: &E) -> AccountId32 { let tee_public = enclave_api.get_ecc_signing_pubkey().unwrap(); diff --git a/service/src/tests/ecalls.rs b/service/src/tests/ecalls.rs index 9bad446334..62f904f67d 100644 --- a/service/src/tests/ecalls.rs +++ b/service/src/tests/ecalls.rs @@ -15,9 +15,10 @@ */ -use crate::{init_shard, tests::commons::test_trusted_getter_signed}; +use crate::tests::commons::test_trusted_getter_signed; use codec::Encode; use itp_enclave_api::{enclave_base::EnclaveBase, EnclaveResult}; +use itp_stf_state_handler::file_io::purge_shard_dir; use log::*; use sp_core::hash::H256; use sp_keyring::AccountKeyring; @@ -26,12 +27,14 @@ pub fn get_state_works(enclave_api: &E) -> EnclaveResult<()> { let alice = AccountKeyring::Alice; let trusted_getter_signed = test_trusted_getter_signed(alice).encode(); let shard = H256::default(); - init_shard(&shard); + enclave_api.init_shard(shard.encode())?; let res = enclave_api.get_state(trusted_getter_signed, shard.encode())?; debug!("got state value: {:?}", hex::encode(res.clone())); //println!("get_state returned {:?}", res); assert!(!res.is_empty()); + purge_shard_dir(&shard); + Ok(()) } diff --git a/service/src/tests/mocks/enclave_api_mock.rs b/service/src/tests/mocks/enclave_api_mock.rs index c21f37033b..0436b16a3a 100644 --- a/service/src/tests/mocks/enclave_api_mock.rs +++ b/service/src/tests/mocks/enclave_api_mock.rs @@ -47,6 +47,10 @@ impl EnclaveBase for EnclaveBaseMock { Ok(genesis_header) } + fn init_shard(&self, _shard: Vec) -> EnclaveResult<()> { + unimplemented!() + } + fn trigger_parentchain_block_import(&self) -> EnclaveResult<()> { unimplemented!() } diff --git a/sidechain/consensus/aura/src/block_importer.rs b/sidechain/consensus/aura/src/block_importer.rs index 749a5a6ac1..f0968b87a4 100644 --- a/sidechain/consensus/aura/src/block_importer.rs +++ b/sidechain/consensus/aura/src/block_importer.rs @@ -237,7 +237,7 @@ impl< let updated_state = mutating_function(Self::SidechainState::new(state))?; self.state_handler - .write(updated_state.ext, write_lock, shard) + .write_after_mutation(updated_state.ext, write_lock, shard) .map_err(|e| ConsensusError::Other(format!("{:?}", e).into()))?; Ok(()) @@ -253,7 +253,7 @@ impl< { let state = self .state_handler - .load_initialized(shard) + .load(shard) .map_err(|e| ConsensusError::Other(format!("{:?}", e).into()))?; verifying_function(Self::SidechainState::new(state)) } diff --git a/sidechain/consensus/aura/src/test/block_importer_tests.rs b/sidechain/consensus/aura/src/test/block_importer_tests.rs index c97386ee5e..4fb95f17eb 100644 --- a/sidechain/consensus/aura/src/test/block_importer_tests.rs +++ b/sidechain/consensus/aura/src/test/block_importer_tests.rs @@ -75,7 +75,7 @@ fn default_authority() -> Pair { fn test_fixtures( parentchain_block_import_trigger: Arc, ) -> (TestBlockImporter, Arc, Arc) { - let state_handler = Arc::new(HandleStateMock::default()); + let state_handler = Arc::new(HandleStateMock::from_shard(shard()).unwrap()); let top_pool_call_operator = Arc::new(TestTopPoolCallOperator::default()); let ocall_api = Arc::new( OnchainMock::default() @@ -101,7 +101,7 @@ fn test_fixtures_with_default_import_trigger( fn empty_encrypted_state_update(state_handler: &HandleStateMock) -> Vec { let apriori_state_hash = - TestSidechainState::new(state_handler.load_initialized(&shard()).unwrap()).state_hash(); + TestSidechainState::new(state_handler.load(&shard()).unwrap()).state_hash(); let empty_state_diff = SgxExternalitiesDiffType::default(); let mut state_update = StateUpdate::new(apriori_state_hash, apriori_state_hash, empty_state_diff).encode(); diff --git a/sidechain/consensus/common/src/block_import.rs b/sidechain/consensus/common/src/block_import.rs index b3543fe60a..414684f617 100644 --- a/sidechain/consensus/common/src/block_import.rs +++ b/sidechain/consensus/common/src/block_import.rs @@ -25,7 +25,7 @@ use its_primitives::traits::{ Block as SidechainBlockT, ShardIdentifierFor, SignedBlock as SignedSidechainBlockTrait, }; use its_state::{LastBlockExt, SidechainState}; -use log::warn; +use log::*; use sp_runtime::traits::Block as ParentchainBlockTrait; use std::vec::Vec; @@ -111,6 +111,12 @@ where let sidechain_block = signed_sidechain_block.block().clone(); let shard = sidechain_block.shard_id(); + debug!( + "Attempting to import sidechain block (number: {}, parentchain hash: {:?})", + signed_sidechain_block.block().block_number(), + signed_sidechain_block.block().layer_one_head() + ); + let peeked_parentchain_header = self.peek_parentchain_header(&sidechain_block, parentchain_header) .unwrap_or_else(|e| { diff --git a/sidechain/consensus/slots/src/slots.rs b/sidechain/consensus/slots/src/slots.rs index e77240aa56..af616db56a 100644 --- a/sidechain/consensus/slots/src/slots.rs +++ b/sidechain/consensus/slots/src/slots.rs @@ -21,7 +21,7 @@ pub use sp_consensus_slots::Slot; -use itp_sgx_io::SealedIO; +use itp_sgx_io::StaticSealedIO; use itp_time_utils::duration_now; use its_consensus_common::Error as ConsensusError; use its_primitives::traits::{ @@ -127,12 +127,12 @@ pub trait GetLastSlot { fn set_last_slot(&mut self, slot: Slot) -> Result<(), ConsensusError>; } -impl> GetLastSlot for T { +impl> GetLastSlot for T { fn get_last_slot(&self) -> Result { - Self::unseal() + T::unseal_from_static_file() } fn set_last_slot(&mut self, slot: Slot) -> Result<(), ConsensusError> { - Self::seal(slot) + T::seal_to_static_file(slot) } } @@ -140,7 +140,7 @@ impl> GetLastSlot for T { pub mod sgx { use super::*; use codec::{Decode, Encode}; - use itp_sgx_io::{seal, unseal, SealedIO}; + use itp_sgx_io::{seal, unseal, StaticSealedIO}; use lazy_static::lazy_static; use std::sync::SgxRwLock; @@ -152,11 +152,11 @@ pub mod sgx { const LAST_SLOT_BIN: &'static str = "last_slot.bin"; - impl SealedIO for LastSlotSeal { + impl StaticSealedIO for LastSlotSeal { type Error = ConsensusError; type Unsealed = Slot; - fn unseal() -> Result { + fn unseal_from_static_file() -> Result { let _ = FILE_LOCK.read().map_err(|e| Self::Error::Other(format!("{:?}", e).into()))?; match unseal(LAST_SLOT_BIN) { @@ -168,7 +168,7 @@ pub mod sgx { } } - fn seal(unsealed: Self::Unsealed) -> Result<(), Self::Error> { + fn seal_to_static_file(unsealed: Self::Unsealed) -> Result<(), Self::Error> { let _ = FILE_LOCK.write().map_err(|e| Self::Error::Other(format!("{:?}", e).into()))?; Ok(unsealed.using_encoded(|bytes| seal(bytes, LAST_SLOT_BIN))?) } @@ -179,7 +179,7 @@ pub mod sgx { mod tests { use super::*; use core::assert_matches::assert_matches; - use itp_sgx_io::SealedIO; + use itp_sgx_io::StaticSealedIO; use itp_types::{Block as ParentchainBlock, Header as ParentchainHeader}; use its_primitives::{ traits::{Block as BlockT, SignBlock}, @@ -193,15 +193,15 @@ mod tests { struct LastSlotSealMock; - impl SealedIO for LastSlotSealMock { + impl StaticSealedIO for LastSlotSealMock { type Error = ConsensusError; type Unsealed = Slot; - fn unseal() -> Result { + fn unseal_from_static_file() -> Result { Ok(slot_from_time_stamp_and_duration(duration_now(), SLOT_DURATION)) } - fn seal(_unsealed: Self::Unsealed) -> Result<(), Self::Error> { + fn seal_to_static_file(_unsealed: Self::Unsealed) -> Result<(), Self::Error> { println!("Seal method stub called."); Ok(()) }