feat(dan/engine): adds single state storage interface (#4368)

Description
---

Adds simple state storage interface for component state that occurs after execution of a WASM contract

Motivation and Context
---
Introduces a simple state storage API for atomic databases and implements it for RwLock, Sqlite and LMDB. 
Each database was implemented to ensure that the trait provides a good abstraction over the atomic read-write capabilities of each database.
 
How Has This Been Tested?
---
Unit test for each implementation

Cargo.toml

@@ -14,8 +14,11 @@ members = [
     "comms/dht",
     "comms/rpc_macros",
     "dan_layer/core",
-    "dan_layer/template_abi",
+    "dan_layer/storage_lmdb",
     "dan_layer/storage_sqlite",
+    "dan_layer/template_abi",
+    "dan_layer/template_lib",
+    "dan_layer/template_macros",
     "common_sqlite",
     "infrastructure/libtor",
     "infrastructure/metrics",

dan_layer/engine/src/lib.rs

@@ -12,4 +12,5 @@ pub mod crypto;
 pub mod instruction;
 pub mod packager;
 pub mod runtime;
+pub mod state_store;
 pub mod traits;

dan_layer/engine/src/state_store/memory.rs

@@ -0,0 +1,148 @@
+//  Copyright 2022. The Tari Project
+//
+//  Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
+//  following conditions are met:
+//
+//  1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
+//  disclaimer.
+//
+//  2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
+//  following disclaimer in the documentation and/or other materials provided with the distribution.
+//
+//  3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote
+//  products derived from this software without specific prior written permission.
+//
+//  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+//  INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+//  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+//  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+//  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+//  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+//  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+use std::{
+    collections::HashMap,
+    sync::{Arc, RwLock, RwLockReadGuard, RwLockWriteGuard},
+};
+
+use anyhow::anyhow;
+
+use crate::state_store::{AtomicDb, StateReader, StateStoreError, StateWriter};
+
+type InnerKvMap = HashMap<Vec<u8>, Vec<u8>>;
+
+#[derive(Debug, Clone, Default)]
+pub struct MemoryStateStore {
+    state: Arc<RwLock<InnerKvMap>>,
+}
+
+pub struct MemoryTransaction<T> {
+    pending: InnerKvMap,
+    guard: T,
+}
+
+impl<'a> AtomicDb<'a> for MemoryStateStore {
+    type Error = anyhow::Error;
+    type ReadAccess = MemoryTransaction<RwLockReadGuard<'a, InnerKvMap>>;
+    type WriteAccess = MemoryTransaction<RwLockWriteGuard<'a, InnerKvMap>>;
+
+    fn read_access(&'a self) -> Result<Self::ReadAccess, Self::Error> {
+        let guard = self.state.read().map_err(|_| anyhow!("Failed to read state"))?;
+
+        Ok(MemoryTransaction {
+            pending: HashMap::default(),
+            guard,
+        })
+    }
+
+    fn write_access(&'a self) -> Result<Self::WriteAccess, Self::Error> {
+        let guard = self.state.write().map_err(|_| anyhow!("Failed to write state"))?;
+
+        Ok(MemoryTransaction {
+            pending: HashMap::default(),
+            guard,
+        })
+    }
+
+    fn commit(&self, mut tx: Self::WriteAccess) -> Result<(), Self::Error> {
+        tx.guard.extend(tx.pending.into_iter());
+        Ok(())
+    }
+}
+
+impl<'a> StateReader for MemoryTransaction<RwLockReadGuard<'a, InnerKvMap>> {
+    fn get_state_raw(&self, key: &[u8]) -> Result<Option<Vec<u8>>, StateStoreError> {
+        Ok(self.pending.get(key).cloned().or_else(|| self.guard.get(key).cloned()))
+    }
+}
+
+impl<'a> StateReader for MemoryTransaction<RwLockWriteGuard<'a, InnerKvMap>> {
+    fn get_state_raw(&self, key: &[u8]) -> Result<Option<Vec<u8>>, StateStoreError> {
+        Ok(self.pending.get(key).cloned().or_else(|| self.guard.get(key).cloned()))
+    }
+}
+
+impl<'a> StateWriter for MemoryTransaction<RwLockWriteGuard<'a, InnerKvMap>> {
+    fn set_state_raw(&mut self, key: &[u8], value: Vec<u8>) -> Result<(), StateStoreError> {
+        self.pending.insert(key.to_vec(), value);
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use tari_template_abi::{Decode, Encode};
+
+    use super::*;
+
+    #[test]
+    fn read_write() {
+        let store = MemoryStateStore::default();
+        let mut access = store.write_access().unwrap();
+        access.set_state_raw(b"abc", vec![1, 2, 3]).unwrap();
+        let res = access.get_state_raw(b"abc").unwrap();
+        assert_eq!(res, Some(vec![1, 2, 3]));
+        let res = access.get_state_raw(b"def").unwrap();
+        assert_eq!(res, None);
+    }
+
+    #[test]
+    fn read_write_rollback_commit() {
+        #[derive(Debug, Encode, Decode, PartialEq, Eq, Clone)]
+        struct UserData {
+            name: String,
+            age: u8,
+        }
+
+        let user_data = UserData {
+            name: "Foo".to_string(),
+            age: 99,
+        };
+
+        let store = MemoryStateStore::default();
+        {
+            let mut access = store.write_access().unwrap();
+            access.set_state(b"abc", user_data.clone()).unwrap();
+            let res = access.get_state(b"abc").unwrap();
+            assert_eq!(res, Some(user_data.clone()));
+            let res = access.get_state::<_, UserData>(b"def").unwrap();
+            assert_eq!(res, None);
+            // Drop without commit rolls back
+        }
+
+        {
+            let access = store.read_access().unwrap();
+            let res = access.get_state::<_, UserData>(b"abc").unwrap();
+            assert_eq!(res, None);
+        }
+
+        {
+            let mut access = store.write_access().unwrap();
+            access.set_state(b"abc", user_data.clone()).unwrap();
+            store.commit(access).unwrap();
+        }
+
+        let access = store.read_access().unwrap();
+        let res = access.get_state(b"abc").unwrap();
+        assert_eq!(res, Some(user_data));
+    }
+}

dan_layer/engine/src/state_store/mod.rs

@@ -0,0 +1,79 @@
+//  Copyright 2022. The Tari Project
+//
+//  Redistribution and use in source and binary forms, with or without modification, are permitted provided that the
+//  following conditions are met:
+//
+//  1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following
+//  disclaimer.
+//
+//  2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the
+//  following disclaimer in the documentation and/or other materials provided with the distribution.
+//
+//  3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote
+//  products derived from this software without specific prior written permission.
+//
+//  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+//  INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+//  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+//  SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+//  SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+//  WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE
+//  USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+mod memory;
+
+use std::{error::Error, io};
+
+use tari_template_abi::{encode, Decode, Encode};
+
+/// Abstraction for any database that has atomic read/write semantics.
+pub trait AtomicDb<'a> {
+    type Error;
+    type ReadAccess: 'a;
+    type WriteAccess: 'a;
+
+    /// Obtain read access to the underlying database
+    fn read_access(&'a self) -> Result<Self::ReadAccess, Self::Error>;
+
+    /// Obtain write access to the underlying database
+    fn write_access(&'a self) -> Result<Self::WriteAccess, Self::Error>;
+
+    fn commit(&self, tx: Self::WriteAccess) -> Result<(), Self::Error>;
+}
+
+pub trait StateReader {
+    fn get_state_raw(&self, key: &[u8]) -> Result<Option<Vec<u8>>, StateStoreError>;
+
+    fn get_state<K: Encode, V: Decode>(&self, key: &K) -> Result<Option<V>, StateStoreError> {
+        let value = self.get_state_raw(&encode(key)?)?;
+        let value = value.map(|v| V::deserialize(&mut v.as_slice())).transpose()?;
+        Ok(value)
+    }
+}
+
+pub trait StateWriter: StateReader {
+    fn set_state_raw(&mut self, key: &[u8], value: Vec<u8>) -> Result<(), StateStoreError>;
+    fn set_state<K: Encode, V: Encode>(&mut self, key: &K, value: V) -> Result<(), StateStoreError> {
+        self.set_state_raw(&encode(key)?, encode(&value)?)
+    }
+}
+
+#[derive(Debug, thiserror::Error)]
+pub enum StateStoreError {
+    #[error("Encoding error: {0}")]
+    EncodingError(#[from] io::Error),
+    #[error(transparent)]
+    Custom(anyhow::Error),
+    #[error("Error: {0}")]
+    CustomStr(String),
+}
+
+impl StateStoreError {
+    pub fn custom<E: Error + Sync + Send + 'static>(e: E) -> Self {
+        StateStoreError::Custom(e.into())
+    }
+
+    pub fn custom_str(e: &str) -> Self {
+        StateStoreError::CustomStr(e.to_string())
+    }
+}

dan_layer/engine/tests/state/src/lib.rs

@@ -41,5 +41,4 @@ mod state_template {
             self.value
         }
     }
-
-}
+}

dan_layer/storage_lmdb/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "tari_dan_storage_lmdb"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+tari_dan_engine = { path = "../engine" }
+tari_storage = { path = "../../infrastructure/storage" }
+
+borsh = "0.9.3"
+lmdb-zero = "0.4.4"
+
+
+[dev-dependencies]
+tempfile = "3.3.0"