From de437de1de56480e410b1cd82819ebc6238689e2 Mon Sep 17 00:00:00 2001 From: canonbrother Date: Wed, 21 Jun 2023 15:06:08 +0800 Subject: [PATCH 1/3] fix(evm): store genesis block into db (#2084) * mv Bytes.rs from ain_grpc to ain_evm * genesis extra_data vec to bytes * genesis to block db add note for initial base fee * rm unuse * refine test * add test into test_runner * cherry pick 866d1a2672226c0fc73c9a8eab5b5c158f234ac7 * safety default set on genesis into block * minor fix on ethlibs_test ci * Clean up work directory in CI workflow * Clean up work dir * Remove debug * minor fix flaky ethlibs_test ci * ci: specify sender, less flaky * solve q, update note --------- Co-authored-by: jouzo Co-authored-by: Niven --- .github/workflows/tests-ethlibs.yaml | 16 ++++-- ci/ethlibs_test/main.sh | 16 +++--- lib/Cargo.lock | 2 +- lib/ain-evm/Cargo.toml | 1 + lib/ain-evm/genesis.json | 18 +++++++ lib/ain-evm/src/block.rs | 44 +++------------- lib/{ain-grpc => ain-evm}/src/bytes.rs | 0 lib/ain-evm/src/evm.rs | 20 +++++--- lib/ain-evm/src/genesis.rs | 7 +-- lib/ain-evm/src/lib.rs | 1 + lib/ain-evm/src/trie.rs | 6 +-- lib/ain-grpc/Cargo.toml | 1 - lib/ain-grpc/src/block.rs | 2 +- lib/ain-grpc/src/call_request.rs | 2 +- lib/ain-grpc/src/lib.rs | 1 - lib/ain-grpc/src/rpc/eth.rs | 2 +- lib/ain-grpc/src/transaction_log.rs | 2 +- lib/ain-grpc/src/transaction_request.rs | 2 +- test/functional/feature_evm_genesis.py | 68 +++++++++++++++++++++++++ test/functional/test_runner.py | 1 + 20 files changed, 143 insertions(+), 69 deletions(-) create mode 100644 lib/ain-evm/genesis.json rename lib/{ain-grpc => ain-evm}/src/bytes.rs (100%) create mode 100755 test/functional/feature_evm_genesis.py diff --git a/.github/workflows/tests-ethlibs.yaml b/.github/workflows/tests-ethlibs.yaml index a1756b2b21..29b41eb96e 100644 --- a/.github/workflows/tests-ethlibs.yaml +++ b/.github/workflows/tests-ethlibs.yaml @@ -40,7 +40,12 @@ jobs: path: | build/src/defid build/src/defi-cli - ci/ethlibs_test/main.sh + + - name: Upload shell commands + uses: actions/upload-artifact@v3 + with: + name: sync + path: ci/ethlibs_test/main.sh node-rpc-tests: runs-on: ubuntu-latest @@ -52,11 +57,16 @@ jobs: with: name: defibins + - name: Download shell commands + uses: actions/download-artifact@v3 + with: + name: sync + - name: Setup permissions - run: chmod uog+x build/src/defid build/src/defi-cli build/src/defid build/src/defi-cli ci/ethlibs_test/main.sh + run: chmod uog+x ./defid ./defi-cli ./main.sh - name: Setup test fixtures - run: ./ci/ethlibs_test/main.sh + run: ./main.sh - name: Setup Go uses: actions/setup-go@v3 diff --git a/ci/ethlibs_test/main.sh b/ci/ethlibs_test/main.sh index c6d235c92b..9ea4f419c5 100755 --- a/ci/ethlibs_test/main.sh +++ b/ci/ethlibs_test/main.sh @@ -7,9 +7,8 @@ set -Eeuo pipefail setup_vars() { # directories and binaries WORK_DIR=${WORK_DIR:-"$(pwd)"} - BUILD_DIR=${BUILD_DIR:-"${WORK_DIR}/build/src"} - DEFID_BIN=${DEFID_BIN:-"${BUILD_DIR}/defid"} - DEFI_CLI_BIN=${DEFI_CLI_BIN:-"${BUILD_DIR}/defi-cli"} + DEFID_BIN=${DEFID_BIN:-"${WORK_DIR}/defid"} + DEFI_CLI_BIN=${DEFI_CLI_BIN:-"${WORK_DIR}/defi-cli"} # fixture OWNERAUTHADDR="mwsZw8nF7pKxWH8eoKL9tPxTpaFkz7QeLU" @@ -68,16 +67,16 @@ setup_fixtures() { # push fixtures $DEFI_CLI_BIN -regtest importprivkey "$PRIVKEY_ALICE" $DEFI_CLI_BIN -regtest importprivkey "$PRIVKEY_BOB" - $DEFI_CLI_BIN -regtest generatetoaddress 100 "$OWNERAUTHADDR" + $DEFI_CLI_BIN -regtest generatetoaddress 120 "$OWNERAUTHADDR" - $DEFI_CLI_BIN -regtest utxostoaccount '{"'"$OWNERAUTHADDR"'":"5000@DFI"}' + $DEFI_CLI_BIN -regtest utxostoaccount '{"'"$OWNERAUTHADDR"'":"500@DFI"}' $DEFI_CLI_BIN -regtest generatetoaddress 1 "$OWNERAUTHADDR" $DEFI_CLI_BIN -regtest setgov '{"ATTRIBUTES":{"v0/params/feature/evm":"true"}}' $DEFI_CLI_BIN -regtest generatetoaddress 1 "$OWNERAUTHADDR" - $DEFI_CLI_BIN -regtest transferdomain '[{"src":{"address":"'"$OWNERAUTHADDR"'", "amount":"2000@DFI", "domain":2}, "dst":{"address":"'"$ALICE"'", "amount":"2000@DFI", "domain":3}}]' + $DEFI_CLI_BIN -regtest transferdomain '[{"src":{"address":"'"$OWNERAUTHADDR"'", "amount":"200@DFI", "domain":2}, "dst":{"address":"'"$ALICE"'", "amount":"200@DFI", "domain":3}}]' $DEFI_CLI_BIN -regtest generatetoaddress 1 "$OWNERAUTHADDR" - + curl http://localhost:19551 \ -H 'content-type:application/json' \ --data-binary \ @@ -86,6 +85,7 @@ setup_fixtures() { "id":"fixture", "method":"eth_sendTransaction", "params":[{ + "from":"'"$ALICE"'", "data":"'"$CONTRACT_COUNTER"'", "value":"0x00", "gas":"0x7a120", @@ -105,6 +105,7 @@ setup_fixtures() { "id":"fixture", "method":"eth_sendTransaction", "params":[{ + "from":"'"$ALICE"'", "data":"'"$CONTRACT_COUNTERCALLER"'", "value":"0x00", "gas":"0x7a120", @@ -119,7 +120,6 @@ setup_fixtures() { main() { setup_vars - print_info start_node init_node setup_fixtures diff --git a/lib/Cargo.lock b/lib/Cargo.lock index c0f536662e..11ed58d593 100644 --- a/lib/Cargo.lock +++ b/lib/Cargo.lock @@ -97,6 +97,7 @@ dependencies = [ "primitive-types", "rand 0.8.5", "rlp", + "rustc-hex", "serde", "serde_json", "sha3", @@ -137,7 +138,6 @@ dependencies = [ "quote", "regex", "rlp", - "rustc-hex", "serde", "serde_json", "syn 2.0.15", diff --git a/lib/ain-evm/Cargo.toml b/lib/ain-evm/Cargo.toml index b305c95d96..44ea4bb55b 100644 --- a/lib/ain-evm/Cargo.toml +++ b/lib/ain-evm/Cargo.toml @@ -24,6 +24,7 @@ ethbloom = "0.13.0" ethereum-types = "0.14.1" serde_json = "1.0.96" statrs = "0.16.0" +rustc-hex = "2.1.0" # Trie dependencies hash-db = "0.16.0" diff --git a/lib/ain-evm/genesis.json b/lib/ain-evm/genesis.json new file mode 100644 index 0000000000..b1dac632d4 --- /dev/null +++ b/lib/ain-evm/genesis.json @@ -0,0 +1,18 @@ +{ + "coinbase": "0x3333333333333333333333333333333333333333", + "difficulty": "0x400000", + "extraData": "0x686f727365", + "gasLimit": "0x1388", + "mixHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "parentHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "nonce": "0x123123123123123f", + "timestamp": "0x539", + "alloc": { + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x09184e72a000" + } + }, + "config": { + "ethash": {} + } +} diff --git a/lib/ain-evm/src/block.rs b/lib/ain-evm/src/block.rs index c6009e527d..40923f191a 100644 --- a/lib/ain-evm/src/block.rs +++ b/lib/ain-evm/src/block.rs @@ -1,16 +1,13 @@ -use ethereum::{Block, BlockAny, PartialHeader, TransactionAny}; +use ethereum::{BlockAny, TransactionAny}; use keccak_hash::H256; use log::debug; use primitive_types::U256; use statrs::statistics::{Data, OrderStatistics}; use std::cmp::{max, Ordering}; -use std::{fs, io::BufReader, path::PathBuf, sync::Arc}; +use std::sync::Arc; -use crate::{ - genesis::GenesisData, - storage::{traits::BlockStorage, Storage}, -}; +use crate::storage::{traits::BlockStorage, Storage}; pub struct BlockHandler { storage: Arc, @@ -23,6 +20,8 @@ pub struct FeeHistoryData { pub reward: Option>>, } +pub const INITIAL_BASE_FEE: U256 = U256([10_000_000_000, 0, 0, 0]); // wei + impl BlockHandler { pub fn new(storage: Arc) -> Self { Self { storage } @@ -49,13 +48,12 @@ impl BlockHandler { pub fn calculate_base_fee(&self, parent_hash: H256) -> U256 { // constants - let initial_base_fee = U256::from(10_000_000_000u64); // wei let base_fee_max_change_denominator = U256::from(8); let elasticity_multiplier = U256::from(2); // first block has 1 gwei base fee if parent_hash == H256::zero() { - return initial_base_fee; + return INITIAL_BASE_FEE; } // get parent gas usage, @@ -83,7 +81,7 @@ impl BlockHandler { U256::one(), ); - max(parent_base_fee + base_fee_per_gas_delta, initial_base_fee) + max(parent_base_fee + base_fee_per_gas_delta, INITIAL_BASE_FEE) } Ordering::Greater => { let gas_used_delta = parent_gas_target - parent_gas_used; @@ -91,7 +89,7 @@ impl BlockHandler { / parent_gas_target / base_fee_max_change_denominator; - max(parent_base_fee - base_fee_per_gas_delta, initial_base_fee) + max(parent_base_fee - base_fee_per_gas_delta, INITIAL_BASE_FEE) } } } @@ -263,29 +261,3 @@ impl BlockHandler { base_fee + priority_fee } } - -pub fn new_block_from_json(path: PathBuf, state_root: H256) -> Result { - let file = fs::File::open(path)?; - let reader = BufReader::new(file); - let genesis: GenesisData = serde_json::from_reader(reader)?; - - Ok(Block::new( - PartialHeader { - state_root, - number: U256::zero(), - timestamp: genesis.timestamp.unwrap_or_default().as_u64(), - beneficiary: genesis.coinbase.unwrap_or_default(), - difficulty: genesis.difficulty.unwrap_or_default(), - extra_data: genesis.extra_data.unwrap_or_default(), - parent_hash: genesis.parent_hash.unwrap_or_default(), - gas_limit: genesis.gas_limit.unwrap_or_default(), - mix_hash: genesis.mix_hash.unwrap_or_default(), - nonce: genesis.nonce.unwrap_or_default(), - receipts_root: Default::default(), - logs_bloom: Default::default(), - gas_used: Default::default(), - }, - Vec::new(), - Vec::new(), - )) -} diff --git a/lib/ain-grpc/src/bytes.rs b/lib/ain-evm/src/bytes.rs similarity index 100% rename from lib/ain-grpc/src/bytes.rs rename to lib/ain-evm/src/bytes.rs diff --git a/lib/ain-evm/src/evm.rs b/lib/ain-evm/src/evm.rs index 28f99a71ed..a9779fb1dc 100644 --- a/lib/ain-evm/src/evm.rs +++ b/lib/ain-evm/src/evm.rs @@ -1,4 +1,5 @@ use crate::backend::{EVMBackend, EVMBackendError, InsufficientBalance, Vicinity}; +use crate::block::INITIAL_BASE_FEE; use crate::executor::TxResponse; use crate::fee::calculate_prepay_gas; use crate::storage::traits::{BlockStorage, PersistentStateError}; @@ -64,7 +65,7 @@ impl EVMHandler { trie_store: Arc::new(TrieDBStore::new()), storage: Arc::clone(&storage), }; - let state_root = + let (state_root, genesis) = TrieDBStore::genesis_state_root_from_json(&handler.trie_store, &handler.storage, path) .expect("Error getting genesis state root from json"); @@ -72,23 +73,26 @@ impl EVMHandler { PartialHeader { state_root, number: U256::zero(), - parent_hash: Default::default(), beneficiary: Default::default(), receipts_root: Default::default(), logs_bloom: Default::default(), - difficulty: Default::default(), - gas_limit: Default::default(), gas_used: Default::default(), - timestamp: Default::default(), - extra_data: Default::default(), - mix_hash: Default::default(), - nonce: Default::default(), + gas_limit: genesis.gas_limit.unwrap_or(U256::from(MAX_GAS_PER_BLOCK)), + extra_data: genesis.extra_data.unwrap_or_default().into(), + parent_hash: genesis.parent_hash.unwrap_or_default(), + mix_hash: genesis.mix_hash.unwrap_or_default(), + nonce: genesis.nonce.unwrap_or_default(), + timestamp: genesis.timestamp.unwrap_or_default().as_u64(), + difficulty: genesis.difficulty.unwrap_or_default(), }, Vec::new(), Vec::new(), ); storage.put_latest_block(Some(&block)); storage.put_block(&block); + // NOTE(canonbrother): set an initial base fee for genesis block + // https://github.com/ethereum/go-ethereum/blob/46ec972c9c56a4e0d97d812f2eaf9e3657c66276/params/protocol_params.go#LL125C2-L125C16 + storage.set_base_fee(block.header.hash(), INITIAL_BASE_FEE); handler } diff --git a/lib/ain-evm/src/genesis.rs b/lib/ain-evm/src/genesis.rs index e1da5653b8..2cb7e26d60 100644 --- a/lib/ain-evm/src/genesis.rs +++ b/lib/ain-evm/src/genesis.rs @@ -1,3 +1,4 @@ +use crate::bytes::Bytes; use ethereum_types::{H160, H256, H64, U256}; use serde::{Deserialize, Serialize}; use std::collections::HashMap; @@ -12,20 +13,20 @@ struct Config { eip158_block: u32, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] pub struct Alloc { pub balance: U256, pub code: Option>, pub storage: Option>, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(rename_all = "camelCase")] pub struct GenesisData { // config: Config, pub coinbase: Option, pub difficulty: Option, - pub extra_data: Option>, + pub extra_data: Option, pub gas_limit: Option, pub nonce: Option, pub timestamp: Option, diff --git a/lib/ain-evm/src/lib.rs b/lib/ain-evm/src/lib.rs index f191f9fe07..c14a8ca32e 100644 --- a/lib/ain-evm/src/lib.rs +++ b/lib/ain-evm/src/lib.rs @@ -1,5 +1,6 @@ mod backend; pub mod block; +pub mod bytes; mod ecrecover; pub mod evm; pub mod executor; diff --git a/lib/ain-evm/src/trie.rs b/lib/ain-evm/src/trie.rs index 527154ecbf..d15973ab0a 100644 --- a/lib/ain-evm/src/trie.rs +++ b/lib/ain-evm/src/trie.rs @@ -56,7 +56,7 @@ impl TrieDBStore { trie_store: &Arc, storage: &Arc, json_file: PathBuf, - ) -> Result { + ) -> Result<(H256, GenesisData), std::io::Error> { let state_root: H256 = GENESIS_STATE_ROOT.parse().unwrap(); let mut backend = EVMBackend::from_root( @@ -71,7 +71,7 @@ impl TrieDBStore { let reader = BufReader::new(file); let genesis: GenesisData = serde_json::from_reader(reader)?; - if let Some(alloc) = genesis.alloc { + if let Some(alloc) = genesis.clone().alloc { for (address, data) in alloc { debug!("Setting data {:#?} for address {:x?}", data, address); let basic = backend.basic(address); @@ -95,7 +95,7 @@ impl TrieDBStore { let state_root = backend.commit(); debug!("Loaded genesis state_root : {:#x}", state_root); - Ok(state_root) + Ok((state_root, genesis)) } pub fn flush(&self) -> Result<(), PersistentStateError> { diff --git a/lib/ain-grpc/Cargo.toml b/lib/ain-grpc/Cargo.toml index 9bdf4c5d46..5b9a6470e7 100644 --- a/lib/ain-grpc/Cargo.toml +++ b/lib/ain-grpc/Cargo.toml @@ -25,7 +25,6 @@ ethereum-types = "0.14.1" hex = "0.4.3" async-trait = "0.1.68" rlp = "0.5.2" -rustc-hex = "2.1.0" [build-dependencies] cxx-gen = "0.7" diff --git a/lib/ain-grpc/src/block.rs b/lib/ain-grpc/src/block.rs index 9811a9fcde..085a27836d 100644 --- a/lib/ain-grpc/src/block.rs +++ b/lib/ain-grpc/src/block.rs @@ -1,4 +1,4 @@ -use crate::bytes::Bytes; +use ain_evm::bytes::Bytes; use ethereum::{BlockAny, TransactionV2}; use ethereum_types::H64; use primitive_types::{H160, H256, U256}; diff --git a/lib/ain-grpc/src/call_request.rs b/lib/ain-grpc/src/call_request.rs index 459f6c743e..53d00d5414 100644 --- a/lib/ain-grpc/src/call_request.rs +++ b/lib/ain-grpc/src/call_request.rs @@ -2,7 +2,7 @@ use ethereum::AccessListItem; use primitive_types::{H160, U256}; use serde::Deserialize; -use crate::bytes::Bytes; +use ain_evm::bytes::Bytes; /// Call request #[derive(Clone, Debug, Default, Eq, PartialEq, Deserialize, Serialize)] diff --git a/lib/ain-grpc/src/lib.rs b/lib/ain-grpc/src/lib.rs index 0f18a89f01..5450dcd492 100644 --- a/lib/ain-grpc/src/lib.rs +++ b/lib/ain-grpc/src/lib.rs @@ -3,7 +3,6 @@ extern crate serde; extern crate serde_json; pub mod block; -mod bytes; pub mod call_request; pub mod codegen; mod impls; diff --git a/lib/ain-grpc/src/rpc/eth.rs b/lib/ain-grpc/src/rpc/eth.rs index 19c2806556..b746f19cc4 100644 --- a/lib/ain-grpc/src/rpc/eth.rs +++ b/lib/ain-grpc/src/rpc/eth.rs @@ -1,7 +1,7 @@ use crate::block::{BlockNumber, RpcBlock, RpcFeeHistory}; -use crate::bytes::Bytes; use crate::call_request::CallRequest; use crate::codegen::types::EthTransactionInfo; +use ain_evm::bytes::Bytes; use crate::receipt::ReceiptResult; use crate::transaction_request::{TransactionMessage, TransactionRequest}; diff --git a/lib/ain-grpc/src/transaction_log.rs b/lib/ain-grpc/src/transaction_log.rs index f10392e18e..d8113ce6a5 100644 --- a/lib/ain-grpc/src/transaction_log.rs +++ b/lib/ain-grpc/src/transaction_log.rs @@ -1,5 +1,5 @@ use crate::block::BlockNumber; -use crate::bytes::Bytes; +use ain_evm::bytes::Bytes; use ain_evm::log::LogIndex; use primitive_types::{H160, H256, U256}; diff --git a/lib/ain-grpc/src/transaction_request.rs b/lib/ain-grpc/src/transaction_request.rs index 0687a834b9..27f31a57a5 100644 --- a/lib/ain-grpc/src/transaction_request.rs +++ b/lib/ain-grpc/src/transaction_request.rs @@ -1,4 +1,4 @@ -use crate::bytes::Bytes; +use ain_evm::bytes::Bytes; use ethereum::{ AccessListItem, EIP1559TransactionMessage, EIP2930TransactionMessage, LegacyTransactionMessage, }; diff --git a/test/functional/feature_evm_genesis.py b/test/functional/feature_evm_genesis.py new file mode 100755 index 0000000000..39246d308c --- /dev/null +++ b/test/functional/feature_evm_genesis.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# Copyright (c) 2014-2019 The Bitcoin Core developers +# Copyright (c) DeFi Blockchain Developers +# Distributed under the MIT software license, see the accompanying +# file LICENSE or http://www.opensource.org/licenses/mit-license.php. +"""Test EVM behaviour""" +import os + +from test_framework.test_framework import DefiTestFramework +from test_framework.util import ( + assert_equal, +) + +class EVMTest(DefiTestFramework): + def set_test_params(self): + ain = os.path.dirname( + os.path.dirname(os.path.dirname(os.path.realpath(__file__))) + ) + genesis = os.path.join(ain, "lib/ain-evm/genesis.json") + + self.num_nodes = 1 + self.setup_clean_chain = True + self.extra_args = [ + [ + "-ethstartstate={}".format(genesis), + "-dummypos=0", + "-txnotokens=0", + "-amkheight=50", + "-bayfrontheight=51", + "-eunosheight=80", + "-fortcanningheight=82", + "-fortcanninghillheight=84", + "-fortcanningroadheight=86", + "-fortcanningcrunchheight=88", + "-fortcanningspringheight=90", + "-fortcanninggreatworldheight=94", + "-fortcanningepilogueheight=96", + "-grandcentralheight=101", + "-nextnetworkupgradeheight=105", + "-subsidytest=1", + "-txindex=1", + ], + ] + + def run_test(self): + self.nodes[0].generate(111) + + ethblock0 = self.nodes[0].eth_getBlockByNumber(0) + assert_equal(ethblock0["difficulty"], "0x400000") + assert_equal(ethblock0["extraData"], "0x686f727365") + assert_equal(ethblock0["gasLimit"], "0x1388") + assert_equal( + ethblock0["parentHash"], + "0x0000000000000000000000000000000000000000000000000000000000000000", + ) + # NOTE(canonbrother): overwritten by block.header.hash + # assert_equal(ethblock0['mixHash'], '0x0000000000000000000000000000000000000000000000000000000000000000') + assert_equal(ethblock0["nonce"], "0x123123123123123f") + assert_equal(ethblock0["timestamp"], "0x539") + + balance = self.nodes[0].eth_getBalance( + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b" + ) + assert_equal(balance, "0x9184e72a000") + + +if __name__ == "__main__": + EVMTest().main() diff --git a/test/functional/test_runner.py b/test/functional/test_runner.py index e4fef220d8..8e3c8306eb 100755 --- a/test/functional/test_runner.py +++ b/test/functional/test_runner.py @@ -293,6 +293,7 @@ 'feature_loan.py', 'feature_evm_contracts.py', 'feature_evm_fee.py', + 'feature_evm_genesis.py', 'feature_evm_rollback.py', 'feature_evm_rpc_transaction.py', 'feature_evm_rpc.py', From 3d51c312f9e2a7392667436dbea4f7bd9de080b6 Mon Sep 17 00:00:00 2001 From: Niven Date: Wed, 21 Jun 2023 15:13:04 +0800 Subject: [PATCH 2/3] CI: Fix test sync workflow (#2091) * Fix to allow defid shutdown transition, refactor for shell scripting * Refactor stopping node into a func * Add monitoring when starting defid node * Change start node log * Add attempt var * Increase timeout threshold for stop_node * Increase timeout threshold * Refactor to start_node_and_wait method, and store PID when starting node * Change to read defid pid config file instead * Add debugging logs, clean up starting node in presync * Format * Fix bug * Fix filepath bug * Sleep node before getting PID * Remove debugging logs --- ci/sync/main.sh | 115 +++++++++++++++++++++++++++--------------------- 1 file changed, 66 insertions(+), 49 deletions(-) diff --git a/ci/sync/main.sh b/ci/sync/main.sh index 828ad556fe..8f11c04f1c 100755 --- a/ci/sync/main.sh +++ b/ci/sync/main.sh @@ -10,29 +10,30 @@ setup_vars() { # Files and directories DATADIR=${DATADIR:-".defi"} - DEBUG_FILE="$DATADIR/debug.log" - CONF_FILE="$DATADIR/defi.conf" - TMP_LOG=debug-tmp-$STOP_BLOCK.log + DEBUG_FILE="${DATADIR}/debug.log" + CONF_FILE="${DATADIR}/defi.conf" + TMP_LOG="debug-tmp-${STOP_BLOCK}.log" PRE_ROLLBACK_LOG="debug-pre-rollback.log" POST_ROLLBACK_LOG="debug-post-rollback.log" BASE_REF=${BASE_REF:-"master"} - BASE_PATH=https://storage.googleapis.com - BUCKET=team-drop - REF_LOG=debug-$STOP_BLOCK.log - REF_LOG_PATH=$BASE_PATH/$BUCKET/$REF_LOG_DIR/$REF_LOG + BASE_PATH="https://storage.googleapis.com" + BUCKET="team-drop" + REF_LOG="debug-${STOP_BLOCK}.log" + REF_LOG_PATH="${BASE_PATH}/${BUCKET}/${REF_LOG_DIR}/${REF_LOG}" # Commands - DEFID_CMD="$DEFID_BIN -datadir=$DATADIR -daemon -debug=accountchange -spv -checkpoints=0" - DEFI_CLI_CMD="$DEFI_CLI_BIN -datadir=$DATADIR" + DEFID_CMD="${DEFID_BIN} -datadir=${DATADIR} -daemon -debug=accountchange -spv -checkpoints=0 -interrupt-block=$((STOP_BLOCK + 1))" + DEFI_CLI_CMD="${DEFI_CLI_BIN} -datadir=${DATADIR}" FETCH="aria2c -x16 -s16" GREP="grep" + ROLLBACK_BLOCK="${START_BLOCK}" BLOCK=0 - ROLLBACK_BLOCK=$START_BLOCK ATTEMPTS=0 MAX_ATTEMPTS=10 MAX_NODE_RESTARTS=5 NODE_RESTARTS=0 + PID="" } _ensure_script_dir() { @@ -48,28 +49,28 @@ _cleanup() { print_info() { echo "======== Sync Test Info ========== - - Block range: $START_BLOCK - $STOP_BLOCK + - Block range: ${START_BLOCK} - ${STOP_BLOCK} - Reference log: - $REF_LOG_PATH + ${REF_LOG_PATH} - snapshot: - https://storage.googleapis.com/team-drop/$BASE_REF-datadir/datadir-$START_BLOCK.tar.gz + https://storage.googleapis.com/team-drop/${BASE_REF}-datadir/datadir-${START_BLOCK}.tar.gz - defid: - $DEFID_CMD + ${DEFID_CMD} - defi-cli: - $DEFI_CLI_CMD + ${DEFI_CLI_CMD} - Create log commands: - $GREP \"AccountChange:\" \"$DEBUG_FILE\" | cut -d\" \" -f2- - $DEFI_CLI_CMD logaccountbalances - $DEFI_CLI_CMD spv_listanchors - $DEFI_CLI_CMD logstoredinterests - $DEFI_CLI_CMD listvaults '{\"verbose\": true}' '{\"limit\":1000000}' - $DEFI_CLI_CMD listtokens '{\"limit\":1000000}' - $DEFI_CLI_CMD getburninfo + ${GREP} \"AccountChange:\" \"${DEBUG_FILE}\" | cut -d\" \" -f2- + ${DEFI_CLI_CMD} logaccountbalances + ${DEFI_CLI_CMD} spv_listanchors + ${DEFI_CLI_CMD} logstoredinterests + ${DEFI_CLI_CMD} listvaults '{\"verbose\": true}' '{\"limit\":1000000}' + ${DEFI_CLI_CMD} listtokens '{\"limit\":1000000}' + ${DEFI_CLI_CMD} getburninfo - defi.conf: $(cat "$CONF_FILE") @@ -93,9 +94,9 @@ get_full_log() { } rollback_and_log() { - echo "ROLLBACK_BLOCK : $ROLLBACK_BLOCK" + echo "ROLLBACK_BLOCK : ${ROLLBACK_BLOCK}" ROLLBACK_HASH=$($DEFI_CLI_CMD getblockhash $((ROLLBACK_BLOCK))) - echo "ROLLBACK_HASH : $ROLLBACK_HASH" + echo "ROLLBACK_HASH : ${ROLLBACK_HASH}" $DEFI_CLI_CMD invalidateblock "$ROLLBACK_HASH" echo "Rolled back to block : $($DEFI_CLI_CMD getblockcount)" @@ -103,26 +104,44 @@ rollback_and_log() { } create_pre_sync_rollback_log() { - local DATADIR_ROLLBACK="$DATADIR-rollback" - local DEFID_CMD="$DEFID_BIN -datadir=$DATADIR_ROLLBACK -daemon -debug=accountchange -spv -rpcport=9999 -port=9998 -connect=0 -checkpoints=0 -interrupt-block=$((START_BLOCK + 1))" - local DEFI_CLI_CMD="$DEFI_CLI_BIN -datadir=$DATADIR_ROLLBACK -rpcport=9999" - local DEBUG_FILE="$DATADIR_ROLLBACK/debug.log" + local DATADIR_ROLLBACK="${DATADIR}-rollback" + local DEFID_CMD="${DEFID_BIN} -datadir=${DATADIR_ROLLBACK} -daemon -debug=accountchange -spv -rpcport=9999 -port=9998 -connect=0 -checkpoints=0 -interrupt-block=$((START_BLOCK + 1))" + local DEFI_CLI_CMD="${DEFI_CLI_BIN} -datadir=${DATADIR_ROLLBACK} -rpcport=9999" + local DEBUG_FILE="${DATADIR_ROLLBACK}/debug.log" cp -r "$DATADIR" "$DATADIR_ROLLBACK" rm -f "$DEBUG_FILE" + start_node_and_wait "$DATADIR_ROLLBACK" + rollback_and_log > "$PRE_ROLLBACK_LOG" + stop_node +} + +start_node_and_wait() { + local data_dir=${1:-${DATADIR}} + echo "Syncing to block height: ${STOP_BLOCK}" $DEFID_CMD sleep 90 - rollback_and_log >"$PRE_ROLLBACK_LOG" - - $DEFI_CLI_CMD stop + # get PID + PID=$(head -1 "./${data_dir}/defid.pid") } -# Start defid -start_node() { - echo "Syncing to block height: $STOP_BLOCK" - $DEFID_CMD -interrupt-block=$((STOP_BLOCK + 1)) - sleep 30 +stop_node() { + local ATTEMPTS=0 + + # check to ensure defid process stops (50s timeout threshold) + if [ -n "$PID" ]; then + $DEFI_CLI_CMD stop + while ps -p "$PID" > /dev/null; do + if [ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]; then + echo "Failed to stop node, exiting" + exit 1 + else + ATTEMPTS=$((ATTEMPTS + 1)) + sleep 5 + fi + done + fi } main() { @@ -133,23 +152,22 @@ main() { setup_vars print_info create_pre_sync_rollback_log - start_node + start_node_and_wait # Sync to target block height while [ "$BLOCK" -lt "$STOP_BLOCK" ]; do if [ "$ATTEMPTS" -gt "$MAX_ATTEMPTS" ]; then if [ "$NODE_RESTARTS" -lt "$MAX_NODE_RESTARTS" ]; then - echo "Node Stuck After $ATTEMPTS attempts, restarting node" - $DEFI_CLI_CMD stop - sleep 20 - start_node + echo "Node Stuck After ${ATTEMPTS} attempts, restarting node" + stop_node + start_node_and_wait NODE_RESTARTS=$((NODE_RESTARTS + 1)) ATTEMPTS=0 else exit 1 fi fi - CUR_BLOCK=$($DEFI_CLI_CMD getblockcount || echo $BLOCK) + CUR_BLOCK=$($DEFI_CLI_CMD getblockcount || echo "$BLOCK") if [ "$CUR_BLOCK" -eq "$BLOCK" ]; then ATTEMPTS=$((ATTEMPTS + 1)) @@ -158,22 +176,21 @@ main() { else ATTEMPTS=0 fi - BLOCK=${CUR_BLOCK:-$BLOCK} - echo "Current block: $BLOCK" + BLOCK=${CUR_BLOCK:-${BLOCK}} + echo "Current block: ${BLOCK}" sleep 20 done # Create temporary log file - get_full_log >>"$TMP_LOG" + get_full_log >> "$TMP_LOG" # Download reference log file - echo "Downloading reference log file : $REF_LOG_PATH" + echo "Downloading reference log file : ${REF_LOG_PATH}" $FETCH "$REF_LOG_PATH" # Create rollback log after sync - rollback_and_log >"$POST_ROLLBACK_LOG" - - $DEFI_CLI_CMD stop + rollback_and_log > "$POST_ROLLBACK_LOG" + stop_node } main "$@" From 6d30278999174056079b5d699f880bbf35295d77 Mon Sep 17 00:00:00 2001 From: kuegi <29012906+kuegi@users.noreply.github.com> Date: Wed, 21 Jun 2023 10:05:20 +0200 Subject: [PATCH 3/3] DVM: Enable DUSD loops in vaults (#1971) * allow DUSD Loops with 100% DUSD coll * added tests * fix lint errors * fix tests * added gc height to test * added testcases * lint * fix test * test: fix oracle * tests: fix oracle update in tests. --------- Co-authored-by: Peter John Bushnell --- src/masternodes/govvariables/attributes.cpp | 6 +- src/masternodes/govvariables/attributes.h | 1 + src/masternodes/mn_checks.cpp | 19 +++++ test/functional/feature_dusd_loans.py | 89 +++++++++++++++++++++ 4 files changed, 114 insertions(+), 1 deletion(-) diff --git a/src/masternodes/govvariables/attributes.cpp b/src/masternodes/govvariables/attributes.cpp index ea6692d6c9..325bce1d0e 100644 --- a/src/masternodes/govvariables/attributes.cpp +++ b/src/masternodes/govvariables/attributes.cpp @@ -196,6 +196,7 @@ const std::map> &ATTRIBUTES::allowedKeys {"gov-payout", DFIPKeys::CFPPayout}, {"emission-unused-fund", DFIPKeys::EmissionUnusedFund}, {"mint-tokens-to-address", DFIPKeys::MintTokens}, + {"allow-dusd-loops", DFIPKeys::AllowDUSDLoops}, }}, {AttributeTypes::Governance, { @@ -273,6 +274,7 @@ const std::map> &ATTRIBUTES::displayKeys {DFIPKeys::CFPPayout, "gov-payout"}, {DFIPKeys::EmissionUnusedFund, "emission-unused-fund"}, {DFIPKeys::MintTokens, "mint-tokens-to-address"}, + {DFIPKeys::AllowDUSDLoops, "allow-dusd-loops"}, }}, {AttributeTypes::Live, { @@ -601,6 +603,7 @@ const std::map( {DFIPKeys::CFPPayout, VerifyBool}, {DFIPKeys::EmissionUnusedFund, VerifyBool}, {DFIPKeys::MintTokens, VerifyBool}, + {DFIPKeys::AllowDUSDLoops, VerifyBool}, }}, {AttributeTypes::Locks, { @@ -811,7 +814,8 @@ Res ATTRIBUTES::ProcessVariable(const std::string &key, typeKey != DFIPKeys::MNSetOwnerAddress && typeKey != DFIPKeys::GovernanceEnabled && typeKey != DFIPKeys::ConsortiumEnabled && typeKey != DFIPKeys::CFPPayout && typeKey != DFIPKeys::EmissionUnusedFund && typeKey != DFIPKeys::MintTokens && - typeKey != DFIPKeys::EVMEnabled && typeKey != DFIPKeys::ICXEnabled) { + typeKey != DFIPKeys::EVMEnabled && typeKey != DFIPKeys::ICXEnabled && + typeKey != DFIPKeys::AllowDUSDLoops) { return DeFiErrors::GovVarVariableUnsupportedFeatureType(typeKey); } } else if (typeId == ParamIDs::Foundation) { diff --git a/src/masternodes/govvariables/attributes.h b/src/masternodes/govvariables/attributes.h index 0db2a225aa..7a1963767b 100644 --- a/src/masternodes/govvariables/attributes.h +++ b/src/masternodes/govvariables/attributes.h @@ -88,6 +88,7 @@ enum DFIPKeys : uint8_t { MintTokens = 't', EVMEnabled = 'u', ICXEnabled = 'v', + AllowDUSDLoops = 'w', }; enum GovernanceKeys : uint8_t { diff --git a/src/masternodes/mn_checks.cpp b/src/masternodes/mn_checks.cpp index 2d8748f551..8c2e37f80c 100644 --- a/src/masternodes/mn_checks.cpp +++ b/src/masternodes/mn_checks.cpp @@ -3009,6 +3009,10 @@ class CCustomTxApplyVisitor : public CCustomTxVisitor { CAmount factorDUSD = 0; CAmount factorDFI = 0; + + auto hasDUSDColl = false; + auto hasOtherColl = false; + for (auto &col : vaultAssets.collaterals) { auto token = mnview.GetCollateralTokenFromAttributes(col.nTokenId); @@ -3020,6 +3024,9 @@ class CCustomTxApplyVisitor : public CCustomTxVisitor { if (tokenDUSD && col.nTokenId == tokenDUSD->first) { totalCollateralsDUSD += col.nValue; factorDUSD = token->factor; + hasDUSDColl= true; + } else { + hasOtherColl = true; } } @@ -3029,6 +3036,18 @@ class CCustomTxApplyVisitor : public CCustomTxVisitor { auto isPostFCE = static_cast(height) >= consensus.FortCanningEpilogueHeight; auto isPostFCR = static_cast(height) >= consensus.FortCanningRoadHeight; auto isPostGC = static_cast(height) >= consensus.GrandCentralHeight; + auto isPostNext = static_cast(height) >= consensus.NextNetworkUpgradeHeight; + + if(isPostNext) { + const CDataStructureV0 enabledKey{AttributeTypes::Param, ParamIDs::Feature, DFIPKeys::AllowDUSDLoops}; + auto attributes = mnview.GetAttributes(); + assert(attributes); + auto DUSDLoopsAllowed= attributes->GetValue(enabledKey, false); + if(DUSDLoopsAllowed && hasDUSDColl && !hasOtherColl) { + return Res::Ok(); //every loan ok when DUSD loops allowed and 100% DUSD collateral + } + } + if (isPostGC) { totalCollateralsDUSD = MultiplyAmounts(totalCollateralsDUSD, factorDUSD); diff --git a/test/functional/feature_dusd_loans.py b/test/functional/feature_dusd_loans.py index c90038fd2c..7d808bd739 100755 --- a/test/functional/feature_dusd_loans.py +++ b/test/functional/feature_dusd_loans.py @@ -31,6 +31,10 @@ def set_test_params(self): self.fortcanningroadheight = 2000 self.fortcanninggreatworldheight = 3000 self.fortcanningepilogueheight = 4000 + self.grandcentralheight = 4500 + self.grandcentralepilogueheight = 5000 + self.nextnetworkupgradeheight = 6000 + self.extra_args = [ ['-txnotokens=0', '-amkheight=1', @@ -43,6 +47,10 @@ def set_test_params(self): f'-fortcanningroadheight={self.fortcanningroadheight}', f'-fortcanninggreatworldheight={self.fortcanninggreatworldheight}', f'-fortcanningepilogueheight={self.fortcanningepilogueheight}', + f'-grandcentralheight={self.grandcentralheight}', + f'-grandcentralepilogueheight={self.grandcentralepilogueheight}', + f'-nextnetworkupgradeheight={self.nextnetworkupgradeheight}', + '-jellyfish_regtest=1', '-txindex=1', '-simulatemainnet=1'] ] @@ -110,6 +118,20 @@ def goto_fce_height(self): blockchainInfo = self.nodes[0].getblockchaininfo() assert_equal(blockchainInfo["softforks"]["fortcanningepilogue"]["active"], True) + def goto_gce_height(self): + blockHeight = self.nodes[0].getblockcount() + if self.grandcentralepilogueheight > blockHeight: + self.nodes[0].generate((self.grandcentralepilogueheight - blockHeight) + 2) + blockchainInfo = self.nodes[0].getblockchaininfo() + assert_equal(blockchainInfo["softforks"]["grandcentralepilogue"]["active"], True) + + def goto_next_height(self): + blockHeight = self.nodes[0].getblockcount() + if self.nextnetworkupgradeheight > blockHeight: + self.nodes[0].generate((self.nextnetworkupgradeheight - blockHeight) + 2) + blockchainInfo = self.nodes[0].getblockchaininfo() + assert_equal(blockchainInfo["softforks"]["nextnetworkupgrade"]["active"], True) + def create_tokens(self): self.symbolDFI = "DFI" self.symboldUSD = "DUSD" @@ -504,6 +526,70 @@ def post_FCE_DFI_minimum_check_takeloan(self): self.rollback_to(block_height) self.rollback_checks([vault_id, vault_id_1]) + def check_looped_dusd(self): + block_height = self.nodes[0].getblockcount() + + vault_id = self.new_vault('LOAN1', ["100.00000000@DUSD"]) + + self.goto_gce_height() + assert_raises_rpc_error(-32600, + ERR_STRING_MIN_COLLATERAL_DFI_PCT, + self.takeloan_withdraw, vault_id, "1.00000000@DUSD", 'takeloan') + + self.nodes[0].setgov({"ATTRIBUTES": {'v0/params/feature/allow-dusd-loops': 'true'}}) + self.nodes[0].generate(1) + + #still not allowed before hardfork + assert_raises_rpc_error(-32600, + ERR_STRING_MIN_COLLATERAL_DFI_PCT, + self.takeloan_withdraw, vault_id, "1.00000000@DUSD", 'takeloan') + self.goto_next_height() + + # min coll ratio still applies + assert_raises_rpc_error(-32600, + "Vault does not have enough collateralization ratio defined by loan scheme - 148 < 150", + self.takeloan_withdraw, vault_id, "67.00000000@DUSD", 'takeloan') + + + self.nodes[0].generate(1) + self.takeloan_withdraw(vault_id, "1.00000000@DUSD", 'takeloan') + self.nodes[0].generate(1) + self.takeloan_withdraw(vault_id, "1.00000000@DUSD", 'withdraw') + self.nodes[0].generate(1) + + # not sure why this is needed like this. but it works + self.update_oracle_price(13000) + #also fails with other crypto in + self.nodes[0].deposittovault(vault_id, self.account0, "100.00000000@BTC") + self.nodes[0].generate(1) + assert_raises_rpc_error(-32600, + ERR_STRING_MIN_COLLATERAL_DFI_PCT, + self.takeloan_withdraw, vault_id, "1.00000000@DUSD", 'takeloan') + + assert_raises_rpc_error(-32600, + ERR_STRING_MIN_COLLATERAL_DFI_PCT, + self.takeloan_withdraw, vault_id, "5.00000000@BTC", 'withdraw') + + # full withdrawal (go to 100% DUSD) should work + self.takeloan_withdraw(vault_id, "100.00000000@BTC", 'withdraw') + self.nodes[0].generate(1) + + + self.nodes[0].setgov({"ATTRIBUTES": {'v0/params/feature/allow-dusd-loops': 'false'}}) + self.nodes[0].generate(1) + + #not allowed if attribute false + + assert_raises_rpc_error(-32600, + ERR_STRING_MIN_COLLATERAL_DFI_PCT, + self.takeloan_withdraw, vault_id, "1.00000000@DUSD", 'takeloan') + assert_raises_rpc_error(-32600, + ERR_STRING_MIN_COLLATERAL_DFI_PCT, + self.takeloan_withdraw, vault_id, "1.00000000@DUSD", 'withdraw') + + self.rollback_to(block_height) + self.rollback_checks([vault_id]) + def run_test(self): # Initial set up self.setup() @@ -520,6 +606,9 @@ def run_test(self): self.update_oracle_price() self.post_FCE_DFI_minimum_check_takeloan() + self.update_oracle_price() + self.check_looped_dusd() + # self.post_FCE_DFI_minimum_check_withdraw() # TODO