Merge develop to v1.9.x (#281)

* Add Account Revision (#276) * NDEV-2651 Remove syscall_stabs (#279) * NDEV-2620: Add executor_state: &impl Database parameter to EventListener::event method (#273) * add keys for Miro, DeBank/ Rabby and Alfie for devnet WL --------- Co-authored-by: Semen Medvedev <[email protected]> Co-authored-by: Miroslav Nedelchev <[email protected]> Co-authored-by: Andrei Silviu Dragnea <[email protected]> Co-authored-by: s-medvedev <[email protected]> Co-authored-by: Kristina Nikolaeva <[email protected]>
neonlabsorg · Feb 26, 2024 · 4cbebed · 4cbebed
1 parent 1efefb6
commit 4cbebed
Show file tree

Hide file tree

Showing 87 changed files with 1,240 additions and 1,097 deletions.
diff --git a/.github/workflows/deploy.py b/.github/workflows/deploy.py
@@ -105,16 +105,19 @@ def run_subprocess(command):
 @cli.command(name="run_tests")
 @click.option('--github_sha')
 @click.option('--neon_test_branch')
-def run_tests(github_sha, neon_test_branch):
+@click.option('--base_ref_branch')
+def run_tests(github_sha, neon_test_branch, base_ref_branch):
     os.environ["EVM_LOADER_IMAGE"] = f"{IMAGE_NAME}:{github_sha}"
 
-    if neon_test_branch in GithubClient.get_branches_list(NEON_TESTS_ENDPOINT) \
+    if GithubClient.is_branch_exist(NEON_TESTS_ENDPOINT, neon_test_branch) \
             and neon_test_branch not in ('master', 'develop'):
         neon_test_image_tag = neon_test_branch
+    elif re.match(VERSION_BRANCH_TEMPLATE, base_ref_branch): # PR to version branch
+        neon_test_image_tag = base_ref_branch
     else:
         neon_test_image_tag = 'latest'
     os.environ["NEON_TESTS_IMAGE"] = f"{NEON_TEST_IMAGE_NAME}:{neon_test_image_tag}"
-
+    click.echo(f"NEON_TESTS_IMAGE: {os.environ['NEON_TESTS_IMAGE']}")
     project_name = f"neon-evm-{github_sha}"
     stop_containers(project_name)
 

diff --git a/.github/workflows/github_api_client.py b/.github/workflows/github_api_client.py
@@ -42,6 +42,15 @@ def get_branches_list(endpoint):
         proxy_branches_obj = requests.get(
             f"{endpoint}/branches?per_page=100").json()
         return [item["name"] for item in proxy_branches_obj]
+    @staticmethod
+    def is_branch_exist(endpoint, branch):
+        if branch:
+            response = requests.get(f"{endpoint}/branches/{branch}")
+            if response.status_code == 200:
+                click.echo(f"The branch {branch} exist in the {endpoint} repository")
+                return True
+        else:
+            return False
 
     def get_proxy_run_info(self, id):
         response = requests.get(

diff --git a/.github/workflows/pipeline.yml b/.github/workflows/pipeline.yml
@@ -64,7 +64,7 @@ jobs:
           elif [[ "${{ steps.tag_creation.outputs.base_branch }}" != "" ]]; then # tag creation
             tag=${{ steps.tag_creation.outputs.base_branch }}
 
-          elif [[ "${{ github.head_ref }}" != "" ]]; then # pr to feature or version branch
+          elif [[ "${{ github.head_ref }}" != "" ]]; then # pr to feature or version branch or develop
             tag=${{ github.head_ref }}
           else
             tag='develop'
@@ -75,7 +75,8 @@ jobs:
         run: |
           python3 ./.github/workflows/deploy.py run_tests \
           --github_sha=${GITHUB_SHA} \
-          --neon_test_branch=${{ steps.neon_test_branch.outputs.value }}
+          --neon_test_branch=${{ steps.neon_test_branch.outputs.value }} \
+          --base_ref_branch=${{ github.base_ref }}
   trigger-proxy-tests:
     runs-on: trigger-runner
     needs:

diff --git a/Dockerfile b/Dockerfile
@@ -29,7 +29,7 @@ RUN cargo fmt --check && \
 
 
 # Add neon_test_invoke_program to the genesis
-FROM neonlabsorg/neon_test_invoke_program:develop AS neon_test_invoke_program
+FROM neonlabsorg/neon_test_programs:latest AS neon_test_programs
 
 # Define solana-image that contains utility
 FROM builder AS base
@@ -40,8 +40,8 @@ COPY --from=evm-loader-builder /opt/neon-evm/evm_loader/target/deploy/evm_loader
 COPY --from=evm-loader-builder /opt/neon-evm/evm_loader/target/deploy/evm_loader-dump.txt /opt/
 COPY --from=evm-loader-builder /opt/neon-evm/evm_loader/target/release/neon-cli /opt/
 COPY --from=evm-loader-builder /opt/neon-evm/evm_loader/target/release/neon-api /opt/
-COPY --from=neon_test_invoke_program /opt/neon_test_invoke_program.so /opt/
-COPY --from=neon_test_invoke_program /opt/neon_test_invoke_program-keypair.json /opt/
+
+COPY --from=neon_test_programs /opt/deploy/ /opt/deploy/
 
 COPY ci/wait-for-solana.sh \
     ci/wait-for-neon.sh \

diff --git a/ci/solana-run-neon.sh b/ci/solana-run-neon.sh
@@ -12,9 +12,6 @@ EVM_LOADER_PATH=${NEON_BIN}/evm_loader.so
 METAPLEX=metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s
 METAPLEX_PATH=${NEON_BIN}/metaplex.so
 
-TEST_INVOKE_PROGRAM_ID_KEYPAIR=${NEON_BIN}/neon_test_invoke_program-keypair.json
-TEST_INVOKE=$(solana address -k ${TEST_INVOKE_PROGRAM_ID_KEYPAIR})
-TEST_INVOKE_PATH=${NEON_BIN}/neon_test_invoke_program.so
 
 VALIDATOR_ARGS=(
   --reset
@@ -23,9 +20,16 @@ VALIDATOR_ARGS=(
   --ticks-per-slot 16
   --upgradeable-program ${EVM_LOADER} ${EVM_LOADER_PATH} ${EVM_LOADER_AUTHORITY_KEYPAIR}
   --bpf-program ${METAPLEX} ${METAPLEX_PATH}
-  --bpf-program ${TEST_INVOKE} ${TEST_INVOKE_PATH}
 )
 
+LIST_OF_TEST_PROGRAMS=("test_invoke_program" "counter" "cross_program_invocation" "transfer_sol" "transfer_tokens")
+
+for program in "${LIST_OF_TEST_PROGRAMS[@]}"; do
+  keypair="${NEON_BIN}/deploy/${program}/${program}-keypair.json"
+  address=$(solana address -k $keypair)
+  VALIDATOR_ARGS+=(--bpf-program $address ${NEON_BIN}/deploy/$program/$program.so)
+done
+
 if [[ -n $GEYSER_PLUGIN_CONFIG ]]; then
   echo "Using geyser plugin with config: $GEYSER_PLUGIN_CONFIG"
   VALIDATOR_ARGS+=(--geyser-plugin-config $GEYSER_PLUGIN_CONFIG)

diff --git a/evm_loader/api/src/api_server/handlers/emulate.rs b/evm_loader/api/src/api_server/handlers/emulate.rs
@@ -1,5 +1,6 @@
 use actix_request_identifier::RequestId;
 use actix_web::{http::StatusCode, post, web::Json, Responder};
+use neon_lib::tracing::tracers::TracerTypeEnum;
 use std::convert::Into;
 use tracing::info;
 
@@ -26,8 +27,14 @@ pub async fn emulate(
     };
 
     process_result(
-        &EmulateCommand::execute(&rpc, state.config.evm_loader, emulate_request.body, None)
-            .await
-            .map_err(Into::into),
+        &EmulateCommand::execute(
+            &rpc,
+            state.config.evm_loader,
+            emulate_request.body,
+            None::<TracerTypeEnum>,
+        )
+        .await
+        .map(|(response, _)| response)
+        .map_err(Into::into),
     )
 }
diff --git a/evm_loader/cli/src/main.rs b/evm_loader/cli/src/main.rs
@@ -30,6 +30,7 @@ use crate::build_info::get_build_info;
 use evm_loader::types::Address;
 use neon_lib::errors::NeonError;
 use neon_lib::rpc::{CallDbClient, RpcEnum};
+use neon_lib::tracing::tracers::TracerTypeEnum;
 use neon_lib::types::TracerDb;
 use solana_clap_utils::keypair::signer_from_path;
 use solana_sdk::signature::Signer;
@@ -45,9 +46,9 @@ async fn run(options: &ArgMatches<'_>) -> NeonCliResult {
             let rpc = build_rpc(options, config).await?;
 
             let request = read_tx_from_stdin()?;
-            emulate::execute(&rpc, config.evm_loader, request, None)
+            emulate::execute(&rpc, config.evm_loader, request, None::<TracerTypeEnum>)
                 .await
-                .map(|result| json!(result))
+                .map(|(result, _)| json!(result))
         }
         ("trace", Some(_)) => {
             let rpc = build_rpc(options, config).await?;
@@ -90,12 +91,12 @@ async fn run(options: &ArgMatches<'_>) -> NeonCliResult {
                 .map(|result| json!(result))
         }
         ("cancel-trx", Some(params)) => {
-            let rpc_client = config.build_solana_rpc_client();
+            let rpc_client = config.build_clone_solana_rpc_client();
             let signer = build_signer(config)?;
 
             let storage_account =
                 pubkey_of(params, "storage_account").expect("storage_account parse error");
-            cancel_trx::execute(&rpc_client, &*signer, config.evm_loader, &storage_account)
+            cancel_trx::execute(rpc_client, &*signer, config.evm_loader, &storage_account)
                 .await
                 .map(|result| json!(result))
         }

diff --git a/evm_loader/lib/Cargo.toml b/evm_loader/lib/Cargo.toml
@@ -9,7 +9,7 @@ edition = "2021"
 thiserror = "1.0"
 anyhow = "1.0"
 bincode = "1.3.1"
-evm-loader = { path = "../program", default-features = false, features = ["log", "async-trait", "serde_json"] }
+evm-loader = { path = "../program", default-features = false, features = ["log", "async-trait"] }
 solana-sdk = "=1.16.23"
 solana-client = "=1.16.23"
 solana-clap-utils = "=1.16.23"

diff --git a/evm_loader/lib/src/account_storage.rs b/evm_loader/lib/src/account_storage.rs
@@ -1,18 +1,21 @@
 use async_trait::async_trait;
 use evm_loader::account::legacy::{
-    LegacyEtherData, LegacyStorageData, TAG_ACCOUNT_CONTRACT_DEPRECATED,
-    TAG_STORAGE_CELL_DEPRECATED,
+    LegacyEtherData, LegacyStorageData, ACCOUNT_PREFIX_LEN_BEFORE_REVISION,
+    TAG_ACCOUNT_BALANCE_BEFORE_REVISION, TAG_ACCOUNT_CONTRACT_BEFORE_REVISION,
+    TAG_ACCOUNT_CONTRACT_DEPRECATED, TAG_STORAGE_CELL_BEFORE_REVISION, TAG_STORAGE_CELL_DEPRECATED,
 };
-use evm_loader::account::{TAG_ACCOUNT_CONTRACT, TAG_STORAGE_CELL};
+use evm_loader::account::{ACCOUNT_PREFIX_LEN, TAG_ACCOUNT_CONTRACT, TAG_STORAGE_CELL};
 use evm_loader::account_storage::find_slot_hash;
 use evm_loader::types::Address;
 use solana_sdk::rent::Rent;
 use solana_sdk::system_program;
-use solana_sdk::sysvar::{slot_hashes, Sysvar};
-use std::collections::{BTreeMap, HashSet};
-use std::{cell::RefCell, collections::HashMap, convert::TryInto, rc::Rc};
+use solana_sdk::sysvar::slot_hashes;
+use std::collections::{BTreeMap, HashMap, HashSet};
+use std::{cell::RefCell, convert::TryInto, rc::Rc};
 
+use crate::account_update::update_account;
 use crate::solana_emulator::get_solana_emulator;
+use crate::NeonResult;
 use crate::{rpc::Rpc, NeonError};
 use ethnum::U256;
 use evm_loader::{
@@ -58,6 +61,7 @@ pub struct EmulatorAccountStorage<'rpc, T: Rpc> {
     chains: Vec<ChainInfo>,
     block_number: u64,
     block_timestamp: i64,
+    rent: Rent,
     state_overrides: Option<AccountOverrides>,
 }
 
@@ -86,6 +90,14 @@ impl<'rpc, T: Rpc + BuildConfigSimulator> EmulatorAccountStorage<'rpc, T> {
             Some(chains) => chains,
         };
 
+        let rent_account = rpc
+            .get_account(&solana_sdk::sysvar::rent::id())
+            .await?
+            .value
+            .ok_or(NeonError::AccountNotFound(solana_sdk::sysvar::rent::id()))?;
+        let rent = bincode::deserialize::<Rent>(&rent_account.data)?;
+        info!("Rent: {rent:?}");
+
         Ok(Self {
             accounts: RefCell::new(HashMap::new()),
             program_id,
@@ -95,6 +107,7 @@ impl<'rpc, T: Rpc + BuildConfigSimulator> EmulatorAccountStorage<'rpc, T> {
             block_number,
             block_timestamp,
             state_overrides,
+            rent,
         })
     }
 
@@ -169,9 +182,13 @@ impl<T: Rpc> EmulatorAccountStorage<'_, T> {
         address: Address,
         chain_id: u64,
         is_writable: bool,
-    ) -> client_error::Result<(Pubkey, Option<Account>, Option<Account>)> {
+    ) -> NeonResult<(Pubkey, Option<Account>, Option<Account>)> {
         let (pubkey, _) = address.find_balance_address(self.program_id(), chain_id);
-        let account = self.use_account(pubkey, is_writable).await?;
+        let mut account = self.use_account(pubkey, is_writable).await?;
+
+        if let Some(account) = &mut account {
+            update_account(&self.program_id, &pubkey, account)?;
+        }
 
         let legacy_account = if account.is_none() && (chain_id == self.default_chain_id()) {
             let (legacy_pubkey, _) = address.find_solana_address(self.program_id());
@@ -187,9 +204,13 @@ impl<T: Rpc> EmulatorAccountStorage<'_, T> {
         &self,
         address: Address,
         is_writable: bool,
-    ) -> client_error::Result<(Pubkey, Option<Account>)> {
+    ) -> NeonResult<(Pubkey, Option<Account>)> {
         let (pubkey, _) = address.find_solana_address(self.program_id());
-        let account = self.use_account(pubkey, is_writable).await?;
+        let mut account = self.use_account(pubkey, is_writable).await?;
+
+        if let Some(account) = &mut account {
+            update_account(&self.program_id, &pubkey, account)?;
+        }
 
         Ok((pubkey, account))
     }
@@ -199,22 +220,24 @@ impl<T: Rpc> EmulatorAccountStorage<'_, T> {
         address: Address,
         index: U256,
         is_writable: bool,
-    ) -> client_error::Result<(Pubkey, Option<Account>)> {
+    ) -> NeonResult<(Pubkey, Option<Account>)> {
         let (base, _) = address.find_solana_address(self.program_id());
         let cell_address = StorageCellAddress::new(self.program_id(), &base, &index);
 
-        let account = self
+        let mut account = self
             .use_account(*cell_address.pubkey(), is_writable)
             .await?;
 
+        if let Some(account) = &mut account {
+            update_account(&self.program_id, cell_address.pubkey(), account)?;
+        }
+
         Ok((*cell_address.pubkey(), account))
     }
 
     pub async fn apply_actions(&mut self, actions: Vec<Action>) -> Result<(), NeonError> {
         info!("apply_actions");
 
-        let rent = Rent::get()?;
-
         let mut new_balance_accounts = HashSet::new();
 
         for action in actions {
@@ -262,12 +285,13 @@ impl<T: Rpc> EmulatorAccountStorage<'_, T> {
                         let empty_size = StorageCell::required_account_size(0);
 
                         let gas = if account.is_none() {
-                            rent.minimum_balance(cell_size)
+                            self.rent.minimum_balance(cell_size)
                         } else {
                             let existing_value = self.storage(address, index).await;
                             if existing_value == [0_u8; 32] {
-                                rent.minimum_balance(cell_size)
-                                    .saturating_sub(rent.minimum_balance(empty_size))
+                                self.rent
+                                    .minimum_balance(cell_size)
+                                    .saturating_sub(self.rent.minimum_balance(empty_size))
                             } else {
                                 0
                             }
@@ -294,7 +318,7 @@ impl<T: Rpc> EmulatorAccountStorage<'_, T> {
                     self.use_contract_account(address, true).await?;
 
                     let space = ContractAccount::required_account_size(&code);
-                    self.gas = self.gas.saturating_add(rent.minimum_balance(space));
+                    self.gas = self.gas.saturating_add(self.rent.minimum_balance(space));
                 }
                 Action::EvmSelfDestruct { address } => {
                     info!("selfdestruct {address}");
@@ -320,7 +344,8 @@ impl<T: Rpc> EmulatorAccountStorage<'_, T> {
         }
 
         self.gas = self.gas.saturating_add(
-            rent.minimum_balance(BalanceAccount::required_account_size())
+            self.rent
+                .minimum_balance(BalanceAccount::required_account_size())
                 .saturating_mul(new_balance_accounts.len() as u64),
         );
 
@@ -331,8 +356,6 @@ impl<T: Rpc> EmulatorAccountStorage<'_, T> {
         let mut accounts = self.accounts.borrow_mut();
         let mut additional_balances = Vec::new();
 
-        let rent = Rent::get()?;
-
         for (key, account) in accounts.iter_mut() {
             let Some(account_data) = account.data.as_mut() else {
                 continue;
@@ -361,10 +384,33 @@ impl<T: Rpc> EmulatorAccountStorage<'_, T> {
 
                 if (legacy_data.code_size > 0) || (legacy_data.generation > 0) {
                     // This is a contract, we need additional gas for conversion
-                    let lamports = rent.minimum_balance(BalanceAccount::required_account_size());
+                    let lamports = self
+                        .rent
+                        .minimum_balance(BalanceAccount::required_account_size());
                     self.gas = self.gas.saturating_add(lamports);
                 }
             }
+
+            if matches!(
+                tag,
+                TAG_ACCOUNT_BALANCE_BEFORE_REVISION
+                    | TAG_ACCOUNT_CONTRACT_BEFORE_REVISION
+                    | TAG_STORAGE_CELL_BEFORE_REVISION
+            ) {
+                const PREFIX_BEFORE: usize = ACCOUNT_PREFIX_LEN_BEFORE_REVISION;
+                const PREFIX_AFTER: usize = ACCOUNT_PREFIX_LEN;
+                const EXPANSION_LEN: usize = PREFIX_AFTER - PREFIX_BEFORE;
+
+                account.is_writable = true;
+                account.is_legacy = true;
+
+                let lamports = self
+                    .rent
+                    .minimum_balance(EXPANSION_LEN)
+                    .saturating_sub(self.rent.minimum_balance(0));
+
+                self.gas = self.gas.saturating_add(lamports);
+            }
         }
 
         for a in additional_balances {
@@ -533,6 +579,16 @@ impl<T: Rpc> AccountStorage for EmulatorAccountStorage<'_, T> {
         self.block_timestamp.try_into().unwrap()
     }
 
+    fn rent(&self) -> &Rent {
+        &self.rent
+    }
+
+    fn return_data(&self) -> Option<(Pubkey, Vec<u8>)> {
+        info!("return_data");
+        // TODO: implement return_data() method with SyncedAccountStorage implementation
+        unimplemented!();
+    }
+
     async fn block_hash(&self, slot: u64) -> [u8; 32] {
         info!("block_hash {slot}");
 
@@ -742,7 +798,6 @@ impl<T: Rpc> AccountStorage for EmulatorAccountStorage<'_, T> {
     ) -> evm_loader::error::Result<()> {
         let instruction = Instruction::new_with_bytes(*program_id, instruction_data, meta.to_vec());
         let solana_emulator = get_solana_emulator().await;
-        //let solana_emulator = self.solana_emulator.lock().expect("Lock solana_emulator");
         solana_emulator
             .emulate_solana_call(self, &instruction, accounts, seeds)
             .await