Merge branch 'contract-dist-deploy2' of https://github.com/tayfunelma…

…s/nearcore into contract-dist-deploy2

chain/chain/Cargo.toml

@@ -58,6 +58,7 @@ near-schema-checker-lib.workspace = true
 
 [dev-dependencies]
 near-primitives = { workspace = true, features = ["clock"] }
+near-test-contracts.workspace = true
 serde_json.workspace = true
 primitive-types.workspace = true
 insta.workspace = true

chain/chain/src/runtime/mod.rs

@@ -476,49 +476,6 @@ impl NightshadeRuntime {
         Ok(result)
     }
 
-    fn precompile_contracts(
-        &self,
-        epoch_id: &EpochId,
-        contract_codes: Vec<ContractCode>,
-    ) -> Result<(), Error> {
-        let _span = tracing::debug_span!(
-            target: "runtime",
-            "precompile_contracts",
-            num_contracts = contract_codes.len())
-        .entered();
-        let protocol_version = self.epoch_manager.get_epoch_protocol_version(epoch_id)?;
-        let runtime_config = self.runtime_config_store.get_config(protocol_version);
-        let compiled_contract_cache: Option<Box<dyn ContractRuntimeCache>> =
-            Some(Box::new(self.compiled_contract_cache.handle()));
-        // Execute precompile_contract in parallel but prevent it from using more than half of all
-        // threads so that node will still function normally.
-        rayon::scope(|scope| {
-            let (slot_sender, slot_receiver) = std::sync::mpsc::channel();
-            // Use up-to half of the threads for the compilation.
-            let max_threads = std::cmp::max(rayon::current_num_threads() / 2, 1);
-            for _ in 0..max_threads {
-                slot_sender.send(()).expect("both sender and receiver are owned here");
-            }
-            for code in contract_codes {
-                slot_receiver.recv().expect("could not receive a slot to compile contract");
-                let contract_cache = compiled_contract_cache.as_deref();
-                let slot_sender = slot_sender.clone();
-                scope.spawn(move |_| {
-                    precompile_contract(
-                        &code,
-                        Arc::clone(&runtime_config.wasm_config),
-                        contract_cache,
-                    )
-                    .ok();
-                    // If this fails, it just means there won't be any more attempts to recv the
-                    // slots
-                    let _ = slot_sender.send(());
-                });
-            }
-        });
-        Ok(())
-    }
-
     fn get_gc_stop_height_impl(&self, block_hash: &CryptoHash) -> Result<BlockHeight, Error> {
         let epoch_manager = self.epoch_manager.read();
         // an epoch must have a first block.
@@ -1349,6 +1306,49 @@ impl RuntimeAdapter for NightshadeRuntime {
     fn compiled_contract_cache(&self) -> &dyn ContractRuntimeCache {
         self.compiled_contract_cache.as_ref()
     }
+
+    fn precompile_contracts(
+        &self,
+        epoch_id: &EpochId,
+        contract_codes: Vec<ContractCode>,
+    ) -> Result<(), Error> {
+        let _span = tracing::debug_span!(
+            target: "runtime",
+            "precompile_contracts",
+            num_contracts = contract_codes.len())
+        .entered();
+        let protocol_version = self.epoch_manager.get_epoch_protocol_version(epoch_id)?;
+        let runtime_config = self.runtime_config_store.get_config(protocol_version);
+        let compiled_contract_cache: Option<Box<dyn ContractRuntimeCache>> =
+            Some(Box::new(self.compiled_contract_cache.handle()));
+        // Execute precompile_contract in parallel but prevent it from using more than half of all
+        // threads so that node will still function normally.
+        rayon::scope(|scope| {
+            let (slot_sender, slot_receiver) = std::sync::mpsc::channel();
+            // Use up-to half of the threads for the compilation.
+            let max_threads = std::cmp::max(rayon::current_num_threads() / 2, 1);
+            for _ in 0..max_threads {
+                slot_sender.send(()).expect("both sender and receiver are owned here");
+            }
+            for code in contract_codes {
+                slot_receiver.recv().expect("could not receive a slot to compile contract");
+                let contract_cache = compiled_contract_cache.as_deref();
+                let slot_sender = slot_sender.clone();
+                scope.spawn(move |_| {
+                    precompile_contract(
+                        &code,
+                        Arc::clone(&runtime_config.wasm_config),
+                        contract_cache,
+                    )
+                    .ok();
+                    // If this fails, it just means there won't be any more attempts to recv the
+                    // slots
+                    let _ = slot_sender.send(());
+                });
+            }
+        });
+        Ok(())
+    }
 }
 
 /// Get the limit on the number of new receipts imposed by the local congestion control.

chain/chain/src/runtime/tests.rs

@@ -2,6 +2,7 @@ use std::collections::BTreeSet;
 
 use crate::types::{ChainConfig, RuntimeStorageConfig};
 use crate::{Chain, ChainGenesis, ChainStoreAccess, DoomslugThresholdMode};
+use assert_matches::assert_matches;
 use near_chain_configs::test_utils::{TESTING_INIT_BALANCE, TESTING_INIT_STAKE};
 use near_epoch_manager::shard_tracker::ShardTracker;
 use near_epoch_manager::{EpochManager, RngSeed};
@@ -16,8 +17,10 @@ use near_primitives::epoch_block_info::BlockInfo;
 use near_primitives::receipt::{ActionReceipt, ReceiptV1};
 use near_primitives::test_utils::create_test_signer;
 use near_primitives::types::validator_stake::{ValidatorStake, ValidatorStakeIter};
+use near_primitives::version::PROTOCOL_VERSION;
 use near_store::flat::{FlatStateChanges, FlatStateDelta, FlatStateDeltaMetadata};
 use near_store::genesis::initialize_genesis_state;
+use near_vm_runner::{get_contract_cache_key, CompiledContract, CompiledContractInfo};
 use num_rational::Ratio;
 use rand::{rngs::StdRng, seq::SliceRandom, SeedableRng};
 
@@ -1828,6 +1831,76 @@ fn test_storage_proof_garbage() {
     assert_eq!(total_size / 1000_000, garbage_size_mb);
 }
 
+/// Tests that precompiling a set of contracts updates the compiled contract cache.
+#[test]
+fn test_precompile_contracts_updates_cache() {
+    struct FakeTestCompiledContractType; // For testing AnyCache.
+    let genesis = Genesis::test(vec!["test0".parse().unwrap()], 1);
+    let store = near_store::test_utils::create_test_store();
+    let tempdir = tempfile::tempdir().unwrap();
+    initialize_genesis_state(store.clone(), &genesis, Some(tempdir.path()));
+    let epoch_manager = EpochManager::new_arc_handle(store.clone(), &genesis.config, None);
+
+    let contract_cache = FilesystemContractRuntimeCache::new(tempdir.path(), None::<&str>)
+        .expect("filesystem contract cache");
+    let runtime = NightshadeRuntime::test_with_runtime_config_store(
+        tempdir.path(),
+        store,
+        contract_cache.handle(),
+        &genesis.config,
+        epoch_manager,
+        RuntimeConfigStore::new(None),
+        StateSnapshotType::EveryEpoch,
+    );
+
+    let contracts = vec![
+        ContractCode::new(near_test_contracts::sized_contract(100).to_vec(), None),
+        ContractCode::new(near_test_contracts::rs_contract().to_vec(), None),
+        ContractCode::new(near_test_contracts::trivial_contract().to_vec(), None),
+    ];
+    let code_hashes: Vec<CryptoHash> = contracts.iter().map(|c| c.hash()).cloned().collect();
+
+    // First check that the cache does not have the contracts.
+    for code_hash in code_hashes.iter() {
+        let cache_key = get_contract_cache_key(
+            *code_hash,
+            &runtime.get_runtime_config(PROTOCOL_VERSION).unwrap().wasm_config,
+        );
+        let contract = contract_cache.get(&cache_key).unwrap();
+        assert!(contract.is_none());
+    }
+
+    runtime.precompile_contracts(&EpochId::default(), contracts).unwrap();
+
+    // Check that the persistent cache contains the compiled contract after precompilation,
+    // but it does not populate the in-memory cache (so that the value is generated by try_lookup call).
+    for code_hash in code_hashes.into_iter() {
+        let cache_key = get_contract_cache_key(
+            code_hash,
+            &runtime.get_runtime_config(PROTOCOL_VERSION).unwrap().wasm_config,
+        );
+
+        let contract = contract_cache.get(&cache_key).unwrap();
+        assert_matches!(
+            contract,
+            Some(CompiledContractInfo { compiled: CompiledContract::Code(_), .. })
+        );
+
+        let result = contract_cache
+            .memory_cache()
+            .try_lookup(
+                cache_key,
+                || Ok::<_, ()>(Box::new(FakeTestCompiledContractType)),
+                |v| {
+                    assert!(v.is::<FakeTestCompiledContractType>());
+                    "compiled code"
+                },
+            )
+            .unwrap();
+        assert_eq!(result, "compiled code");
+    }
+}
+
 fn stake(
     nonce: Nonce,
     signer: &Signer,

chain/chain/src/test_utils/kv_runtime.rs

@@ -57,7 +57,7 @@ use near_store::{
     set_genesis_hash, set_genesis_state_roots, DBCol, ShardTries, Store, StoreUpdate, Trie,
     TrieChanges, WrappedTrieChanges,
 };
-use near_vm_runner::{ContractRuntimeCache, NoContractRuntimeCache};
+use near_vm_runner::{ContractCode, ContractRuntimeCache, NoContractRuntimeCache};
 use num_rational::Ratio;
 use rand::Rng;
 use std::cmp::Ordering;
@@ -1580,4 +1580,13 @@ impl RuntimeAdapter for KeyValueRuntime {
     fn compiled_contract_cache(&self) -> &dyn ContractRuntimeCache {
         &self.contract_cache
     }
+
+    fn precompile_contracts(
+        &self,
+        _epoch_id: &EpochId,
+        _contract_codes: Vec<ContractCode>,
+    ) -> Result<(), Error> {
+        // Note that KeyValueRuntime does not use compiled contract cache, so this is no-op.
+        Ok(())
+    }
 }

chain/chain/src/types.rs

@@ -40,6 +40,7 @@ use near_primitives::views::{QueryRequest, QueryResponse};
 use near_schema_checker_lib::ProtocolSchema;
 use near_store::flat::FlatStorageManager;
 use near_store::{PartialStorage, ShardTries, Store, Trie, WrappedTrieChanges};
+use near_vm_runner::ContractCode;
 use near_vm_runner::ContractRuntimeCache;
 use num_rational::Rational32;
 use tracing::instrument;
@@ -536,6 +537,13 @@ pub trait RuntimeAdapter: Send + Sync {
         -> Result<RuntimeConfig, Error>;
 
     fn compiled_contract_cache(&self) -> &dyn ContractRuntimeCache;
+
+    /// Precompiles the contracts and stores them in the compiled contract cache.
+    fn precompile_contracts(
+        &self,
+        epoch_id: &EpochId,
+        contract_codes: Vec<ContractCode>,
+    ) -> Result<(), Error>;
 }
 
 /// The last known / checked height and time when we have processed it.