fix(mocknet): Only have up to 4 rpc nodes on mocknet (#2973)

* Originally mocknet was 50% non-validating nodes (this configuration was inherited from devnet)
* We want mostly validating nodes since non-validating nodes are less interesting for testing (contribute less to failure modes, e.g. block production)
* We need at least some non-validating nodes to test code paths when the node is not a block producer
* We settled on 4 as a reasonable number.
* The number of nodes in the mocknet needed to be increased to 54 as a result because the current validator "seat" model did not allow 46 genesis validators, but does allow 50. Note this problem will no longer exist when near/NEPs#83 is implemented.
* The timeouts on transactions are also increased to be less sensitive to missing blocks due to offline validators (this is important for the outage test in particular, where we want a test failure to really mean that the network stalled, not just that several blocks in a row were skipped since that is an expected outcome from losing 25% of validators)

pytest/lib/mocknet.py

@@ -9,7 +9,7 @@
 from rc import run, pmap
 
 NUM_SECONDS_PER_YEAR = 3600 * 24 * 365
-NUM_NODES = 50
+NUM_NODES = 54
 NODE_BASE_NAME = 'mocknet-node'
 NODE_USERNAME = 'ubuntu'
 NODE_SSH_KEY_PATH = '~/.ssh/near_ops'
@@ -133,7 +133,7 @@ def get_metrics(node):
 
 # Sends the transaction to the network via `node` and checks for success.
 # Some retrying is done when the node returns a Timeout error.
-def send_transaction(node, tx, tx_hash, account_id, timeout=60):
+def send_transaction(node, tx, tx_hash, account_id, timeout=120):
     response = node.send_tx_and_wait(tx, timeout)
     loop_start = time.time()
     missing_count = 0
@@ -143,15 +143,15 @@ def send_transaction(node, tx, tx_hash, account_id, timeout=60):
             print(
                 f'WARN: transaction {tx_hash} returned Timout, checking status again.'
             )
-            time.sleep(2)
+            time.sleep(5)
             response = node.get_tx(tx_hash, account_id)
         elif "doesn't exist" in error_data:
             missing_count += 1
             print(
                 f'WARN: transaction {tx_hash} falied to be recieved by the node, checking again.'
             )
             if missing_count < 20:
-                time.sleep(2)
+                time.sleep(5)
                 response = node.get_tx(tx_hash, account_id)
             else:
                 print(f'WARN: re-sending transaction {tx_hash}.')

pytest/tests/mocknet/outage.py

@@ -14,7 +14,8 @@
 # Get the list of current validators, sorted by node index
 validators = sorted(nodes[0].get_validators()['result']['current_validators'],
                     key=lambda v: int(v['account_id'][4:]))
-assert len(validators) == (mocknet.NUM_NODES // 2)
+num_rpc_nodes = min(mocknet.NUM_NODES // 2, 4)
+assert len(validators) == mocknet.NUM_NODES - num_rpc_nodes
 
 # Take down ~25% of stake.
 total_stake = sum(map(lambda v: int(v['stake']), validators))