fix(horizon_sync): try sync with next next peer if current one fails (t…

…ari-project#5699)

Description
---
* In case the horizon sync fails with a peer, try to sync with the next
peer on the list instead of aborting.

Motivation and Context
---
Up until now, most errors when trying to sync with an individual peer
would cause the whole horizon sync process to exit with error. We want
to try to sync with the next peer instead of aborting.

How Has This Been Tested?
---
Tests pass

What process can a PR reviewer use to test or verify this change?
---
Code review

Breaking Changes
---
- [x] None
- [ ] Requires data directory on base node to be deleted
- [ ] Requires hard fork
- [ ] Other - Please specify

base_layer/core/src/base_node/sync/horizon_state_sync/synchronizer.rs

@@ -186,19 +186,30 @@ impl<'a, B: BlockchainBackend + 'static> HorizonStateSynchronization<'a, B> {
         );
         for (i, sync_peer) in self.sync_peers.iter().enumerate() {
             self.hooks.call_on_starting_hook(sync_peer);
-            let mut connection = self.connectivity.dial_peer(sync_peer.node_id().clone()).await?;
+            let mut connection = match self.connectivity.dial_peer(sync_peer.node_id().clone()).await {
+                Ok(conn) => conn,
+                Err(err) => {
+                    warn!(target: LOG_TARGET, "Failed to connect to sync peer `{}`: {}", sync_peer.node_id(), err);
+                    continue;
+                },
+            };
             let config = RpcClient::builder()
                 .with_deadline(self.config.rpc_deadline)
                 .with_deadline_grace_period(Duration::from_secs(3));
-            let mut client = connection.connect_rpc_using_builder(config).await?;
+            let mut client = match connection.connect_rpc_using_builder(config).await {
+                Ok(rpc_client) => rpc_client,
+                Err(err) => {
+                    warn!(target: LOG_TARGET, "Failed to establish RPC coonection with sync peer `{}`: {}", sync_peer.node_id(), err);
+                    continue;
+                },
+            };
 
             match self.begin_sync(sync_peer.clone(), &mut client, header).await {
                 Ok(_) => match self.finalize_horizon_sync(sync_peer).await {
                     Ok(_) => return Ok(()),
                     Err(err) => {
                         self.ban_peer_on_bannable_error(sync_peer, &err).await?;
                         warn!(target: LOG_TARGET, "Error during sync:{}", err);
-                        return Err(err);
                     },
                 },
                 Err(err @ HorizonSyncError::RpcError(RpcError::ReplyTimeout)) |
@@ -212,7 +223,6 @@ impl<'a, B: BlockchainBackend + 'static> HorizonStateSynchronization<'a, B> {
                 Err(err) => {
                     self.ban_peer_on_bannable_error(sync_peer, &err).await?;
                     warn!(target: LOG_TARGET, "Error during sync:{}", err);
-                    return Err(err);
                 },
             }
         }