Cosmos grpc fallbackprovider (hyperlane-xyz#3139)

### Description

Implements grpc fallback provider logic for cosmos

- initially tried implementing the fallback provider deprioritization
logic at middleware level like in the EVM. The difference between ethers
and cosmrs is that in the latter, middleware can only live at the
transport layer (`tower` crate level).
- based on this github
[issue](hyperium/tonic#733), that actually
doesn't look possible, because the http::Request type isn't `Clone` so
it can't be submitted to multiple providers
- ended up implementing the fallback provider at the application layer,
by keeping an array of grpc channels
- There is now a `call` method in `hyperlane_core::FallbackProvider`
which I'm actually really happy with. This method handles the
fallbackprovider-specific logic by taking in an async closure, running
it on each provider, and iterating providers if the closure call fails.
In `grpc.rs` you can see how this is slightly verbose but I think it's
quite manageable. The only part that bugs me is having to duplicate
`Pin::from(Box::from(future))`, but that's need afaict because the
regular closure returns an anonymous type
- adds `grpcUrls` and `customGrpcUrls` config items
- tests the cosmos fallback provider e2e

### Drive-by changes

<!--
Are there any minor or drive-by changes also included?
-->

### Related issues

- Fixes: hyperlane-xyz/issues#998

### Backward compatibility

<!--
Are these changes backward compatible? Are there any infrastructure
implications, e.g. changes that would prohibit deploying older commits
using this infra tooling?

Yes/No
-->

### Testing

<!--
What kind of testing have these changes undergone?

None/Manual/Unit Tests
-->

rust/Cargo.toml

@@ -97,7 +97,7 @@ hyper = "0.14"
 hyper-tls = "0.5.0"
 injective-protobuf = "0.2.2"
 injective-std = "0.1.5"
-itertools = "0.11.0"
+itertools = "*"
 jobserver = "=0.1.26"
 jsonrpc-core = "18.0"
 k256 = { version = "0.13.1", features = ["std", "ecdsa"] }

rust/agents/relayer/Cargo.toml

@@ -34,7 +34,7 @@ tokio = { workspace = true, features = ["rt", "macros", "parking_lot"] }
 tracing-futures.workspace = true
 tracing.workspace = true
 
-hyperlane-core = { path = "../../hyperlane-core", features = ["agent"] }
+hyperlane-core = { path = "../../hyperlane-core", features = ["agent", "fallback-provider"] }
 hyperlane-base = { path = "../../hyperlane-base" }
 hyperlane-ethereum = { path = "../../chains/hyperlane-ethereum" }
 

rust/agents/validator/Cargo.toml

@@ -24,7 +24,7 @@ tokio = { workspace = true, features = ["rt", "macros", "parking_lot"] }
 tracing-futures.workspace = true
 tracing.workspace = true
 
-hyperlane-core = { path = "../../hyperlane-core", features = ["agent"] }
+hyperlane-core = { path = "../../hyperlane-core", features = ["agent", "fallback-provider"] }
 hyperlane-base = { path = "../../hyperlane-base" }
 hyperlane-ethereum = { path = "../../chains/hyperlane-ethereum" }
 hyperlane-cosmos = { path = "../../chains/hyperlane-cosmos" }

rust/chains/hyperlane-cosmos/Cargo.toml

@@ -22,6 +22,7 @@ hyper = { workspace = true }
 hyper-tls = { workspace = true }
 injective-protobuf = { workspace = true }
 injective-std = { workspace = true }
+itertools = { workspace = true }
 once_cell = { workspace = true }
 protobuf = { workspace = true }
 ripemd = { workspace = true }
@@ -38,4 +39,4 @@ tracing = { workspace = true }
 tracing-futures = { workspace = true }
 url = { workspace = true }
 
-hyperlane-core = { path = "../../hyperlane-core" }
+hyperlane-core = { path = "../../hyperlane-core", features = ["fallback-provider"]}

rust/chains/hyperlane-cosmos/src/error.rs

@@ -1,5 +1,6 @@
 use cosmrs::proto::prost;
 use hyperlane_core::ChainCommunicationError;
+use std::fmt::Debug;
 
 /// Errors from the crates specific to the hyperlane-cosmos
 /// implementation.
@@ -28,6 +29,9 @@ pub enum HyperlaneCosmosError {
     /// Tonic error
     #[error("{0}")]
     Tonic(#[from] tonic::transport::Error),
+    /// Tonic codegen error
+    #[error("{0}")]
+    TonicGenError(#[from] tonic::codegen::StdError),
     /// Tendermint RPC Error
     #[error(transparent)]
     TendermintError(#[from] tendermint_rpc::error::Error),
@@ -37,6 +41,9 @@ pub enum HyperlaneCosmosError {
     /// Protobuf error
     #[error("{0}")]
     Protobuf(#[from] protobuf::ProtobufError),
+    /// Fallback providers failed
+    #[error("Fallback providers failed. (Errors: {0:?})")]
+    FallbackProvidersFailed(Vec<HyperlaneCosmosError>),
 }
 
 impl From<HyperlaneCosmosError> for ChainCommunicationError {

rust/chains/hyperlane-cosmos/src/lib.rs

@@ -16,6 +16,7 @@ mod multisig_ism;
 mod payloads;
 mod providers;
 mod routing_ism;
+mod rpc_clients;
 mod signers;
 mod trait_builder;
 mod types;

rust/chains/hyperlane-cosmos/src/payloads/aggregate_ism.rs

@@ -1,11 +1,11 @@
 use serde::{Deserialize, Serialize};
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct VerifyRequest {
     pub verify: VerifyRequestInner,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct VerifyRequestInner {
     pub metadata: String,
     pub message: String,

rust/chains/hyperlane-cosmos/src/payloads/general.rs

@@ -1,6 +1,6 @@
 use serde::{Deserialize, Serialize};
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct EmptyStruct {}
 
 #[derive(Serialize, Deserialize, Debug, Clone)]

rust/chains/hyperlane-cosmos/src/payloads/ism_routes.rs

@@ -1,12 +1,12 @@
 use super::general::EmptyStruct;
 use serde::{Deserialize, Serialize};
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct IsmRouteRequest {
     pub route: IsmRouteRequestInner,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct IsmRouteRequestInner {
     pub message: String, // hexbinary
 }
@@ -16,22 +16,22 @@ pub struct IsmRouteRespnose {
     pub ism: String,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct QueryRoutingIsmGeneralRequest<T> {
     pub routing_ism: T,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct QueryRoutingIsmRouteResponse {
     pub ism: String,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct QueryIsmGeneralRequest<T> {
     pub ism: T,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct QueryIsmModuleTypeRequest {
     pub module_type: EmptyStruct,
 }

rust/chains/hyperlane-cosmos/src/payloads/mailbox.rs

@@ -3,52 +3,52 @@ use serde::{Deserialize, Serialize};
 use super::general::EmptyStruct;
 
 // Requests
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct GeneralMailboxQuery<T> {
     pub mailbox: T,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct CountRequest {
     pub count: EmptyStruct,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct NonceRequest {
     pub nonce: EmptyStruct,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct RecipientIsmRequest {
     pub recipient_ism: RecipientIsmRequestInner,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct RecipientIsmRequestInner {
     pub recipient_addr: String, // hexbinary
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct DefaultIsmRequest {
     pub default_ism: EmptyStruct,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct DeliveredRequest {
     pub message_delivered: DeliveredRequestInner,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct DeliveredRequestInner {
     pub id: String, // hexbinary
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ProcessMessageRequest {
     pub process: ProcessMessageRequestInner,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct ProcessMessageRequestInner {
     pub metadata: String,
     pub message: String,

rust/chains/hyperlane-cosmos/src/payloads/merkle_tree_hook.rs

@@ -4,24 +4,24 @@ use super::general::EmptyStruct;
 
 const TREE_DEPTH: usize = 32;
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct MerkleTreeGenericRequest<T> {
     pub merkle_hook: T,
 }
 
 // --------- Requests ---------
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct MerkleTreeRequest {
     pub tree: EmptyStruct,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct MerkleTreeCountRequest {
     pub count: EmptyStruct,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct CheckPointRequest {
     pub check_point: EmptyStruct,
 }

rust/chains/hyperlane-cosmos/src/payloads/multisig_ism.rs

@@ -1,11 +1,11 @@
 use serde::{Deserialize, Serialize};
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct VerifyInfoRequest {
     pub verify_info: VerifyInfoRequestInner,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct VerifyInfoRequestInner {
     pub message: String, // hexbinary
 }

rust/chains/hyperlane-cosmos/src/payloads/validator_announce.rs

@@ -2,17 +2,17 @@ use serde::{Deserialize, Serialize};
 
 use super::general::EmptyStruct;
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct GetAnnouncedValidatorsRequest {
     pub get_announced_validators: EmptyStruct,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct GetAnnounceStorageLocationsRequest {
     pub get_announce_storage_locations: GetAnnounceStorageLocationsRequestInner,
 }
 
-#[derive(Serialize, Deserialize, Debug)]
+#[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct GetAnnounceStorageLocationsRequestInner {
     pub validators: Vec<String>,
 }