Fabo/298 tendermint reconnect (#300)

* reconnect on tendermint disconnect

* cleanup

* comments

* Update cosmos-node-subscription.js

lib/block-listeners/cosmos-node-subscription.js

@@ -22,10 +22,16 @@ class CosmosNodeSubscription {
       name: `${this.network.id}_update`
     })
 
+    this.connectTendermint(this.network)
+  }
+
+  async connectTendermint(network) {
+    console.log('Connecting to Tendermint on', network.rpc_url)
     // Create a RPC subscription for each network that will react to new block events.
     Tendermint()
-      .connect(this.network.rpc_url)
+      .connect(network.rpc_url)
       .then(connectedClient => {
+        console.log('Connected to Tendermint on', network.rpc_url)
         connectedClient.subscribe({ query: "tm.event='NewBlock'" }, event => {
           if (this.lastupdate) {
             const diff = Date.now() - this.lastupdate
@@ -37,17 +43,29 @@ class CosmosNodeSubscription {
             WAIT_FOR_BLOCK_DELAY
           )
         })
+
+        // on connection lost, reconnect to tendermint + Sentry error
+        connectedClient.ondisconnect = () => {
+          console.log('Lost connection to Tendermint for', network.rpc_url)
+
+          Sentry.withScope(function(scope) {
+            scope.setExtra('network', network.id)
+            scope.setExtra('rpc_url', network.rpc_url)
+            Sentry.captureException(new Error(`Lost Tendermint connection`))
+          })
+          setTimeout(() => this.connectTendermint(network), 3000)
+        }
       })
       .catch(e => {
         Sentry.withScope(function(scope) {
           scope.setExtra('network', network.id)
           scope.setExtra('rpc_url', network.rpc_url)
           Sentry.captureException(e)
         })
-      })
 
-    // disabled immediate validator query as, as the `latest` query is buggy
-    // this.newBlockHandler()
+        // if can't connect, retry
+        setTimeout(() => this.connectTendermint(network), 3000)
+      })
   }
 
   // For each block event, we fetch the block information and publish a message.