diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 2a59d032fb..69f04fca1c 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -1,6 +1,6 @@ name: AlertOnPR -on: +on: pull_request: types: [opened] jobs: @@ -8,10 +8,32 @@ jobs: name: Compile runs-on: ubuntu-latest steps: - - uses: rtCamp/action-slack-notify@master - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} - SLACK_CHANNEL: 'pr_alerts' - SLACK_USERNAME: 'luniebot' - SLACK_ICON: 'https://emoji.slack-edge.com/TH6F97TDF/lunie/4ac63c1d435c04dc.png' - SLACK_MESSAGE: ${{ github.event.pull_request.title }} | ${{ github.event.pull_request.body }} | ${{ github.event.pull_request._links.html.href }} + - name: curl message + run: | + curl -X POST ${{ secrets.SLACK_WEBHOOK }} \ + -H 'Content-Type: application/json; charset=utf-8' \ + --data-binary " + { + \"channel\": \"#pr_alerts\", + \"username\": \"pr_alert\", + \"icon_url\": \"https://emoji.slack-edge.com/TH6F97TDF/lunie/4ac63c1d435c04dc.png\", + \"attachments\": [ + { + \"fallback\": \"Required plain-text summary of the attachment.\", + \"color\": \"#36a64f\", + \"author_name\": \"${{github.event.pull_request.user.login}}\", + \"author_link\": \"${{github.event.pull_request.user.url}}\", + \"author_icon\": \"${{github.event.pull_request.user.avatar_url}}\", + \"title\": \"${{github.event.pull_request.title}}\", + \"title_link\": \"${{github.event.pull_request.html_url}}\", + \"text\": \"${{github.event.pull_request.body}}\", + \"fields\": [ + { + \"title\": \"Repo\", + \"value\": \"${{github.event.repository.full_name}}\", + \"short\": false + } + ] + } + ] + }" diff --git a/lib/block-listeners/cosmos-node-subscription.js b/lib/block-listeners/cosmos-node-subscription.js index be5233ba8b..ce1ea876ce 100644 --- a/lib/block-listeners/cosmos-node-subscription.js +++ b/lib/block-listeners/cosmos-node-subscription.js @@ -9,6 +9,7 @@ const { const Sentry = require('@sentry/node') const WAIT_FOR_BLOCK_DELAY = 5000 +const EXPECTED_MAX_BLOCK_WINDOW = 120000 // This class establishes an rpc connection to Tendermint. // Used for listening to events, such as new blocks. @@ -21,22 +22,54 @@ class CosmosNodeSubscription { this.metric = io.metric({ name: `${this.network.id}_update` }) + this.chainHangup = undefined + this.connectTendermint(this.network) + } + + async connectTendermint(network) { + console.log('Connecting to Tendermint on', network.rpc_url) // Create a RPC subscription for each network that will react to new block events. Tendermint() - .connect(this.network.rpc_url) + .connect(network.rpc_url) .then(connectedClient => { + console.log('Connected to Tendermint on', network.rpc_url) connectedClient.subscribe({ query: "tm.event='NewBlock'" }, event => { + // this tracks the block times + // issue: this will only trigger if there are actually blocks I guess if (this.lastupdate) { const diff = Date.now() - this.lastupdate this.metric.set(diff) } this.lastupdate = Date.now() + setTimeout( () => this.newBlockHandler(event.block.header.height), WAIT_FOR_BLOCK_DELAY ) + + // if there are no new blocks for some time, trigger an error + // TODO: show this error automatically in the UI + if (this.chainHangup) clearTimeout(this.chainHangup) + this.chainHangup = setTimeout(() => { + console.error(`Chain ${this.network.id} seems to have halted.`) + Sentry.captureException( + new Error(`Chain ${this.network.id} seems to have halted.`) + ) + }, EXPECTED_MAX_BLOCK_WINDOW) }) + + // on connection lost, reconnect to tendermint + Sentry error + connectedClient.ondisconnect = () => { + console.log('Lost connection to Tendermint for', network.rpc_url) + + Sentry.withScope(function(scope) { + scope.setExtra('network', network.id) + scope.setExtra('rpc_url', network.rpc_url) + Sentry.captureException(new Error(`Lost Tendermint connection`)) + }) + setTimeout(() => this.connectTendermint(network), 3000) + } }) .catch(e => { Sentry.withScope(function(scope) { @@ -44,10 +77,10 @@ class CosmosNodeSubscription { scope.setExtra('rpc_url', network.rpc_url) Sentry.captureException(e) }) - }) - // disabled immediate validator query as, as the `latest` query is buggy - // this.newBlockHandler() + // if can't connect, retry + setTimeout(() => this.connectTendermint(network), 3000) + }) } // For each block event, we fetch the block information and publish a message.