feat: add Solana RPC status check and some refactor around RPC status…

… check (#2751)

* add Solana RPC status check and refactor EVM and bitcoin RPC status check

* add changelog entry

* add RPC alert latency to config file

* enable live test by ENABLE_LIVE_TEST environment variable

* unified RPC error alert message

* move rpcAlertLatency argument out of rpc methods

* unified param 'rpcAlertLatency' as type int64 when passing cfg.RPCAlertLatency to NewObserver()

---------

Co-authored-by: Dmitry S <[email protected]>

changelog.md

@@ -46,6 +46,11 @@
 * [2681](https://github.com/zeta-chain/node/pull/2681) - implement `MsgUpdateERC20CustodyPauseStatus` to pause or unpause ERC20 Custody contract (to be used for the migration process for smart contract V2)
 * [2644](https://github.com/zeta-chain/node/pull/2644) - add created_timestamp to cctx status
 * [2673](https://github.com/zeta-chain/node/pull/2673) - add relayer key importer, encryption and decryption
+* [2633](https://github.com/zeta-chain/node/pull/2633) - support for stateful precompiled contracts
+* [2751](https://github.com/zeta-chain/node/pull/2751) - add RPC status check for Solana chain
+* [2788](https://github.com/zeta-chain/node/pull/2788) - add common importable zetacored rpc package
+* [2784](https://github.com/zeta-chain/node/pull/2784) - staking precompiled contract
+* [2795](https://github.com/zeta-chain/node/pull/2795) - support restricted address in Solana
 * [2825](https://github.com/zeta-chain/node/pull/2825) - add Bitcoin inscriptions support
 
 ### Refactor

zetaclient/chains/base/observer.go

@@ -7,6 +7,7 @@ import (
 	"strconv"
 	"sync"
 	"sync/atomic"
+	"time"
 
 	lru "github.com/hashicorp/golang-lru"
 	"github.com/pkg/errors"
@@ -60,6 +61,9 @@ type Observer struct {
 	// lastTxScanned is the last transaction hash scanned by the observer
 	lastTxScanned string
 
+	// rpcAlertLatency is the threshold of RPC latency to trigger an alert
+	rpcAlertLatency time.Duration
+
 	// blockCache is the cache for blocks
 	blockCache *lru.Cache
 
@@ -92,6 +96,7 @@ func NewObserver(
 	tss interfaces.TSSSigner,
 	blockCacheSize int,
 	headerCacheSize int,
+	rpcAlertLatency int64,
 	ts *metrics.TelemetryServer,
 	database *db.DB,
 	logger Logger,
@@ -104,6 +109,7 @@ func NewObserver(
 		lastBlock:        0,
 		lastBlockScanned: 0,
 		lastTxScanned:    "",
+		rpcAlertLatency:  time.Duration(rpcAlertLatency) * time.Second,
 		ts:               ts,
 		db:               database,
 		mu:               &sync.Mutex{},
@@ -452,6 +458,27 @@ func (ob *Observer) PostVoteInbound(
 	return ballot, err
 }
 
+// AlertOnRPCLatency prints an alert if the RPC latency exceeds the threshold.
+// Returns true if the RPC latency is too high.
+func (ob *Observer) AlertOnRPCLatency(latestBlockTime time.Time, defaultAlertLatency time.Duration) bool {
+	// use configured alert latency if set
+	alertLatency := defaultAlertLatency
+	if ob.rpcAlertLatency > 0 {
+		alertLatency = ob.rpcAlertLatency
+	}
+
+	// latest block should not be too old
+	elapsedTime := time.Since(latestBlockTime)
+	if elapsedTime > alertLatency {
+		ob.logger.Chain.Error().
+			Msgf("RPC is stale: latest block is %.0f seconds old, RPC down or chain stuck (check explorer)?", elapsedTime.Seconds())
+		return true
+	}
+
+	ob.logger.Chain.Info().Msgf("RPC is OK: latest block is %.0f seconds old", elapsedTime.Seconds())
+	return false
+}
+
 // EnvVarLatestBlockByChain returns the environment variable for the last block by chain.
 func EnvVarLatestBlockByChain(chain chains.Chain) string {
 	return fmt.Sprintf("CHAIN_%d_SCAN_FROM_BLOCK", chain.ChainId)