Merge dev to main

.github/workflows/publish.yml

@@ -4,7 +4,6 @@ on:
   push:
     branches:
     - 'main'
-    - 'dev'
     tags:
     - '*'
 
@@ -15,7 +14,7 @@ jobs:
      install-dependencies-command: 'sudo apt-get install libzmq3-dev'
      run-unit-tests: true
      run-integration-tests: true
-     
+
   docker_pipeline:
     needs: ["lint_test"]
     uses: babylonlabs-io/.github/.github/workflows/[email protected]

CONTRIBUTING.md

@@ -0,0 +1,5 @@
+# Contributing
+
+Staking-indexer repository follows the same contributing rules as
+[Babylon node](https://github.com/babylonlabs-io/babylon/blob/main/CONTRIBUTING.md)
+repository.

RELEASE_PROCESS.md

@@ -0,0 +1,5 @@
+# Release Process
+
+Staking-indexer repository follows the same release process rules as
+[Babylon node](https://github.com/babylonlabs-io/babylon/blob/main/RELEASE_PROCESS.md)
+repository.

cmd/sid/cli/btc_headers.go

@@ -17,16 +17,19 @@ import (
 	"github.com/babylonlabs-io/staking-indexer/config"
 	"github.com/babylonlabs-io/staking-indexer/log"
 	"github.com/babylonlabs-io/staking-indexer/utils"
-
-	sdkmath "cosmossdk.io/math"
 )
 
 const (
 	outputFileFlag        = "output"
+	withHeightFlag        = "with-height"
 	defaultOutputFileName = "btc-headers.json"
 	filePermission        = 0600
 )
 
+type HeadersState struct {
+	BtcHeaders []*bbnbtclightclienttypes.BTCHeaderInfo `json:"btc_headers,omitempty"`
+}
+
 var BtcHeaderCommand = cli.Command{
 	Name:        "btc-headers",
 	Usage:       "Output a range of BTC headers into a JSON file.",
@@ -43,6 +46,10 @@ var BtcHeaderCommand = cli.Command{
 			Usage: "The path to the output file",
 			Value: filepath.Join(config.DefaultHomeDir, defaultOutputFileName),
 		},
+		cli.BoolFlag{
+			Name:  withHeightFlag,
+			Usage: "If it should fill the BTC block height property",
+		},
 	},
 	Action: btcHeaders,
 }
@@ -92,18 +99,18 @@ func btcHeaders(ctx *cli.Context) error {
 		return fmt.Errorf("failed to initialize the BTC client: %w", err)
 	}
 
-	btcHeaders, err := BtcHeaderInfoList(btcClient, fromBlock, toBlock)
+	btcHeaders, err := BtcHeaderInfoList(btcClient, fromBlock, toBlock, ctx.Bool(withHeightFlag))
 	if err != nil {
 		return fmt.Errorf("failed to get BTC headers: %w", err)
 	}
 
-	genState := bbnbtclightclienttypes.GenesisState{
+	headersState := HeadersState{
 		BtcHeaders: btcHeaders,
 	}
 
-	bz, err := json.MarshalIndent(genState, "", "  ")
+	bz, err := json.MarshalIndent(headersState, "", "  ")
 	if err != nil {
-		return fmt.Errorf("failed to generate json to set to output file %+v: %w", genState, err)
+		return fmt.Errorf("failed to generate json to set to output file %+v: %w", headersState, err)
 	}
 
 	outputFilePath := ctx.String(outputFileFlag)
@@ -121,23 +128,25 @@ func btcHeaders(ctx *cli.Context) error {
 }
 
 // BtcHeaderInfoList queries the btc client for (fromBlk ~ toBlk) BTC blocks, converting to BTCHeaderInfo.
-func BtcHeaderInfoList(btcClient btcscanner.Client, fromBlk, toBlk uint64) ([]*bbnbtclightclienttypes.BTCHeaderInfo, error) {
+func BtcHeaderInfoList(btcClient btcscanner.Client, fromBlk, toBlk uint64, withHeight bool) ([]*bbnbtclightclienttypes.BTCHeaderInfo, error) {
 	btcHeaders := make([]*bbnbtclightclienttypes.BTCHeaderInfo, 0, toBlk-fromBlk+1)
-	var currenWork = sdkmath.ZeroUint()
 
 	for blkHeight := fromBlk; blkHeight <= toBlk; blkHeight++ {
 		blkHeader, err := btcClient.GetBlockHeaderByHeight(blkHeight)
 		if err != nil {
 			return nil, fmt.Errorf("failed to get block height %d from BTC client: %w", blkHeight, err)
 		}
 
-		headerWork := bbnbtclightclienttypes.CalcHeaderWork(blkHeader)
-		currenWork = bbnbtclightclienttypes.CumulativeWork(headerWork, currenWork)
-
 		headerBytes := babylontypes.NewBTCHeaderBytesFromBlockHeader(blkHeader)
+		info := &bbnbtclightclienttypes.BTCHeaderInfo{
+			Header: &headerBytes,
+		}
+
+		if withHeight {
+			info.Height = blkHeight
+		}
 
-		bbnBtcHeaderInfo := bbnbtclightclienttypes.NewBTCHeaderInfo(&headerBytes, headerBytes.Hash(), blkHeight, &currenWork)
-		btcHeaders = append(btcHeaders, bbnBtcHeaderInfo)
+		btcHeaders = append(btcHeaders, info)
 	}
 	return btcHeaders, nil
 }

cmd/sid/cli/btc_headers_test.go

@@ -36,7 +36,7 @@ func FuzzBtcHeaders(f *testing.F) {
 				Return(idxBlock.Header, nil).AnyTimes()
 		}
 
-		infos, err := cli.BtcHeaderInfoList(mockBtcClient, startHeight, endHeight)
+		infos, err := cli.BtcHeaderInfoList(mockBtcClient, startHeight, endHeight, true)
 		require.NoError(t, err)
 		require.EqualValues(t, len(infos), numBlocks)
 
@@ -45,7 +45,6 @@ func FuzzBtcHeaders(f *testing.F) {
 			headerBytes := babylontypes.NewBTCHeaderBytesFromBlockHeader(idxBlock.Header)
 			require.Equal(t, info.Header, &headerBytes)
 			require.EqualValues(t, info.Height, idxBlock.Height)
-			require.EqualValues(t, info.Hash, headerBytes.Hash())
 		}
 	})
 }

config/config.go

@@ -31,12 +31,13 @@ var (
 
 // Config is the main config for the fpd cli command
 type Config struct {
-	LogLevel       string         `long:"loglevel" description:"Logging level for all subsystems" choice:"trace" choice:"debug" choice:"info" choice:"warn" choice:"error" choice:"fatal"`
-	BitcoinNetwork string         `long:"bitcoinnetwork" description:"Bitcoin network to run on" choice:"mainnet" choice:"regtest" choice:"testnet" choice:"simnet" choice:"signet"`
-	BTCConfig      *BTCConfig     `group:"btcconfig" namespace:"btcconfig"`
-	DatabaseConfig *DBConfig      `group:"dbconfig" namespace:"dbconfig"`
-	QueueConfig    *QueueConfig   `group:"queueconfig" namespace:"queueconfig"`
-	MetricsConfig  *MetricsConfig `group:"metricsconfig" namespace:"metricsconfig"`
+	LogLevel          string         `long:"loglevel" description:"Logging level for all subsystems" choice:"trace" choice:"debug" choice:"info" choice:"warn" choice:"error" choice:"fatal"`
+	BitcoinNetwork    string         `long:"bitcoinnetwork" description:"Bitcoin network to run on" choice:"mainnet" choice:"regtest" choice:"testnet" choice:"simnet" choice:"signet"`
+	ExtraEventEnabled bool           `long:"extraeventenabled" description:"Whether emitting non-default events is allowed"`
+	BTCConfig         *BTCConfig     `group:"btcconfig" namespace:"btcconfig"`
+	DatabaseConfig    *DBConfig      `group:"dbconfig" namespace:"dbconfig"`
+	QueueConfig       *QueueConfig   `group:"queueconfig" namespace:"queueconfig"`
+	MetricsConfig     *MetricsConfig `group:"metricsconfig" namespace:"metricsconfig"`
 
 	BTCNetParams chaincfg.Params
 }

consumer/event_consumer.go

@@ -10,5 +10,6 @@ type EventConsumer interface {
 	PushUnbondingEvent(ev *client.UnbondingStakingEvent) error
 	PushWithdrawEvent(ev *client.WithdrawStakingEvent) error
 	PushBtcInfoEvent(ev *client.BtcInfoEvent) error
+	PushConfirmedInfoEvent(ev *client.ConfirmedInfoEvent) error
 	Stop() error
 }

indexer/indexer.go

@@ -383,6 +383,18 @@ func (si *StakingIndexer) HandleConfirmedBlock(b *types.IndexedBlock) error {
 		return fmt.Errorf("failed to save the last processed height: %w", err)
 	}
 
+	if si.cfg.ExtraEventEnabled {
+		// emit ConfirmedInfoEvent to send the confirmed height and tvl
+		confirmedTvl, err := si.is.GetConfirmedTvl()
+		if err != nil {
+			return fmt.Errorf("failed to get the confirmed tvl: %w", err)
+		}
+		confirmedInfoEvent := queuecli.NewConfirmedInfoEvent(uint64(b.Height), confirmedTvl)
+		if err := si.consumer.PushConfirmedInfoEvent(&confirmedInfoEvent); err != nil {
+			return fmt.Errorf("failed to push the confirmed info event: %w", err)
+		}
+	}
+
 	// record metrics
 	lastProcessedBtcHeight.Set(float64(b.Height))
 

itest/e2e_test.go

@@ -12,7 +12,6 @@ import (
 
 	"github.com/babylonlabs-io/babylon/btcstaking"
 	bbndatagen "github.com/babylonlabs-io/babylon/testutil/datagen"
-	bbnbtclightclienttypes "github.com/babylonlabs-io/babylon/x/btclightclient/types"
 	queuecli "github.com/babylonlabs-io/staking-queue-client/client"
 	"github.com/btcsuite/btcd/btcec/v2"
 	"github.com/btcsuite/btcd/btcutil"
@@ -99,11 +98,14 @@ func TestStakingLifeCycle(t *testing.T) {
 	stakingTxHash := stakingTx.TxHash()
 	tm.SendTxWithNConfirmations(t, stakingTx, int(k))
 
+	tm.CheckConfirmedInfoEvent(t, 100, 0)
+
 	// check that the staking tx is already stored
 	_ = tm.WaitForStakingTxStored(t, stakingTxHash)
 
 	// check the staking event is received by the queue
 	tm.CheckNextStakingEvent(t, stakingTxHash)
+	tm.CheckConfirmedInfoEvent(t, 102, uint64(testStakingData.StakingAmount))
 
 	// wait for the staking tx expires
 	if uint64(testStakingData.StakingTime) > k {
@@ -635,20 +637,15 @@ func TestTimeBasedCap(t *testing.T) {
 func TestBtcHeaders(t *testing.T) {
 	r := rand.New(rand.NewSource(10))
 	blocksPerRetarget := 2016
-	genState := bbnbtclightclienttypes.DefaultGenesis()
 
 	initBlocksQnt := r.Intn(15) + blocksPerRetarget
 	btcd, btcClient := StartBtcClientAndBtcHandler(t, initBlocksQnt)
 
 	// from zero height
-	infos, err := cli.BtcHeaderInfoList(btcClient, 0, uint64(initBlocksQnt))
+	infos, err := cli.BtcHeaderInfoList(btcClient, 0, uint64(initBlocksQnt), true)
 	require.NoError(t, err)
 	require.Equal(t, len(infos), initBlocksQnt+1)
 
-	// should be valid on genesis, start from zero height.
-	genState.BtcHeaders = infos
-	require.NoError(t, genState.Validate())
-
 	generatedBlocksQnt := r.Intn(15) + 2
 	btcd.GenerateBlocks(generatedBlocksQnt)
 	totalBlks := initBlocksQnt + generatedBlocksQnt
@@ -657,22 +654,15 @@ func TestBtcHeaders(t *testing.T) {
 	fromBlockHeight := blocksPerRetarget - 1
 	toBlockHeight := totalBlks - 2
 
-	infos, err = cli.BtcHeaderInfoList(btcClient, uint64(fromBlockHeight), uint64(toBlockHeight))
+	infos, err = cli.BtcHeaderInfoList(btcClient, uint64(fromBlockHeight), uint64(toBlockHeight), true)
 	require.NoError(t, err)
 	require.Equal(t, len(infos), int(toBlockHeight-fromBlockHeight)+1)
-
-	// try to check if it is valid on genesis, should fail is not retarget block.
-	genState.BtcHeaders = infos
-	require.EqualError(t, genState.Validate(), "genesis block must be a difficulty adjustment block")
+	require.EqualValues(t, infos[len(infos)-1].Height, uint64(toBlockHeight))
 
 	// from retarget block
-	infos, err = cli.BtcHeaderInfoList(btcClient, uint64(blocksPerRetarget), uint64(totalBlks))
+	infos, err = cli.BtcHeaderInfoList(btcClient, uint64(blocksPerRetarget), uint64(totalBlks), true)
 	require.NoError(t, err)
 	require.Equal(t, len(infos), int(totalBlks-blocksPerRetarget)+1)
-
-	// check if it is valid on genesis
-	genState.BtcHeaders = infos
-	require.NoError(t, genState.Validate())
 }
 
 func getCovenantPrivKeys(t *testing.T) []*btcec.PrivateKey {

itest/test_manager.go

@@ -39,23 +39,24 @@ import (
 )
 
 type TestManager struct {
-	Config             *config.Config
-	Db                 kvdb.Backend
-	Si                 *indexer.StakingIndexer
-	BS                 *btcscanner.BtcPoller
-	WalletPrivKey      *btcec.PrivateKey
-	serverStopper      *signal.Interceptor
-	wg                 *sync.WaitGroup
-	BitcoindHandler    *BitcoindTestHandler
-	WalletClient       *rpcclient.Client
-	MinerAddr          btcutil.Address
-	DirPath            string
-	QueueConsumer      *queuemngr.QueueManager
-	StakingEventChan   <-chan queuecli.QueueMessage
-	UnbondingEventChan <-chan queuecli.QueueMessage
-	WithdrawEventChan  <-chan queuecli.QueueMessage
-	BtcInfoEventChan   <-chan queuecli.QueueMessage
-	VersionedParams    *parser.ParsedGlobalParams
+	Config                 *config.Config
+	Db                     kvdb.Backend
+	Si                     *indexer.StakingIndexer
+	BS                     *btcscanner.BtcPoller
+	WalletPrivKey          *btcec.PrivateKey
+	serverStopper          *signal.Interceptor
+	wg                     *sync.WaitGroup
+	BitcoindHandler        *BitcoindTestHandler
+	WalletClient           *rpcclient.Client
+	MinerAddr              btcutil.Address
+	DirPath                string
+	QueueConsumer          *queuemngr.QueueManager
+	StakingEventChan       <-chan queuecli.QueueMessage
+	UnbondingEventChan     <-chan queuecli.QueueMessage
+	WithdrawEventChan      <-chan queuecli.QueueMessage
+	BtcInfoEventChan       <-chan queuecli.QueueMessage
+	ConfirmedInfoEventChan <-chan queuecli.QueueMessage
+	VersionedParams        *parser.ParsedGlobalParams
 }
 
 // bitcoin params used for testing
@@ -146,6 +147,8 @@ func StartWithBitcoinHandler(t *testing.T, h *BitcoindTestHandler, minerAddress
 	require.NoError(t, err)
 	unconfirmedEventChan, err := queueConsumer.BtcInfoQueue.ReceiveMessages()
 	require.NoError(t, err)
+	confirmedInfoEventChan, err := queueConsumer.ConfirmedInfoQueue.ReceiveMessages()
+	require.NoError(t, err)
 
 	db, err := cfg.DatabaseConfig.GetDbBackend()
 	require.NoError(t, err)
@@ -176,22 +179,23 @@ func StartWithBitcoinHandler(t *testing.T, h *BitcoindTestHandler, minerAddress
 	time.Sleep(3 * time.Second)
 
 	return &TestManager{
-		Config:             cfg,
-		Si:                 si,
-		BS:                 scanner,
-		serverStopper:      &interceptor,
-		wg:                 &wg,
-		BitcoindHandler:    h,
-		WalletClient:       rpcclient,
-		WalletPrivKey:      walletPrivKey.PrivKey,
-		MinerAddr:          minerAddress,
-		DirPath:            dirPath,
-		QueueConsumer:      queueConsumer,
-		StakingEventChan:   stakingEventChan,
-		UnbondingEventChan: unbondingEventChan,
-		WithdrawEventChan:  withdrawEventChan,
-		BtcInfoEventChan:   unconfirmedEventChan,
-		VersionedParams:    versionedParams,
+		Config:                 cfg,
+		Si:                     si,
+		BS:                     scanner,
+		serverStopper:          &interceptor,
+		wg:                     &wg,
+		BitcoindHandler:        h,
+		WalletClient:           rpcclient,
+		WalletPrivKey:          walletPrivKey.PrivKey,
+		MinerAddr:              minerAddress,
+		DirPath:                dirPath,
+		QueueConsumer:          queueConsumer,
+		StakingEventChan:       stakingEventChan,
+		UnbondingEventChan:     unbondingEventChan,
+		WithdrawEventChan:      withdrawEventChan,
+		BtcInfoEventChan:       unconfirmedEventChan,
+		ConfirmedInfoEventChan: confirmedInfoEventChan,
+		VersionedParams:        versionedParams,
 	}
 }
 
@@ -214,7 +218,10 @@ func ReStartFromHeight(t *testing.T, tm *TestManager, height uint64) *TestManage
 func DefaultStakingIndexerConfig(homePath string) *config.Config {
 	defaultConfig := config.DefaultConfigWithHome(homePath)
 
-	// both wallet and node are bicoind
+	// enable emitting extra events for testing
+	defaultConfig.ExtraEventEnabled = true
+
+	// both wallet and node are bitcoind
 	defaultConfig.BTCNetParams = *regtestParams
 
 	bitcoindHost := "127.0.0.1:18443"
@@ -361,6 +368,23 @@ func (tm *TestManager) CheckNextWithdrawEvent(t *testing.T, stakingTxHash chainh
 	require.NoError(t, err)
 }
 
+func (tm *TestManager) CheckConfirmedInfoEvent(t *testing.T, height, tvl uint64) {
+	var confirmedInfoEv queuecli.ConfirmedInfoEvent
+
+	for {
+		confirmedInfoEventBytes := <-tm.ConfirmedInfoEventChan
+		err := tm.QueueConsumer.ConfirmedInfoQueue.DeleteMessage(confirmedInfoEventBytes.Receipt)
+		require.NoError(t, err)
+		err = json.Unmarshal([]byte(confirmedInfoEventBytes.Body), &confirmedInfoEv)
+		require.NoError(t, err)
+		if height != confirmedInfoEv.Height {
+			continue
+		}
+		require.Equal(t, confirmedInfoEv.Tvl, tvl)
+		return
+	}
+}
+
 func (tm *TestManager) CheckNextUnconfirmedEvent(t *testing.T, confirmedTvl, totalTvl uint64) {
 	var btcInfoEvent queuecli.BtcInfoEvent
 
@@ -376,6 +400,7 @@ func (tm *TestManager) CheckNextUnconfirmedEvent(t *testing.T, confirmedTvl, tot
 		if totalTvl != btcInfoEvent.UnconfirmedTvl {
 			continue
 		}
+
 		return
 	}
 }