added page and limit to finalities by keystones request

added page and limit to finalities by keystones request, defaulting limit to 100 if 0 and setting an upper limit of 100 per page

api/bfgapi/bfgapi.go

@@ -150,6 +150,8 @@ type BTCFinalityByRecentKeystonesResponse struct {
 
 type BTCFinalityByKeystonesRequest struct {
 	L2Keystones []hemi.L2Keystone `json:"l2_keystones"`
+	Page        uint32            `json:"page,omitempty"`
+	Limit       uint32            `json:"limit,omitempty"`
 }
 
 type BTCFinalityByKeystonesResponse struct {

api/bssapi/bssapi.go

@@ -82,6 +82,8 @@ type BTCFinalityByRecentKeystonesResponse struct {
 
 type BTCFinalityByKeystonesRequest struct {
 	L2Keystones []hemi.L2Keystone `json:"l2_keystones"`
+	Page        uint32            `json:"page,omitempty"`
+	Limit       uint32            `json:"limit,omitempty"`
 }
 
 type BTCFinalityByKeystonesResponse struct {

database/bfgd/database.go

@@ -34,7 +34,7 @@ type Database interface {
 	PopBasisUpdateBTCFields(ctx context.Context, pb *PopBasis) (int64, error)
 
 	L2BTCFinalityMostRecent(ctx context.Context, limit uint32) ([]L2BTCFinality, error)
-	L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2KeystoneAbrevHashes []database.ByteArray) ([]L2BTCFinality, error)
+	L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2KeystoneAbrevHashes []database.ByteArray, page uint32, limit uint32) ([]L2BTCFinality, error)
 
 	BtcBlockCanonicalHeight(ctx context.Context) (uint64, error)
 

database/bfgd/database_ext_test.go

@@ -1598,6 +1598,8 @@ func TestL2BtcFinalitiesByL2Keystone(t *testing.T) {
 	finalities, err := db.L2BTCFinalityByL2KeystoneAbrevHash(
 		ctx,
 		[]database.ByteArray{firstKeystone.Hash},
+		0,
+		100,
 	)
 	if err != nil {
 		t.Fatal(err)
@@ -1640,6 +1642,8 @@ func TestL2BtcFinalitiesByL2KeystoneNotPublishedHeight(t *testing.T) {
 	finalities, err := db.L2BTCFinalityByL2KeystoneAbrevHash(
 		ctx,
 		[]database.ByteArray{firstKeystone.Hash},
+		0,
+		100,
 	)
 	if err != nil {
 		t.Fatal(err)

database/bfgd/postgres/postgres.go

@@ -799,14 +799,21 @@ func (p *pgdb) L2BTCFinalityMostRecent(ctx context.Context, limit uint32) ([]bfg
 
 // L2BTCFinalityByL2KeystoneAbrevHash queries for finalities by L2KeystoneAbrevHash
 // and returns them descending by l2_block_number
-func (p *pgdb) L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2KeystoneAbrevHashes []database.ByteArray) ([]bfgd.L2BTCFinality, error) {
+func (p *pgdb) L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2KeystoneAbrevHashes []database.ByteArray, page uint32, limit uint32) ([]bfgd.L2BTCFinality, error) {
 	log.Tracef("L2BTCFinalityByL2KeystoneAbrevHash")
 	defer log.Tracef("L2BTCFinalityByL2KeystoneAbrevHash exit")
 
 	if len(l2KeystoneAbrevHashes) > 100 {
 		return nil, errors.New("l2KeystoneAbrevHashes cannot be longer than 100")
 	}
 
+	// don't let users query for more than 100 at a time, default 0 to 100 for
+	// backwards compatibility
+	if limit > 100 || limit == 0 {
+		log.Tracef("limit was set to %d, defaulting to 100", limit)
+		limit = 100
+	}
+
 	sql := fmt.Sprintf(`
 		SELECT
 			btc_blocks_can.hash,
@@ -831,6 +838,10 @@ func (p *pgdb) L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2Keyston
 		WHERE l2_keystones.l2_keystone_abrev_hash = ANY($1)
 
 		ORDER BY l2_keystones.l2_block_number DESC
+
+		OFFSET $2
+
+		LIMIT $3
 	`, effectiveHeightSql)
 
 	l2KeystoneAbrevHashesStr := [][]byte{}
@@ -841,7 +852,7 @@ func (p *pgdb) L2BTCFinalityByL2KeystoneAbrevHash(ctx context.Context, l2Keyston
 	// XXX this doesn't go here
 	log.Infof("the hashes are %v", l2KeystoneAbrevHashesStr)
 
-	rows, err := p.db.QueryContext(ctx, sql, pq.Array(l2KeystoneAbrevHashesStr))
+	rows, err := p.db.QueryContext(ctx, sql, pq.Array(l2KeystoneAbrevHashesStr), page*limit, limit)
 	if err != nil {
 		return nil, err
 	}

e2e/e2e_ext_test.go

@@ -2654,6 +2654,210 @@ func TestGetFinalitiesByL2KeystoneBSS(t *testing.T) {
 	}
 }
 
+func TestGetFinalitiesByL2KeystoneBSSWithPagination(t *testing.T) {
+	db, pgUri, sdb, cleanup := createTestDB(context.Background(), t)
+	defer func() {
+		db.Close()
+		sdb.Close()
+		cleanup()
+	}()
+
+	ctx, cancel := defaultTestContext()
+	defer cancel()
+
+	_, _, bfgWsurl, _ := createBfgServer(ctx, t, pgUri, "", 1000)
+
+	_, _, bssWsurl := createBssServer(ctx, t, bfgWsurl)
+
+	btcBlock := createBtcBlock(ctx, t, db, 1, 998, []byte{}, 1) // finality should be 1000 - 998 - 9 + 1 = -6
+	createBtcBlock(ctx, t, db, 1, -1, []byte{}, 2)              // finality should be 1000 - 1000 - 9 + 1 = -8 (unpublished)
+	createBtcBlock(ctx, t, db, 1, 1000, btcBlock.Hash, 3)       // finality should be 1000 - 1000 - 9 + 1 = -8
+	expectedFinalitiesDesc := []int32{-8, -6}
+
+	c, _, err := websocket.Dial(ctx, bssWsurl, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.CloseNow()
+
+	assertPing(ctx, t, c, bssapi.CmdPingRequest)
+
+	bws := &bssWs{
+		conn: protocol.NewWSConn(c),
+	}
+
+	// first and second btcBlocks
+	recentFinalities, err := db.L2BTCFinalityMostRecent(ctx, 100)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	l2Keystones := []hemi.L2Keystone{}
+	for _, r := range recentFinalities[1:] {
+		l, err := hemi.L2BTCFinalityFromBfgd(&r, 0, 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+		l2Keystones = append(l2Keystones, l.L2Keystone)
+	}
+
+	receivedFinalities := []hemi.L2BTCFinality{}
+
+	for i := range 2 {
+		finalityRequest := bssapi.BTCFinalityByKeystonesRequest{
+			L2Keystones: l2Keystones,
+			Page:        uint32(i),
+			Limit:       1,
+		}
+
+		err = bssapi.Write(ctx, bws.conn, "someid", finalityRequest)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var v protocol.Message
+		err = wsjson.Read(ctx, c, &v)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if v.Header.Command != bssapi.CmdBTCFinalityByKeystonesResponse {
+			t.Fatalf("received unexpected command: %s", v.Header.Command)
+		}
+
+		finalityResponse := bssapi.BTCFinalityByRecentKeystonesResponse{}
+		err = json.Unmarshal(v.Payload, &finalityResponse)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		t.Logf("length is %d", len(finalityResponse.L2BTCFinalities))
+
+		if len(finalityResponse.L2BTCFinalities) != 1 {
+			t.Fatalf("unexpected length %v", len(finalityResponse.L2BTCFinalities))
+		}
+
+		receivedFinalities = append(receivedFinalities, finalityResponse.L2BTCFinalities[0])
+	}
+
+	expectedResponse := []hemi.L2BTCFinality{}
+	for i, r := range recentFinalities[1:] {
+		f, err := hemi.L2BTCFinalityFromBfgd(&r, 0, 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		f.BTCFinality = expectedFinalitiesDesc[i]
+		expectedResponse = append(expectedResponse, *f)
+	}
+
+	diff := deep.Equal(expectedResponse, receivedFinalities)
+	if len(diff) > 0 {
+		t.Fatalf("unexpected diff %s", diff)
+	}
+}
+
+func TestGetFinalitiesByL2KeystoneBFGWithPagination(t *testing.T) {
+	db, pgUri, sdb, cleanup := createTestDB(context.Background(), t)
+	defer func() {
+		db.Close()
+		sdb.Close()
+		cleanup()
+	}()
+
+	ctx, cancel := defaultTestContext()
+	defer cancel()
+
+	_, _, bfgWsurl, _ := createBfgServer(ctx, t, pgUri, "", 1000)
+
+	btcBlock := createBtcBlock(ctx, t, db, 1, 998, []byte{}, 1) // finality should be 1000 - 998 - 9 + 1 = -6
+	createBtcBlock(ctx, t, db, 1, -1, []byte{}, 2)              // finality should be 1000 - 1000 - 9 + 1 = -8 (unpublished)
+	createBtcBlock(ctx, t, db, 1, 1000, btcBlock.Hash, 3)       // finality should be 1000 - 1000 - 9 + 1 = -8
+	expectedFinalitiesDesc := []int32{-8, -6}
+
+	c, _, err := websocket.Dial(ctx, bfgWsurl, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.CloseNow()
+
+	assertPing(ctx, t, c, bfgapi.CmdPingRequest)
+
+	bws := &bssWs{
+		conn: protocol.NewWSConn(c),
+	}
+
+	// first and second btcBlocks
+	recentFinalities, err := db.L2BTCFinalityMostRecent(ctx, 100)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	l2Keystones := []hemi.L2Keystone{}
+	for _, r := range recentFinalities[1:] {
+		l, err := hemi.L2BTCFinalityFromBfgd(&r, 0, 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+		l2Keystones = append(l2Keystones, l.L2Keystone)
+	}
+
+	receivedFinalities := []hemi.L2BTCFinality{}
+
+	for i := range 2 {
+		finalityRequest := bfgapi.BTCFinalityByKeystonesRequest{
+			L2Keystones: l2Keystones,
+			Page:        uint32(i),
+			Limit:       1,
+		}
+
+		err = bfgapi.Write(ctx, bws.conn, "someid", finalityRequest)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var v protocol.Message
+		err = wsjson.Read(ctx, c, &v)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		if v.Header.Command != bfgapi.CmdBTCFinalityByKeystonesResponse {
+			t.Fatalf("received unexpected command: %s", v.Header.Command)
+		}
+
+		finalityResponse := bfgapi.BTCFinalityByRecentKeystonesResponse{}
+		err = json.Unmarshal(v.Payload, &finalityResponse)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		t.Logf("length is %d", len(finalityResponse.L2BTCFinalities))
+
+		if len(finalityResponse.L2BTCFinalities) != 1 {
+			t.Fatalf("unexpected length %v", len(finalityResponse.L2BTCFinalities))
+		}
+
+		receivedFinalities = append(receivedFinalities, finalityResponse.L2BTCFinalities[0])
+	}
+
+	expectedResponse := []hemi.L2BTCFinality{}
+	for i, r := range recentFinalities[1:] {
+		f, err := hemi.L2BTCFinalityFromBfgd(&r, 0, 0)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		f.BTCFinality = expectedFinalitiesDesc[i]
+		expectedResponse = append(expectedResponse, *f)
+	}
+
+	diff := deep.Equal(expectedResponse, receivedFinalities)
+	if len(diff) > 0 {
+		t.Fatalf("unexpected diff %s", diff)
+	}
+}
+
 func TestGetFinalitiesByL2KeystoneBSSLowerServerHeight(t *testing.T) {
 	db, pgUri, sdb, cleanup := createTestDB(context.Background(), t)
 	defer func() {

service/bfg/bfg.go

@@ -1275,6 +1275,8 @@ func (s *Server) handleBtcFinalityByKeystonesRequest(ctx context.Context, bfkr *
 	finalities, err := s.db.L2BTCFinalityByL2KeystoneAbrevHash(
 		ctx,
 		l2KeystoneAbrevHashes,
+		bfkr.Page,
+		bfkr.Limit,
 	)
 	if err != nil {
 		e := protocol.NewInternalErrorf("l2 keystones: %w", err)

service/bss/bss.go

@@ -290,6 +290,8 @@ func (s *Server) handleBtcFinalityByKeystonesRequest(ctx context.Context, msg *b
 
 	response, err := s.callBFG(ctx, &bfgapi.BTCFinalityByKeystonesRequest{
 		L2Keystones: msg.L2Keystones,
+		Limit:       msg.Limit,
+		Page:        msg.Page,
 	})
 	if err != nil {
 		e := protocol.NewInternalErrorf("btc finality keystones: %w", err)