From a78e73f71b9cd009b81e6c63e0278303cb73ab40 Mon Sep 17 00:00:00 2001 From: Urvi Date: Wed, 18 Oct 2023 21:59:52 -0700 Subject: [PATCH 01/14] Use FastBatchInsertBuilder to insert to insert into claimable_balances and claimable_balance_claimants tables --- .../actions/claimable_balance_test.go | 76 +++---- .../claimable_balance_batch_insert_builder.go | 47 +++++ ...e_balance_claimant_batch_insert_builder.go | 35 ++-- .../db2/history/claimable_balances.go | 34 +--- .../db2/history/claimable_balances_test.go | 169 +++++----------- ..._claimable_balance_batch_insert_builder.go | 27 +++ ...e_balance_claimant_batch_insert_builder.go | 14 +- .../db2/history/mock_q_claimable_balances.go | 14 +- .../internal/ingest/processor_runner.go | 9 +- .../internal/ingest/processor_runner_test.go | 35 +++- .../claimable_balances_change_processor.go | 97 +++++---- ...laimable_balances_change_processor_test.go | 191 ++++++++---------- .../horizon/internal/ingest/verify_test.go | 12 ++ 13 files changed, 375 insertions(+), 385 deletions(-) create mode 100644 services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go create mode 100644 services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go diff --git a/services/horizon/internal/actions/claimable_balance_test.go b/services/horizon/internal/actions/claimable_balance_test.go index ff59b96134..05ff7532c2 100644 --- a/services/horizon/internal/actions/claimable_balance_test.go +++ b/services/horizon/internal/actions/claimable_balance_test.go @@ -1,6 +1,7 @@ package actions import ( + "database/sql" "net/http/httptest" "testing" @@ -20,6 +21,9 @@ func TestGetClaimableBalanceByID(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{tt.HorizonSession()} + q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) + defer q.SessionInterface.Rollback() + accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" asset := xdr.MustNewCreditAsset("USD", accountID) balanceID := xdr.ClaimableBalanceId{ @@ -43,20 +47,22 @@ func TestGetClaimableBalanceByID(t *testing.T) { LastModifiedLedger: 123, } - err = q.UpsertClaimableBalances(tt.Ctx, []history.ClaimableBalance{cBalance}) - tt.Assert.NoError(err) + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() + tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(10) + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() for _, claimant := range cBalance.Claimants { claimant := history.ClaimableBalanceClaimant{ BalanceID: cBalance.BalanceID, Destination: claimant.Destination, LastModifiedLedger: cBalance.LastModifiedLedger, } - err = claimantsInsertBuilder.Add(tt.Ctx, claimant) - tt.Assert.NoError(err) + tt.Assert.NoError(claimantsInsertBuilder.Add(claimant)) } + tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) + handler := GetClaimableBalanceByIDHandler{} response, err := handler.GetResource(httptest.NewRecorder(), makeRequest( t, @@ -148,6 +154,9 @@ func TestGetClaimableBalances(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{tt.HorizonSession()} + q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) + defer q.SessionInterface.Rollback() + entriesMeta := []struct { id xdr.Hash accountID string @@ -187,25 +196,24 @@ func TestGetClaimableBalances(t *testing.T) { hCBs = append(hCBs, cb) } - err := q.UpsertClaimableBalances(tt.Ctx, hCBs) - tt.Assert.NoError(err) - - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(10) + balanceInsertbuilder := q.NewClaimableBalanceBatchInsertBuilder() + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() for _, cBalance := range hCBs { + tt.Assert.NoError(balanceInsertbuilder.Add(cBalance)) + for _, claimant := range cBalance.Claimants { claimant := history.ClaimableBalanceClaimant{ BalanceID: cBalance.BalanceID, Destination: claimant.Destination, LastModifiedLedger: cBalance.LastModifiedLedger, } - err = claimantsInsertBuilder.Add(tt.Ctx, claimant) - tt.Assert.NoError(err) + tt.Assert.NoError(claimantsInsertBuilder.Add(claimant)) } } - err = claimantsInsertBuilder.Exec(tt.Ctx) - tt.Assert.NoError(err) + tt.Assert.NoError(balanceInsertbuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) handler := GetClaimableBalancesHandler{} response, err := handler.GetResourcePage(httptest.NewRecorder(), makeRequest( @@ -284,11 +292,8 @@ func TestGetClaimableBalances(t *testing.T) { tt.Assert.NoError(err) tt.Assert.Len(response, 0) - // new claimable balances are ingest and one of them updated, they should appear in the next pages - cbToBeUpdated := hCBs[3] - cbToBeUpdated.Sponsor = null.StringFrom("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML") - cbToBeUpdated.LastModifiedLedger = 1238 - q.UpsertClaimableBalances(tt.Ctx, []history.ClaimableBalance{cbToBeUpdated}) + balanceInsertbuilder = q.NewClaimableBalanceBatchInsertBuilder() + claimantsInsertBuilder = q.NewClaimableBalanceClaimantBatchInsertBuilder() entriesMeta = []struct { id xdr.Hash @@ -316,23 +321,21 @@ func TestGetClaimableBalances(t *testing.T) { hCBs = append(hCBs, entry) } - err = q.UpsertClaimableBalances(tt.Ctx, hCBs) - tt.Assert.NoError(err) - for _, cBalance := range hCBs { + tt.Assert.NoError(balanceInsertbuilder.Add(cBalance)) + for _, claimant := range cBalance.Claimants { claimant := history.ClaimableBalanceClaimant{ BalanceID: cBalance.BalanceID, Destination: claimant.Destination, LastModifiedLedger: cBalance.LastModifiedLedger, } - err = claimantsInsertBuilder.Add(tt.Ctx, claimant) - tt.Assert.NoError(err) + tt.Assert.NoError(claimantsInsertBuilder.Add(claimant)) } } - err = claimantsInsertBuilder.Exec(tt.Ctx) - tt.Assert.NoError(err) + tt.Assert.NoError(balanceInsertbuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) response, err = handler.GetResourcePage(httptest.NewRecorder(), makeRequest( t, @@ -372,21 +375,6 @@ func TestGetClaimableBalances(t *testing.T) { q, )) - tt.Assert.NoError(err) - tt.Assert.Len(response, 1) - - tt.Assert.Equal(cbToBeUpdated.BalanceID, response[0].(protocol.ClaimableBalance).BalanceID) - - response, err = handler.GetResourcePage(httptest.NewRecorder(), makeRequest( - t, - map[string]string{ - "limit": "2", - "cursor": response[0].(protocol.ClaimableBalance).PagingToken(), - }, - map[string]string{}, - q, - )) - tt.Assert.NoError(err) tt.Assert.Len(response, 0) @@ -404,9 +392,7 @@ func TestGetClaimableBalances(t *testing.T) { tt.Assert.NoError(err) tt.Assert.Len(response, 2) - tt.Assert.Equal(cbToBeUpdated.BalanceID, response[0].(protocol.ClaimableBalance).BalanceID) - - tt.Assert.Equal(hCBs[1].BalanceID, response[1].(protocol.ClaimableBalance).BalanceID) + tt.Assert.Equal(hCBs[1].BalanceID, response[0].(protocol.ClaimableBalance).BalanceID) response, err = handler.GetResourcePage(httptest.NewRecorder(), makeRequest( t, @@ -422,8 +408,6 @@ func TestGetClaimableBalances(t *testing.T) { tt.Assert.NoError(err) tt.Assert.Len(response, 1) - tt.Assert.Equal(hCBs[0].BalanceID, response[0].(protocol.ClaimableBalance).BalanceID) - // filter by asset response, err = handler.GetResourcePage(httptest.NewRecorder(), makeRequest( t, @@ -492,7 +476,7 @@ func TestGetClaimableBalances(t *testing.T) { )) tt.Assert.NoError(err) - tt.Assert.Len(response, 3) + tt.Assert.Len(response, 2) for _, resource := range response { tt.Assert.Equal( "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML", diff --git a/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go b/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go new file mode 100644 index 0000000000..08ab005eb8 --- /dev/null +++ b/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go @@ -0,0 +1,47 @@ +package history + +import ( + "context" + + "github.com/stellar/go/support/db" + "github.com/stellar/go/xdr" +) + +// ClaimableBalanceBatchInsertBuilder is used to insert claimable balance into the +// claimable_balances table +type ClaimableBalanceBatchInsertBuilder interface { + Add(claimableBalance ClaimableBalance) error + Exec(ctx context.Context, session db.SessionInterface) error + Reset() error +} + +// ClaimableBalanceBatchInsertBuilder is a simple wrapper around db.FastBatchInsertBuilder +type claimableBalanceBatchInsertBuilder struct { + encodingBuffer *xdr.EncodingBuffer + builder db.FastBatchInsertBuilder + table string +} + +// NewClaimableBalanceBatchInsertBuilder constructs a new ClaimableBalanceBatchInsertBuilder instance +func (q *Q) NewClaimableBalanceBatchInsertBuilder() ClaimableBalanceBatchInsertBuilder { + return &claimableBalanceBatchInsertBuilder{ + encodingBuffer: xdr.NewEncodingBuffer(), + builder: db.FastBatchInsertBuilder{}, + table: "claimable_balances", + } +} + +// Add adds a new claimable balance to the batch +func (i *claimableBalanceBatchInsertBuilder) Add(claimableBalance ClaimableBalance) error { + return i.builder.RowStruct(claimableBalance) +} + +// Exec inserts claimable balance rows to the database +func (i *claimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + return i.builder.Exec(ctx, session, i.table) +} + +func (i *claimableBalanceBatchInsertBuilder) Reset() error{ + i.builder.Reset() + return nil +} diff --git a/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go b/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go index a1a13d84db..60a2af8419 100644 --- a/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go @@ -10,33 +10,38 @@ import ( // ClaimableBalanceClaimantBatchInsertBuilder is used to insert transactions into the // history_transactions table type ClaimableBalanceClaimantBatchInsertBuilder interface { - Add(ctx context.Context, claimableBalanceClaimant ClaimableBalanceClaimant) error - Exec(ctx context.Context) error + Add(claimableBalanceClaimant ClaimableBalanceClaimant) error + Exec(ctx context.Context, session db.SessionInterface) error + Reset() error } -// ClaimableBalanceClaimantBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder +// ClaimableBalanceClaimantBatchInsertBuilder is a simple wrapper around db.FastBatchInsertBuilder type claimableBalanceClaimantBatchInsertBuilder struct { encodingBuffer *xdr.EncodingBuffer - builder db.BatchInsertBuilder + builder db.FastBatchInsertBuilder + table string } // NewClaimableBalanceClaimantBatchInsertBuilder constructs a new ClaimableBalanceClaimantBatchInsertBuilder instance -func (q *Q) NewClaimableBalanceClaimantBatchInsertBuilder(maxBatchSize int) ClaimableBalanceClaimantBatchInsertBuilder { +func (q *Q) NewClaimableBalanceClaimantBatchInsertBuilder() ClaimableBalanceClaimantBatchInsertBuilder { return &claimableBalanceClaimantBatchInsertBuilder{ encodingBuffer: xdr.NewEncodingBuffer(), - builder: db.BatchInsertBuilder{ - Table: q.GetTable("claimable_balance_claimants"), - MaxBatchSize: maxBatchSize, - Suffix: "ON CONFLICT (id, destination) DO UPDATE SET last_modified_ledger=EXCLUDED.last_modified_ledger", - }, + builder: db.FastBatchInsertBuilder{}, + table: "claimable_balance_claimants", } } -// Add adds a new transaction to the batch -func (i *claimableBalanceClaimantBatchInsertBuilder) Add(ctx context.Context, claimableBalanceClaimant ClaimableBalanceClaimant) error { - return i.builder.RowStruct(ctx, claimableBalanceClaimant) +// Add adds a new claimant for a claimable Balance to the batch +func (i *claimableBalanceClaimantBatchInsertBuilder) Add(claimableBalanceClaimant ClaimableBalanceClaimant) error { + return i.builder.RowStruct(claimableBalanceClaimant) } -func (i *claimableBalanceClaimantBatchInsertBuilder) Exec(ctx context.Context) error { - return i.builder.Exec(ctx) +// Exec flushes the entire batch into the database +func (i *claimableBalanceClaimantBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + return i.builder.Exec(ctx, session, i.table) +} + +func (i *claimableBalanceClaimantBatchInsertBuilder) Reset() error { + i.builder.Reset() + return nil } diff --git a/services/horizon/internal/db2/history/claimable_balances.go b/services/horizon/internal/db2/history/claimable_balances.go index 48722a11c3..e10536da4e 100644 --- a/services/horizon/internal/db2/history/claimable_balances.go +++ b/services/horizon/internal/db2/history/claimable_balances.go @@ -126,12 +126,12 @@ type Claimant struct { // QClaimableBalances defines claimable-balance-related related queries. type QClaimableBalances interface { - UpsertClaimableBalances(ctx context.Context, cb []ClaimableBalance) error RemoveClaimableBalances(ctx context.Context, ids []string) (int64, error) RemoveClaimableBalanceClaimants(ctx context.Context, ids []string) (int64, error) GetClaimableBalancesByID(ctx context.Context, ids []string) ([]ClaimableBalance, error) CountClaimableBalances(ctx context.Context) (int, error) - NewClaimableBalanceClaimantBatchInsertBuilder(maxBatchSize int) ClaimableBalanceClaimantBatchInsertBuilder + NewClaimableBalanceClaimantBatchInsertBuilder() ClaimableBalanceClaimantBatchInsertBuilder + NewClaimableBalanceBatchInsertBuilder() ClaimableBalanceBatchInsertBuilder GetClaimantsByClaimableBalances(ctx context.Context, ids []string) (map[string][]ClaimableBalanceClaimant, error) } @@ -171,36 +171,6 @@ func (q *Q) GetClaimantsByClaimableBalances(ctx context.Context, ids []string) ( return claimantsMap, err } -// UpsertClaimableBalances upserts a batch of claimable balances in the claimable_balances table. -// There's currently no limit of the number of offers this method can -// accept other than 2GB limit of the query string length what should be enough -// for each ledger with the current limits. -func (q *Q) UpsertClaimableBalances(ctx context.Context, cbs []ClaimableBalance) error { - var id, claimants, asset, amount, sponsor, lastModifiedLedger, flags []interface{} - - for _, cb := range cbs { - id = append(id, cb.BalanceID) - claimants = append(claimants, cb.Claimants) - asset = append(asset, cb.Asset) - amount = append(amount, cb.Amount) - sponsor = append(sponsor, cb.Sponsor) - lastModifiedLedger = append(lastModifiedLedger, cb.LastModifiedLedger) - flags = append(flags, cb.Flags) - } - - upsertFields := []upsertField{ - {"id", "text", id}, - {"claimants", "jsonb", claimants}, - {"asset", "text", asset}, - {"amount", "bigint", amount}, - {"sponsor", "text", sponsor}, - {"last_modified_ledger", "integer", lastModifiedLedger}, - {"flags", "int", flags}, - } - - return q.upsertRows(ctx, "claimable_balances", "id", upsertFields) -} - // RemoveClaimableBalances deletes claimable balances table. // Returns number of rows affected and error. func (q *Q) RemoveClaimableBalances(ctx context.Context, ids []string) (int64, error) { diff --git a/services/horizon/internal/db2/history/claimable_balances_test.go b/services/horizon/internal/db2/history/claimable_balances_test.go index 238b03da68..c59c958ed8 100644 --- a/services/horizon/internal/db2/history/claimable_balances_test.go +++ b/services/horizon/internal/db2/history/claimable_balances_test.go @@ -1,6 +1,7 @@ package history import ( + "database/sql" "fmt" "testing" @@ -15,6 +16,8 @@ func TestRemoveClaimableBalance(t *testing.T) { defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} + q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) + defer q.SessionInterface.Rollback() accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" asset := xdr.MustNewCreditAsset("USD", accountID) @@ -39,8 +42,9 @@ func TestRemoveClaimableBalance(t *testing.T) { Amount: 10, } - err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance}) - tt.Assert.NoError(err) + balanceBatchInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() + tt.Assert.NoError(balanceBatchInsertBuilder.Add(cBalance)) + tt.Assert.NoError(balanceBatchInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) r, err := q.FindClaimableBalanceByID(tt.Ctx, id) tt.Assert.NoError(err) @@ -63,6 +67,8 @@ func TestRemoveClaimableBalanceClaimants(t *testing.T) { defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} + q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) + defer q.SessionInterface.Rollback() accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" asset := xdr.MustNewCreditAsset("USD", accountID) @@ -87,20 +93,9 @@ func TestRemoveClaimableBalanceClaimants(t *testing.T) { Amount: 10, } - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(10) - - for _, claimant := range cBalance.Claimants { - claimant := ClaimableBalanceClaimant{ - BalanceID: cBalance.BalanceID, - Destination: claimant.Destination, - LastModifiedLedger: cBalance.LastModifiedLedger, - } - err = claimantsInsertBuilder.Add(tt.Ctx, claimant) - tt.Assert.NoError(err) - } - - err = claimantsInsertBuilder.Exec(tt.Ctx) - tt.Assert.NoError(err) + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() + tt.Assert.NoError(insertClaimants(claimantsInsertBuilder, cBalance)) + tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) removed, err := q.RemoveClaimableBalanceClaimants(tt.Ctx, []string{id}) tt.Assert.NoError(err) @@ -113,6 +108,9 @@ func TestFindClaimableBalancesByDestination(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} + q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) + defer q.Rollback() + dest1 := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" dest2 := "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H" @@ -138,19 +136,11 @@ func TestFindClaimableBalancesByDestination(t *testing.T) { Amount: 10, } - err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance}) - tt.Assert.NoError(err) + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(10) - for _, claimant := range cBalance.Claimants { - claimant := ClaimableBalanceClaimant{ - BalanceID: cBalance.BalanceID, - Destination: claimant.Destination, - LastModifiedLedger: cBalance.LastModifiedLedger, - } - err = claimantsInsertBuilder.Add(tt.Ctx, claimant) - tt.Assert.NoError(err) - } + tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) + tt.Assert.NoError(insertClaimants(claimantsInsertBuilder, cBalance)) balanceID = xdr.ClaimableBalanceId{ Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0, @@ -179,21 +169,11 @@ func TestFindClaimableBalancesByDestination(t *testing.T) { Amount: 10, } - err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance}) - tt.Assert.NoError(err) + tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) + tt.Assert.NoError(insertClaimants(claimantsInsertBuilder, cBalance)) - for _, claimant := range cBalance.Claimants { - claimant := ClaimableBalanceClaimant{ - BalanceID: cBalance.BalanceID, - Destination: claimant.Destination, - LastModifiedLedger: cBalance.LastModifiedLedger, - } - err = claimantsInsertBuilder.Add(tt.Ctx, claimant) - tt.Assert.NoError(err) - } - - err = claimantsInsertBuilder.Exec(tt.Ctx) - tt.Assert.NoError(err) + tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) query := ClaimableBalancesQuery{ PageQuery: db2.MustPageQuery("", false, "", 10), @@ -233,20 +213,19 @@ func TestFindClaimableBalancesByDestination(t *testing.T) { tt.Assert.Len(cbs, 1) } -func insertClaimants(q *Q, tt *test.T, cBalance ClaimableBalance) error { - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(10) +func insertClaimants(claimantsInsertBuilder ClaimableBalanceClaimantBatchInsertBuilder, cBalance ClaimableBalance) error { for _, claimant := range cBalance.Claimants { claimant := ClaimableBalanceClaimant{ BalanceID: cBalance.BalanceID, Destination: claimant.Destination, LastModifiedLedger: cBalance.LastModifiedLedger, } - err := claimantsInsertBuilder.Add(tt.Ctx, claimant) + err := claimantsInsertBuilder.Add(claimant) if err != nil { return err } } - return claimantsInsertBuilder.Exec(tt.Ctx) + return nil } type claimableBalanceQueryResult struct { @@ -279,6 +258,9 @@ func TestFindClaimableBalancesByDestinationWithLimit(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} + q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) + defer q.Rollback() + assetIssuer := "GA25GQLHJU3LPEJXEIAXK23AWEA5GWDUGRSHTQHDFT6HXHVMRULSQJUJ" asset1 := xdr.MustNewCreditAsset("ASSET1", assetIssuer) asset2 := xdr.MustNewCreditAsset("ASSET2", assetIssuer) @@ -318,8 +300,11 @@ func TestFindClaimableBalancesByDestinationWithLimit(t *testing.T) { LastModifiedLedger: 123, Amount: 10, } - err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance1}) - tt.Assert.NoError(err) + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() + + tt.Assert.NoError(balanceInsertBuilder.Add(cBalance1)) + tt.Assert.NoError(insertClaimants(claimantsInsertBuilder, cBalance1)) claimants2 := []Claimant{ { @@ -345,14 +330,12 @@ func TestFindClaimableBalancesByDestinationWithLimit(t *testing.T) { LastModifiedLedger: 456, Amount: 10, } - err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance2}) - tt.Assert.NoError(err) - err = insertClaimants(q, tt, cBalance1) - tt.Assert.NoError(err) + tt.Assert.NoError(balanceInsertBuilder.Add(cBalance2)) + tt.Assert.NoError(insertClaimants(claimantsInsertBuilder, cBalance2)) - err = insertClaimants(q, tt, cBalance2) - tt.Assert.NoError(err) + tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) pageQuery := db2.MustPageQuery("", false, "", 1) @@ -414,73 +397,15 @@ func TestFindClaimableBalancesByDestinationWithLimit(t *testing.T) { }) } -func TestUpdateClaimableBalance(t *testing.T) { - tt := test.Start(t) - defer tt.Finish() - test.ResetHorizonDB(t, tt.HorizonDB) - q := &Q{tt.HorizonSession()} - - accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" - lastModifiedLedgerSeq := xdr.Uint32(123) - asset := xdr.MustNewCreditAsset("USD", accountID) - balanceID := xdr.ClaimableBalanceId{ - Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0, - V0: &xdr.Hash{1, 2, 3}, - } - id, err := xdr.MarshalHex(balanceID) - tt.Assert.NoError(err) - cBalance := ClaimableBalance{ - BalanceID: id, - Claimants: []Claimant{ - { - Destination: accountID, - Predicate: xdr.ClaimPredicate{ - Type: xdr.ClaimPredicateTypeClaimPredicateUnconditional, - }, - }, - }, - Asset: asset, - LastModifiedLedger: 123, - Amount: 10, - } - - err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance}) - tt.Assert.NoError(err) - - // add sponsor - cBalance2 := ClaimableBalance{ - BalanceID: id, - Claimants: []Claimant{ - { - Destination: accountID, - Predicate: xdr.ClaimPredicate{ - Type: xdr.ClaimPredicateTypeClaimPredicateUnconditional, - }, - }, - }, - Asset: asset, - LastModifiedLedger: 123 + 1, - Amount: 10, - Sponsor: null.StringFrom("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), - } - - err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance2}) - tt.Assert.NoError(err) - - cbs := []ClaimableBalance{} - err = q.Select(tt.Ctx, &cbs, selectClaimableBalances) - tt.Assert.NoError(err) - tt.Assert.Len(cbs, 1) - tt.Assert.Equal("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML", cbs[0].Sponsor.String) - tt.Assert.Equal(uint32(lastModifiedLedgerSeq+1), cbs[0].LastModifiedLedger) -} - func TestFindClaimableBalance(t *testing.T) { tt := test.Start(t) defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} + q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) + defer q.SessionInterface.Rollback() + accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" asset := xdr.MustNewCreditAsset("USD", accountID) balanceID := xdr.ClaimableBalanceId{ @@ -504,11 +429,11 @@ func TestFindClaimableBalance(t *testing.T) { Amount: 10, } - err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance}) - tt.Assert.NoError(err) + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() + tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) + tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) cb, err := q.FindClaimableBalanceByID(tt.Ctx, id) - tt.Assert.NoError(err) tt.Assert.Equal(cBalance.BalanceID, cb.BalanceID) tt.Assert.Equal(cBalance.Asset, cb.Asset) @@ -525,6 +450,9 @@ func TestGetClaimableBalancesByID(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} + q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) + defer q.SessionInterface.Rollback() + accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" asset := xdr.MustNewCreditAsset("USD", accountID) balanceID := xdr.ClaimableBalanceId{ @@ -548,8 +476,9 @@ func TestGetClaimableBalancesByID(t *testing.T) { Amount: 10, } - err = q.UpsertClaimableBalances(tt.Ctx, []ClaimableBalance{cBalance}) - tt.Assert.NoError(err) + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() + tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) + tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) r, err := q.GetClaimableBalancesByID(tt.Ctx, []string{id}) tt.Assert.NoError(err) diff --git a/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go new file mode 100644 index 0000000000..10a0ad699e --- /dev/null +++ b/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go @@ -0,0 +1,27 @@ +package history + +import ( + "context" + "github.com/stellar/go/support/db" + + "github.com/stretchr/testify/mock" +) + +type MockClaimableBalanceBatchInsertBuilder struct { + mock.Mock +} + +func (m *MockClaimableBalanceBatchInsertBuilder) Add(claimableBalance ClaimableBalance) error { + a := m.Called(claimableBalance) + return a.Error(0) +} + +func (m *MockClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + a := m.Called(ctx, session) + return a.Error(0) +} + +func (m *MockClaimableBalanceBatchInsertBuilder) Reset() error { + a := m.Called() + return a.Error(0) +} diff --git a/services/horizon/internal/db2/history/mock_claimable_balance_claimant_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_claimable_balance_claimant_batch_insert_builder.go index 63e2788e53..2f70d289d1 100644 --- a/services/horizon/internal/db2/history/mock_claimable_balance_claimant_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/mock_claimable_balance_claimant_batch_insert_builder.go @@ -2,6 +2,7 @@ package history import ( "context" + "github.com/stellar/go/support/db" "github.com/stretchr/testify/mock" ) @@ -10,12 +11,17 @@ type MockClaimableBalanceClaimantBatchInsertBuilder struct { mock.Mock } -func (m *MockClaimableBalanceClaimantBatchInsertBuilder) Add(ctx context.Context, claimableBalanceClaimant ClaimableBalanceClaimant) error { - a := m.Called(ctx, claimableBalanceClaimant) +func (m *MockClaimableBalanceClaimantBatchInsertBuilder) Add(claimableBalanceClaimant ClaimableBalanceClaimant) error { + a := m.Called(claimableBalanceClaimant) return a.Error(0) } -func (m *MockClaimableBalanceClaimantBatchInsertBuilder) Exec(ctx context.Context) error { - a := m.Called(ctx) +func (m *MockClaimableBalanceClaimantBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + a := m.Called(ctx, session) + return a.Error(0) +} + +func (m *MockClaimableBalanceClaimantBatchInsertBuilder) Reset() error { + a := m.Called() return a.Error(0) } diff --git a/services/horizon/internal/db2/history/mock_q_claimable_balances.go b/services/horizon/internal/db2/history/mock_q_claimable_balances.go index 2493d9ebea..64b65cf1a3 100644 --- a/services/horizon/internal/db2/history/mock_q_claimable_balances.go +++ b/services/horizon/internal/db2/history/mock_q_claimable_balances.go @@ -21,11 +21,6 @@ func (m *MockQClaimableBalances) GetClaimableBalancesByID(ctx context.Context, i return a.Get(0).([]ClaimableBalance), a.Error(1) } -func (m *MockQClaimableBalances) UpsertClaimableBalances(ctx context.Context, cbs []ClaimableBalance) error { - a := m.Called(ctx, cbs) - return a.Error(0) -} - func (m *MockQClaimableBalances) RemoveClaimableBalances(ctx context.Context, ids []string) (int64, error) { a := m.Called(ctx, ids) return a.Get(0).(int64), a.Error(1) @@ -36,11 +31,16 @@ func (m *MockQClaimableBalances) RemoveClaimableBalanceClaimants(ctx context.Con return a.Get(0).(int64), a.Error(1) } -func (m *MockQClaimableBalances) NewClaimableBalanceClaimantBatchInsertBuilder(maxBatchSize int) ClaimableBalanceClaimantBatchInsertBuilder { - a := m.Called(maxBatchSize) +func (m *MockQClaimableBalances) NewClaimableBalanceClaimantBatchInsertBuilder() ClaimableBalanceClaimantBatchInsertBuilder { + a := m.Called() return a.Get(0).(ClaimableBalanceClaimantBatchInsertBuilder) } +func (m *MockQClaimableBalances) NewClaimableBalanceBatchInsertBuilder() ClaimableBalanceBatchInsertBuilder { + a := m.Called() + return a.Get(0).(ClaimableBalanceBatchInsertBuilder) +} + func (m *MockQClaimableBalances) GetClaimantsByClaimableBalances(ctx context.Context, ids []string) (map[string][]ClaimableBalanceClaimant, error) { a := m.Called(ctx, ids) return a.Get(0).(map[string][]ClaimableBalanceClaimant), a.Error(1) diff --git a/services/horizon/internal/ingest/processor_runner.go b/services/horizon/internal/ingest/processor_runner.go index cff8960c1d..2bc5657e21 100644 --- a/services/horizon/internal/ingest/processor_runner.go +++ b/services/horizon/internal/ingest/processor_runner.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "fmt" + "github.com/stellar/go/support/db" "time" "github.com/stellar/go/ingest" @@ -109,6 +110,7 @@ func (s *ProcessorRunner) DisableMemoryStatsLogging() { func buildChangeProcessor( historyQ history.IngestionQ, + session db.SessionInterface, changeStats *ingest.StatsChangeProcessor, source ingestionSource, ledgerSequence uint32, @@ -127,7 +129,7 @@ func buildChangeProcessor( processors.NewAssetStatsProcessor(historyQ, networkPassphrase, useLedgerCache), processors.NewSignersProcessor(historyQ, useLedgerCache), processors.NewTrustLinesProcessor(historyQ), - processors.NewClaimableBalancesChangeProcessor(historyQ), + processors.NewClaimableBalancesChangeProcessor(historyQ, session), processors.NewLiquidityPoolsChangeProcessor(historyQ, ledgerSequence), }) } @@ -239,6 +241,7 @@ func (s *ProcessorRunner) RunHistoryArchiveIngestion( changeStats := ingest.StatsChangeProcessor{} changeProcessor := buildChangeProcessor( s.historyQ, + s.session, &changeStats, historyArchiveSource, checkpointLedger, @@ -404,6 +407,10 @@ func (s *ProcessorRunner) RunAllProcessorsOnLedger(ledger xdr.LedgerCloseMeta) ( groupChangeProcessors := buildChangeProcessor( s.historyQ, +<<<<<<< HEAD +======= + s.session, +>>>>>>> 8f1836b1 (Use FastBatchInsertBuilder to insert to insert into claimable_balances and claimable_balance_claimants tables) &changeStatsProcessor, ledgerSource, ledger.LedgerSequence(), diff --git a/services/horizon/internal/ingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go index 4af1a5be11..dfbb9aed6a 100644 --- a/services/horizon/internal/ingest/processor_runner_test.go +++ b/services/horizon/internal/ingest/processor_runner_test.go @@ -49,9 +49,10 @@ func TestProcessorRunnerRunHistoryArchiveIngestionGenesis(t *testing.T) { q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize). Return(mockAccountSignersBatchInsertBuilder).Once() - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize). + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() - + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). + Return(&history.MockClaimableBalanceBatchInsertBuilder{}).Once() q.MockQAssetStats.On("InsertAssetStats", ctx, []history.ExpAssetStat{}, 100000). Return(nil) @@ -116,9 +117,10 @@ func TestProcessorRunnerRunHistoryArchiveIngestionHistoryArchive(t *testing.T) { mockAccountSignersBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize). Return(mockAccountSignersBatchInsertBuilder).Once() - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize). + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() - + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). + Return(&history.MockClaimableBalanceBatchInsertBuilder{}).Once() q.MockQAssetStats.On("InsertAssetStats", ctx, []history.ExpAssetStat{}, 100000). Return(nil) @@ -153,9 +155,10 @@ func TestProcessorRunnerRunHistoryArchiveIngestionProtocolVersionNotSupported(t defer mock.AssertExpectationsForObjects(t, mockAccountSignersBatchInsertBuilder) q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize). Return(mockAccountSignersBatchInsertBuilder).Once() - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize). + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() - + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). + Return(&history.MockClaimableBalanceBatchInsertBuilder{}).Once() q.MockQAssetStats.On("InsertAssetStats", ctx, []history.ExpAssetStat{}, 100000). Return(nil) @@ -186,8 +189,10 @@ func TestProcessorRunnerBuildChangeProcessor(t *testing.T) { // Twice = checking ledgerSource and historyArchiveSource q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize). Return(&history.MockAccountSignersBatchInsertBuilder{}).Twice() - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize). + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Twice() + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). + Return(&history.MockClaimableBalanceBatchInsertBuilder{}).Twice() runner := ProcessorRunner{ ctx: ctx, historyQ: q, @@ -216,7 +221,11 @@ func TestProcessorRunnerBuildChangeProcessor(t *testing.T) { filters: &MockFilters{}, } +<<<<<<< HEAD processor = buildChangeProcessor(runner.historyQ, stats, historyArchiveSource, 456, "") +======= + processor = buildChangeProcessor(runner.historyQ, &db.MockSession{}, stats, historyArchiveSource, 456, "") +>>>>>>> 8f1836b1 (Use FastBatchInsertBuilder to insert to insert into claimable_balances and claimable_balance_claimants tables) assert.IsType(t, &groupChangeProcessors{}, processor) assert.IsType(t, &statsChangeProcessor{}, processor.processors[0]) @@ -313,6 +322,12 @@ func TestProcessorRunnerWithFilterEnabled(t *testing.T) { q.MockQTransactions.On("NewTransactionFilteredTmpBatchInsertBuilder"). Return(mockTransactionsFilteredTmpBatchInsertBuilder) + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). + Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() + + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). + Return(&history.MockClaimableBalanceBatchInsertBuilder{}).Once() + q.On("DeleteTransactionsFilteredTmpOlderThan", ctx, mock.AnythingOfType("uint64")). Return(int64(0), nil) @@ -377,6 +392,12 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) { mockSession, ).Return(nil) + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). + Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() + + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). + Return(&history.MockClaimableBalanceBatchInsertBuilder{}).Once() + runner := ProcessorRunner{ ctx: ctx, config: config, diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go index a929927323..04579e47fc 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go @@ -6,20 +6,24 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/errors" + "github.com/stellar/go/support/db" "github.com/stellar/go/xdr" ) type ClaimableBalancesChangeProcessor struct { - encodingBuffer *xdr.EncodingBuffer - qClaimableBalances history.QClaimableBalances - cache *ingest.ChangeCompactor - claimantsInsertBuilder history.ClaimableBalanceClaimantBatchInsertBuilder + encodingBuffer *xdr.EncodingBuffer + qClaimableBalances history.QClaimableBalances + cache *ingest.ChangeCompactor + claimantsInsertBuilder history.ClaimableBalanceClaimantBatchInsertBuilder + claimableBalanceInsertBuilder history.ClaimableBalanceBatchInsertBuilder + session db.SessionInterface } -func NewClaimableBalancesChangeProcessor(Q history.QClaimableBalances) *ClaimableBalancesChangeProcessor { +func NewClaimableBalancesChangeProcessor(Q history.QClaimableBalances, session db.SessionInterface) *ClaimableBalancesChangeProcessor { p := &ClaimableBalancesChangeProcessor{ encodingBuffer: xdr.NewEncodingBuffer(), qClaimableBalances: Q, + session: session, } p.reset() return p @@ -27,7 +31,8 @@ func NewClaimableBalancesChangeProcessor(Q history.QClaimableBalances) *Claimabl func (p *ClaimableBalancesChangeProcessor) reset() { p.cache = ingest.NewChangeCompactor() - p.claimantsInsertBuilder = p.qClaimableBalances.NewClaimableBalanceClaimantBatchInsertBuilder(maxBatchSize) + p.claimantsInsertBuilder = p.qClaimableBalances.NewClaimableBalanceClaimantBatchInsertBuilder() + p.claimableBalanceInsertBuilder = p.qClaimableBalances.NewClaimableBalanceBatchInsertBuilder() } func (p *ClaimableBalancesChangeProcessor) ProcessChange(ctx context.Context, change ingest.Change) error { @@ -53,20 +58,19 @@ func (p *ClaimableBalancesChangeProcessor) ProcessChange(ctx context.Context, ch func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { var ( - cbsToUpsert []history.ClaimableBalance + cbsToInsert []history.ClaimableBalance cbIDsToDelete []string ) changes := p.cache.GetChanges() for _, change := range changes { - switch { - case change.Pre == nil && change.Post != nil: + if change.Post != nil { // Created row, err := p.ledgerEntryToRow(change.Post) if err != nil { return err } - cbsToUpsert = append(cbsToUpsert, row) - case change.Pre != nil && change.Post == nil: + cbsToInsert = append(cbsToInsert, row) + } else { // Removed cBalance := change.Pre.Data.MustClaimableBalance() id, err := p.encodingBuffer.MarshalHex(cBalance.BalanceId) @@ -74,39 +78,11 @@ func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { return err } cbIDsToDelete = append(cbIDsToDelete, id) - default: - // Updated - row, err := p.ledgerEntryToRow(change.Post) - if err != nil { - return err - } - cbsToUpsert = append(cbsToUpsert, row) } } - if len(cbsToUpsert) > 0 { - if err := p.qClaimableBalances.UpsertClaimableBalances(ctx, cbsToUpsert); err != nil { - return errors.Wrap(err, "error executing upsert") - } - - // Add ClaimableBalanceClaimants - for _, cb := range cbsToUpsert { - for _, claimant := range cb.Claimants { - claimant := history.ClaimableBalanceClaimant{ - BalanceID: cb.BalanceID, - Destination: claimant.Destination, - LastModifiedLedger: cb.LastModifiedLedger, - } - if err := p.claimantsInsertBuilder.Add(ctx, claimant); err != nil { - return errors.Wrap(err, "error adding to claimantsInsertBuilder") - } - } - } - - err := p.claimantsInsertBuilder.Exec(ctx) - if err != nil { - return errors.Wrap(err, "error executing claimantsInsertBuilder") - } + if err := p.InsertClaimableBalanceAndClaimants(ctx, cbsToInsert); err != nil { + return errors.Wrap(err, "error inserting claimable balance") } if len(cbIDsToDelete) > 0 { @@ -132,6 +108,45 @@ func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { return nil } +func (p *ClaimableBalancesChangeProcessor) InsertClaimableBalanceAndClaimants(ctx context.Context, claimableBalances []history.ClaimableBalance) error { + if len(claimableBalances) == 0 { + return nil + } + + defer p.claimantsInsertBuilder.Reset() + defer p.claimableBalanceInsertBuilder.Reset() + + for _, cb := range claimableBalances { + + if err := p.claimableBalanceInsertBuilder.Add(cb); err != nil { + return errors.Wrap(err, "error executing insert") + } + // Add claimants + for _, claimant := range cb.Claimants { + claimant := history.ClaimableBalanceClaimant{ + BalanceID: cb.BalanceID, + Destination: claimant.Destination, + LastModifiedLedger: cb.LastModifiedLedger, + } + + if err := p.claimantsInsertBuilder.Add(claimant); err != nil { + return errors.Wrap(err, "error adding to claimantsInsertBuilder") + } + } + } + + err := p.claimantsInsertBuilder.Exec(ctx, p.session) + if err != nil { + return errors.Wrap(err, "error executing claimableBalanceInsertBuilder") + } + + err = p.claimableBalanceInsertBuilder.Exec(ctx, p.session) + if err != nil { + return errors.Wrap(err, "error executing claimantsInsertBuilder") + } + return nil +} + func buildClaimants(claimants []xdr.Claimant) history.Claimants { hClaimants := history.Claimants{} for _, c := range claimants { diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go index 70cd43bba5..e4fa4c2d2d 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go @@ -11,6 +11,7 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" + "github.com/stellar/go/support/db" "github.com/stellar/go/xdr" ) @@ -20,21 +21,32 @@ func TestClaimableBalancesChangeProcessorTestSuiteState(t *testing.T) { type ClaimableBalancesChangeProcessorTestSuiteState struct { suite.Suite - ctx context.Context - processor *ClaimableBalancesChangeProcessor - mockQ *history.MockQClaimableBalances - mockBatchInsertBuilder *history.MockClaimableBalanceClaimantBatchInsertBuilder + ctx context.Context + processor *ClaimableBalancesChangeProcessor + mockQ *history.MockQClaimableBalances + mockClaimantsBatchInsertBuilder *history.MockClaimableBalanceClaimantBatchInsertBuilder + mockClaimableBalanceBatchInsertBuilder *history.MockClaimableBalanceBatchInsertBuilder + session db.SessionInterface } func (s *ClaimableBalancesChangeProcessorTestSuiteState) SetupTest() { s.ctx = context.Background() - s.mockBatchInsertBuilder = &history.MockClaimableBalanceClaimantBatchInsertBuilder{} + s.mockClaimantsBatchInsertBuilder = &history.MockClaimableBalanceClaimantBatchInsertBuilder{} + s.mockClaimableBalanceBatchInsertBuilder = &history.MockClaimableBalanceBatchInsertBuilder{} + s.mockQ = &history.MockQClaimableBalances{} + s.session = &db.MockSession{} + s.mockQ. + On("NewClaimableBalanceClaimantBatchInsertBuilder"). + Return(s.mockClaimantsBatchInsertBuilder).Once() s.mockQ. - On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize). - Return(s.mockBatchInsertBuilder).Once() + On("NewClaimableBalanceBatchInsertBuilder"). + Return(s.mockClaimableBalanceBatchInsertBuilder).Once() + + s.mockClaimantsBatchInsertBuilder.On("Reset").Return(nil).Once() + s.mockClaimableBalanceBatchInsertBuilder.On("Reset").Return(nil).Once() - s.processor = NewClaimableBalancesChangeProcessor(s.mockQ) + s.processor = NewClaimableBalancesChangeProcessor(s.mockQ, s.session) } func (s *ClaimableBalancesChangeProcessorTestSuiteState) TearDownTest() { @@ -67,27 +79,27 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteState) TestCreatesClaimableBal } id, err := xdr.MarshalHex(balanceID) s.Assert().NoError(err) - s.mockQ.On("UpsertClaimableBalances", s.ctx, []history.ClaimableBalance{ - { - BalanceID: id, - Claimants: []history.Claimant{ - { - Destination: "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML", - }, + s.mockClaimableBalanceBatchInsertBuilder.On("Add", history.ClaimableBalance{ + BalanceID: id, + Claimants: []history.Claimant{ + { + Destination: "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML", }, - Asset: cBalance.Asset, - Amount: cBalance.Amount, - LastModifiedLedger: uint32(lastModifiedLedgerSeq), }, + Asset: cBalance.Asset, + Amount: cBalance.Amount, + LastModifiedLedger: uint32(lastModifiedLedgerSeq), }).Return(nil).Once() - s.mockBatchInsertBuilder.On("Add", s.ctx, history.ClaimableBalanceClaimant{ + s.mockClaimableBalanceBatchInsertBuilder.On("Exec", s.ctx, s.session).Return(nil).Once() + + s.mockClaimantsBatchInsertBuilder.On("Add", history.ClaimableBalanceClaimant{ BalanceID: id, Destination: "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML", LastModifiedLedger: uint32(lastModifiedLedgerSeq), }).Return(nil).Once() - s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() + s.mockClaimantsBatchInsertBuilder.On("Exec", s.ctx, s.session).Return(nil).Once() err = s.processor.ProcessChange(s.ctx, ingest.Change{ Type: xdr.LedgerEntryTypeClaimableBalance, @@ -109,25 +121,39 @@ func TestClaimableBalancesChangeProcessorTestSuiteLedger(t *testing.T) { type ClaimableBalancesChangeProcessorTestSuiteLedger struct { suite.Suite - ctx context.Context - processor *ClaimableBalancesChangeProcessor - mockQ *history.MockQClaimableBalances - mockBatchInsertBuilder *history.MockClaimableBalanceClaimantBatchInsertBuilder + ctx context.Context + processor *ClaimableBalancesChangeProcessor + mockQ *history.MockQClaimableBalances + mockClaimantsBatchInsertBuilder *history.MockClaimableBalanceClaimantBatchInsertBuilder + mockClaimableBalanceBatchInsertBuilder *history.MockClaimableBalanceBatchInsertBuilder + session db.SessionInterface } func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) SetupTest() { s.ctx = context.Background() - s.mockBatchInsertBuilder = &history.MockClaimableBalanceClaimantBatchInsertBuilder{} + s.mockClaimantsBatchInsertBuilder = &history.MockClaimableBalanceClaimantBatchInsertBuilder{} + s.mockClaimableBalanceBatchInsertBuilder = &history.MockClaimableBalanceBatchInsertBuilder{} s.mockQ = &history.MockQClaimableBalances{} + s.session = &db.MockSession{} + s.mockQ. + On("NewClaimableBalanceClaimantBatchInsertBuilder"). + Return(s.mockClaimantsBatchInsertBuilder).Twice() s.mockQ. - On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize). - Return(s.mockBatchInsertBuilder).Once() + On("NewClaimableBalanceBatchInsertBuilder"). + Return(s.mockClaimableBalanceBatchInsertBuilder).Twice() + + s.mockClaimantsBatchInsertBuilder.On("Reset").Return(nil).Once() + s.mockClaimableBalanceBatchInsertBuilder.On("Reset").Return(nil).Once() - s.processor = NewClaimableBalancesChangeProcessor(s.mockQ) + s.mockClaimantsBatchInsertBuilder.On("Exec", s.ctx, s.session).Return(nil).Once() + s.mockClaimableBalanceBatchInsertBuilder.On("Exec", s.ctx, s.session).Return(nil).Once() + + s.processor = NewClaimableBalancesChangeProcessor(s.mockQ, s.session) } func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) TearDownTest() { s.Assert().NoError(s.processor.Commit(s.ctx)) + s.processor.reset() s.mockQ.AssertExpectations(s.T()) } @@ -160,9 +186,23 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) TestNewClaimableBalanc }, }, } - s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() - err := s.processor.ProcessChange(s.ctx, ingest.Change{ + id, err := xdr.MarshalHex(balanceID) + s.Assert().NoError(err) + + // We use LedgerEntryChangesCache so all changes are squashed + s.mockClaimableBalanceBatchInsertBuilder.On( + "Add", + history.ClaimableBalance{ + BalanceID: id, + Claimants: []history.Claimant{}, + Asset: cBalance.Asset, + Amount: cBalance.Amount, + LastModifiedLedger: uint32(lastModifiedLedgerSeq), + }, + ).Return(nil).Once() + + err = s.processor.ProcessChange(s.ctx, ingest.Change{ Type: xdr.LedgerEntryTypeClaimableBalance, Pre: nil, Post: &entry, @@ -192,89 +232,16 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) TestNewClaimableBalanc }) s.Assert().NoError(err) - id, err := xdr.MarshalHex(balanceID) - s.Assert().NoError(err) // We use LedgerEntryChangesCache so all changes are squashed - s.mockQ.On( - "UpsertClaimableBalances", - s.ctx, - []history.ClaimableBalance{ - { - BalanceID: id, - Claimants: []history.Claimant{}, - Asset: cBalance.Asset, - Amount: cBalance.Amount, - LastModifiedLedger: uint32(lastModifiedLedgerSeq), - Sponsor: null.StringFrom("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), - }, - }, - ).Return(nil).Once() -} - -func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) TestUpdateClaimableBalance() { - balanceID := xdr.ClaimableBalanceId{ - Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0, - V0: &xdr.Hash{1, 2, 3}, - } - cBalance := xdr.ClaimableBalanceEntry{ - BalanceId: balanceID, - Claimants: []xdr.Claimant{}, - Asset: xdr.MustNewCreditAsset("USD", "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), - Amount: 10, - } - lastModifiedLedgerSeq := xdr.Uint32(123) - - pre := xdr.LedgerEntry{ - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeClaimableBalance, - ClaimableBalance: &cBalance, - }, - LastModifiedLedgerSeq: lastModifiedLedgerSeq - 1, - Ext: xdr.LedgerEntryExt{ - V: 1, - V1: &xdr.LedgerEntryExtensionV1{ - SponsoringId: nil, - }, - }, - } - - // add sponsor - updated := xdr.LedgerEntry{ - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeClaimableBalance, - ClaimableBalance: &cBalance, - }, - LastModifiedLedgerSeq: lastModifiedLedgerSeq, - Ext: xdr.LedgerEntryExt{ - V: 1, - V1: &xdr.LedgerEntryExtensionV1{ - SponsoringId: xdr.MustAddressPtr("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), - }, - }, - } - s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() - - err := s.processor.ProcessChange(s.ctx, ingest.Change{ - Type: xdr.LedgerEntryTypeClaimableBalance, - Pre: &pre, - Post: &updated, - }) - s.Assert().NoError(err) - - id, err := xdr.MarshalHex(balanceID) - s.Assert().NoError(err) - s.mockQ.On( - "UpsertClaimableBalances", - s.ctx, - []history.ClaimableBalance{ - { - BalanceID: id, - Claimants: []history.Claimant{}, - Asset: cBalance.Asset, - Amount: cBalance.Amount, - LastModifiedLedger: uint32(lastModifiedLedgerSeq), - Sponsor: null.StringFrom("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), - }, + s.mockClaimableBalanceBatchInsertBuilder.On( + "Add", + history.ClaimableBalance{ + BalanceID: id, + Claimants: []history.Claimant{}, + Asset: cBalance.Asset, + Amount: cBalance.Amount, + LastModifiedLedger: uint32(lastModifiedLedgerSeq), + Sponsor: null.StringFrom("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), }, ).Return(nil).Once() } diff --git a/services/horizon/internal/ingest/verify_test.go b/services/horizon/internal/ingest/verify_test.go index 47d29cb6ee..9e2ddad72c 100644 --- a/services/horizon/internal/ingest/verify_test.go +++ b/services/horizon/internal/ingest/verify_test.go @@ -271,6 +271,9 @@ func TestStateVerifierLockBusy(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{&db.Session{DB: tt.HorizonDB}} + q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) + defer q.SessionInterface.Rollback() + checkpointLedger := uint32(63) changeProcessor := buildChangeProcessor(q, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") @@ -324,8 +327,15 @@ func TestStateVerifier(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{&db.Session{DB: tt.HorizonDB}} + q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) + defer q.SessionInterface.Rollback() + checkpointLedger := uint32(63) +<<<<<<< HEAD changeProcessor := buildChangeProcessor(q, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") +======= + changeProcessor := buildChangeProcessor(q, q.SessionInterface, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") +>>>>>>> 8f1836b1 (Use FastBatchInsertBuilder to insert to insert into claimable_balances and claimable_balance_claimants tables) mockChangeReader := &ingest.MockChangeReader{} gen := randxdr.NewGenerator() @@ -354,6 +364,8 @@ func TestStateVerifier(t *testing.T) { tt.Assert.NoError(changeProcessor.Commit(tt.Ctx)) tt.Assert.Equal(len(xdr.LedgerEntryTypeMap), len(coverage)) + q.SessionInterface.Commit() + q.UpdateLastLedgerIngest(tt.Ctx, checkpointLedger) mockChangeReader.On("Read").Return(ingest.Change{}, io.EOF).Twice() From 4241c30ef5ee2f8c53c82d212b17903cecc8f048 Mon Sep 17 00:00:00 2001 From: Urvi Date: Thu, 19 Oct 2023 09:23:24 -0700 Subject: [PATCH 02/14] Fix formatting --- .../db2/history/claimable_balance_batch_insert_builder.go | 2 +- .../ingest/processors/claimable_balances_change_processor.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go b/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go index 08ab005eb8..300bb314ad 100644 --- a/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go @@ -41,7 +41,7 @@ func (i *claimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session d return i.builder.Exec(ctx, session, i.table) } -func (i *claimableBalanceBatchInsertBuilder) Reset() error{ +func (i *claimableBalanceBatchInsertBuilder) Reset() error { i.builder.Reset() return nil } diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go index 04579e47fc..699694c28e 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go @@ -5,8 +5,8 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/support/errors" "github.com/stellar/go/support/db" + "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) From 7892ec01c9a79d0bb3fda405548a3d0ad173c626 Mon Sep 17 00:00:00 2001 From: Urvi Date: Fri, 20 Oct 2023 14:37:06 -0700 Subject: [PATCH 03/14] Convert byte[] to string to bypass encoding --- services/horizon/internal/db2/history/claimable_balances.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/services/horizon/internal/db2/history/claimable_balances.go b/services/horizon/internal/db2/history/claimable_balances.go index e10536da4e..a584e84080 100644 --- a/services/horizon/internal/db2/history/claimable_balances.go +++ b/services/horizon/internal/db2/history/claimable_balances.go @@ -107,7 +107,8 @@ type ClaimableBalance struct { type Claimants []Claimant func (c Claimants) Value() (driver.Value, error) { - return json.Marshal(c) + val, err := json.Marshal(c) + return string(val), err } func (c *Claimants) Scan(value interface{}) error { From a22f712f116d40d9e914fc5bd2a5cc9f998f4b26 Mon Sep 17 00:00:00 2001 From: Urvi Date: Fri, 20 Oct 2023 15:44:57 -0700 Subject: [PATCH 04/14] Add a comment --- services/horizon/internal/db2/history/claimable_balances.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/horizon/internal/db2/history/claimable_balances.go b/services/horizon/internal/db2/history/claimable_balances.go index a584e84080..1b9254991d 100644 --- a/services/horizon/internal/db2/history/claimable_balances.go +++ b/services/horizon/internal/db2/history/claimable_balances.go @@ -107,6 +107,11 @@ type ClaimableBalance struct { type Claimants []Claimant func (c Claimants) Value() (driver.Value, error) { + // Convert the byte array into a string as a workaround to bypass buggy encoding in the pq driver + // (More info about this bug here https://github.com/stellar/go/issues/5086#issuecomment-1773215436). + // By doing so, the data will be written as a string rather than hex encoded bytes. + // According to this https://www.postgresql.org/docs/current/datatype-json.html + // this may even improve write performance due to reduced conversion overhead. val, err := json.Marshal(c) return string(val), err } From 285b1723a78c289dd774cf72694f476edf62857c Mon Sep 17 00:00:00 2001 From: Urvi Date: Mon, 30 Oct 2023 18:18:08 -0700 Subject: [PATCH 05/14] Fix formatting --- .../processors/claimable_balances_change_processor_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go index e4fa4c2d2d..f7ca1c7b11 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go @@ -13,6 +13,7 @@ import ( "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/db" "github.com/stellar/go/xdr" + "github.com/stretchr/testify/suite" ) func TestClaimableBalancesChangeProcessorTestSuiteState(t *testing.T) { From 73e80b20c8ed78f4e1caa28668a3c7ceae9e7658 Mon Sep 17 00:00:00 2001 From: Urvi Date: Mon, 30 Oct 2023 20:21:55 -0700 Subject: [PATCH 06/14] Fix linter errors --- .../actions/claimable_balance_test.go | 13 ++++++++-- .../claimable_balance_batch_insert_builder.go | 6 ++--- ...e_balance_claimant_batch_insert_builder.go | 6 ++--- .../db2/history/claimable_balances_test.go | 26 ++++++++++++++----- ..._claimable_balance_batch_insert_builder.go | 5 ++-- ...e_balance_claimant_batch_insert_builder.go | 5 ++-- .../horizon/internal/ingest/verify_test.go | 16 ++++++------ 7 files changed, 49 insertions(+), 28 deletions(-) diff --git a/services/horizon/internal/actions/claimable_balance_test.go b/services/horizon/internal/actions/claimable_balance_test.go index 05ff7532c2..5a3ce4c7ba 100644 --- a/services/horizon/internal/actions/claimable_balance_test.go +++ b/services/horizon/internal/actions/claimable_balance_test.go @@ -21,8 +21,10 @@ func TestGetClaimableBalanceByID(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{tt.HorizonSession()} - q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) - defer q.SessionInterface.Rollback() + tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + defer func() { + _ = q.SessionInterface.Rollback() + }() accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" asset := xdr.MustNewCreditAsset("USD", accountID) @@ -154,8 +156,15 @@ func TestGetClaimableBalances(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{tt.HorizonSession()} +<<<<<<< HEAD q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) defer q.SessionInterface.Rollback() +======= + tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + defer func() { + _ = q.SessionInterface.Rollback() + }() +>>>>>>> c7bd7ac1 (Fix linter errors) entriesMeta := []struct { id xdr.Hash diff --git a/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go b/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go index 300bb314ad..0b1621014f 100644 --- a/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go @@ -12,7 +12,7 @@ import ( type ClaimableBalanceBatchInsertBuilder interface { Add(claimableBalance ClaimableBalance) error Exec(ctx context.Context, session db.SessionInterface) error - Reset() error + Reset() } // ClaimableBalanceBatchInsertBuilder is a simple wrapper around db.FastBatchInsertBuilder @@ -41,7 +41,7 @@ func (i *claimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session d return i.builder.Exec(ctx, session, i.table) } -func (i *claimableBalanceBatchInsertBuilder) Reset() error { +// Reset clears out the current batch of claimable balances +func (i *claimableBalanceBatchInsertBuilder) Reset() { i.builder.Reset() - return nil } diff --git a/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go b/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go index 60a2af8419..458ade97e4 100644 --- a/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go @@ -12,7 +12,7 @@ import ( type ClaimableBalanceClaimantBatchInsertBuilder interface { Add(claimableBalanceClaimant ClaimableBalanceClaimant) error Exec(ctx context.Context, session db.SessionInterface) error - Reset() error + Reset() } // ClaimableBalanceClaimantBatchInsertBuilder is a simple wrapper around db.FastBatchInsertBuilder @@ -41,7 +41,7 @@ func (i *claimableBalanceClaimantBatchInsertBuilder) Exec(ctx context.Context, s return i.builder.Exec(ctx, session, i.table) } -func (i *claimableBalanceClaimantBatchInsertBuilder) Reset() error { +// Reset clears out the current batch of claimants +func (i *claimableBalanceClaimantBatchInsertBuilder) Reset() { i.builder.Reset() - return nil } diff --git a/services/horizon/internal/db2/history/claimable_balances_test.go b/services/horizon/internal/db2/history/claimable_balances_test.go index c59c958ed8..cc03867e91 100644 --- a/services/horizon/internal/db2/history/claimable_balances_test.go +++ b/services/horizon/internal/db2/history/claimable_balances_test.go @@ -16,8 +16,10 @@ func TestRemoveClaimableBalance(t *testing.T) { defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) - defer q.SessionInterface.Rollback() + tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + defer func() { + _ = q.SessionInterface.Rollback() + }() accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" asset := xdr.MustNewCreditAsset("USD", accountID) @@ -67,8 +69,10 @@ func TestRemoveClaimableBalanceClaimants(t *testing.T) { defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) - defer q.SessionInterface.Rollback() + tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + defer func() { + _ = q.SessionInterface.Rollback() + }() accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" asset := xdr.MustNewCreditAsset("USD", accountID) @@ -108,8 +112,10 @@ func TestFindClaimableBalancesByDestination(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) - defer q.Rollback() + tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + defer func() { + _ = q.SessionInterface.Rollback() + }() dest1 := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" dest2 := "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H" @@ -405,6 +411,10 @@ func TestFindClaimableBalance(t *testing.T) { q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) defer q.SessionInterface.Rollback() + tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + defer func() { + _ = q.SessionInterface.Rollback() + }() accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" asset := xdr.MustNewCreditAsset("USD", accountID) @@ -452,6 +462,10 @@ func TestGetClaimableBalancesByID(t *testing.T) { q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) defer q.SessionInterface.Rollback() + tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + defer func() { + _ = q.SessionInterface.Rollback() + }() accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" asset := xdr.MustNewCreditAsset("USD", accountID) diff --git a/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go index 10a0ad699e..fbd73bb6cd 100644 --- a/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go @@ -21,7 +21,6 @@ func (m *MockClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context, sessi return a.Error(0) } -func (m *MockClaimableBalanceBatchInsertBuilder) Reset() error { - a := m.Called() - return a.Error(0) +func (m *MockClaimableBalanceBatchInsertBuilder) Reset() { + m.Called() } diff --git a/services/horizon/internal/db2/history/mock_claimable_balance_claimant_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_claimable_balance_claimant_batch_insert_builder.go index 2f70d289d1..785df6fd16 100644 --- a/services/horizon/internal/db2/history/mock_claimable_balance_claimant_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/mock_claimable_balance_claimant_batch_insert_builder.go @@ -21,7 +21,6 @@ func (m *MockClaimableBalanceClaimantBatchInsertBuilder) Exec(ctx context.Contex return a.Error(0) } -func (m *MockClaimableBalanceClaimantBatchInsertBuilder) Reset() error { - a := m.Called() - return a.Error(0) +func (m *MockClaimableBalanceClaimantBatchInsertBuilder) Reset() { + m.Called() } diff --git a/services/horizon/internal/ingest/verify_test.go b/services/horizon/internal/ingest/verify_test.go index 9e2ddad72c..53441bf00d 100644 --- a/services/horizon/internal/ingest/verify_test.go +++ b/services/horizon/internal/ingest/verify_test.go @@ -271,8 +271,10 @@ func TestStateVerifierLockBusy(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{&db.Session{DB: tt.HorizonDB}} - q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) - defer q.SessionInterface.Rollback() + tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + defer func() { + _ = q.SessionInterface.Rollback() + }() checkpointLedger := uint32(63) changeProcessor := buildChangeProcessor(q, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") @@ -327,15 +329,13 @@ func TestStateVerifier(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{&db.Session{DB: tt.HorizonDB}} - q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) - defer q.SessionInterface.Rollback() + tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + defer func() { + _ = q.SessionInterface.Rollback() + }() checkpointLedger := uint32(63) -<<<<<<< HEAD - changeProcessor := buildChangeProcessor(q, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") -======= changeProcessor := buildChangeProcessor(q, q.SessionInterface, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") ->>>>>>> 8f1836b1 (Use FastBatchInsertBuilder to insert to insert into claimable_balances and claimable_balance_claimants tables) mockChangeReader := &ingest.MockChangeReader{} gen := randxdr.NewGenerator() From 6fe1e5a1df333c2d6e1aec1e882a705aff5a3862 Mon Sep 17 00:00:00 2001 From: Urvi Date: Thu, 2 Nov 2023 12:51:39 -0700 Subject: [PATCH 07/14] Resolve Merge issues --- .../actions/claimable_balance_test.go | 22 +++++----- .../claimable_balance_batch_insert_builder.go | 2 +- ...e_balance_claimant_batch_insert_builder.go | 8 ++-- .../db2/history/claimable_balances.go | 2 - .../db2/history/claimable_balances_test.go | 16 +++----- .../internal/ingest/processor_runner.go | 4 -- .../internal/ingest/processor_runner_test.go | 6 +-- .../claimable_balances_change_processor.go | 23 ++++++----- ...laimable_balances_change_processor_test.go | 40 +------------------ .../horizon/internal/ingest/verify_test.go | 8 ++-- 10 files changed, 42 insertions(+), 89 deletions(-) diff --git a/services/horizon/internal/actions/claimable_balance_test.go b/services/horizon/internal/actions/claimable_balance_test.go index 5a3ce4c7ba..add2dac150 100644 --- a/services/horizon/internal/actions/claimable_balance_test.go +++ b/services/horizon/internal/actions/claimable_balance_test.go @@ -21,7 +21,7 @@ func TestGetClaimableBalanceByID(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{tt.HorizonSession()} - tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { _ = q.SessionInterface.Rollback() }() @@ -156,15 +156,10 @@ func TestGetClaimableBalances(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{tt.HorizonSession()} -<<<<<<< HEAD - q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) - defer q.SessionInterface.Rollback() -======= - tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { _ = q.SessionInterface.Rollback() }() ->>>>>>> c7bd7ac1 (Fix linter errors) entriesMeta := []struct { id xdr.Hash @@ -206,6 +201,7 @@ func TestGetClaimableBalances(t *testing.T) { } balanceInsertbuilder := q.NewClaimableBalanceBatchInsertBuilder() + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() for _, cBalance := range hCBs { @@ -301,6 +297,7 @@ func TestGetClaimableBalances(t *testing.T) { tt.Assert.NoError(err) tt.Assert.Len(response, 0) + // new claimable balances are ingested, they should appear in the next pages balanceInsertbuilder = q.NewClaimableBalanceBatchInsertBuilder() claimantsInsertBuilder = q.NewClaimableBalanceClaimantBatchInsertBuilder() @@ -324,13 +321,12 @@ func TestGetClaimableBalances(t *testing.T) { }, } - hCBs = nil for _, e := range entriesMeta { entry := buildClaimableBalance(tt, e.id, e.accountID, e.ledger, e.asset) hCBs = append(hCBs, entry) } - for _, cBalance := range hCBs { + for _, cBalance := range hCBs[4:] { tt.Assert.NoError(balanceInsertbuilder.Add(cBalance)) for _, claimant := range cBalance.Claimants { @@ -370,7 +366,7 @@ func TestGetClaimableBalances(t *testing.T) { tt.Assert.Len(response, 2) // response should be the first 2 elements of entries - for i, entry := range hCBs { + for i, entry := range hCBs[4:] { tt.Assert.Equal(entry.BalanceID, response[i].(protocol.ClaimableBalance).BalanceID) } @@ -401,7 +397,9 @@ func TestGetClaimableBalances(t *testing.T) { tt.Assert.NoError(err) tt.Assert.Len(response, 2) - tt.Assert.Equal(hCBs[1].BalanceID, response[0].(protocol.ClaimableBalance).BalanceID) + tt.Assert.Equal(hCBs[5].BalanceID, response[0].(protocol.ClaimableBalance).BalanceID) + + tt.Assert.Equal(hCBs[4].BalanceID, response[1].(protocol.ClaimableBalance).BalanceID) response, err = handler.GetResourcePage(httptest.NewRecorder(), makeRequest( t, @@ -417,6 +415,8 @@ func TestGetClaimableBalances(t *testing.T) { tt.Assert.NoError(err) tt.Assert.Len(response, 1) + tt.Assert.Equal(hCBs[0].BalanceID, response[0].(protocol.ClaimableBalance).BalanceID) + // filter by asset response, err = handler.GetResourcePage(httptest.NewRecorder(), makeRequest( t, diff --git a/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go b/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go index 0b1621014f..93a6abd9de 100644 --- a/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go @@ -36,7 +36,7 @@ func (i *claimableBalanceBatchInsertBuilder) Add(claimableBalance ClaimableBalan return i.builder.RowStruct(claimableBalance) } -// Exec inserts claimable balance rows to the database +// Exec writes the batch of claimable balances to the database. func (i *claimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { return i.builder.Exec(ctx, session, i.table) } diff --git a/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go b/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go index 458ade97e4..37eaa67e35 100644 --- a/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go @@ -7,8 +7,8 @@ import ( "github.com/stellar/go/xdr" ) -// ClaimableBalanceClaimantBatchInsertBuilder is used to insert transactions into the -// history_transactions table +// ClaimableBalanceClaimantBatchInsertBuilder is used to insert claimants into the +// claimable_balance_claimants table type ClaimableBalanceClaimantBatchInsertBuilder interface { Add(claimableBalanceClaimant ClaimableBalanceClaimant) error Exec(ctx context.Context, session db.SessionInterface) error @@ -31,12 +31,12 @@ func (q *Q) NewClaimableBalanceClaimantBatchInsertBuilder() ClaimableBalanceClai } } -// Add adds a new claimant for a claimable Balance to the batch +// Add adds a new claimant to the batch func (i *claimableBalanceClaimantBatchInsertBuilder) Add(claimableBalanceClaimant ClaimableBalanceClaimant) error { return i.builder.RowStruct(claimableBalanceClaimant) } -// Exec flushes the entire batch into the database +// Exec writes the batch of claimants to the database. func (i *claimableBalanceClaimantBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { return i.builder.Exec(ctx, session, i.table) } diff --git a/services/horizon/internal/db2/history/claimable_balances.go b/services/horizon/internal/db2/history/claimable_balances.go index 1b9254991d..5490bef11c 100644 --- a/services/horizon/internal/db2/history/claimable_balances.go +++ b/services/horizon/internal/db2/history/claimable_balances.go @@ -110,8 +110,6 @@ func (c Claimants) Value() (driver.Value, error) { // Convert the byte array into a string as a workaround to bypass buggy encoding in the pq driver // (More info about this bug here https://github.com/stellar/go/issues/5086#issuecomment-1773215436). // By doing so, the data will be written as a string rather than hex encoded bytes. - // According to this https://www.postgresql.org/docs/current/datatype-json.html - // this may even improve write performance due to reduced conversion overhead. val, err := json.Marshal(c) return string(val), err } diff --git a/services/horizon/internal/db2/history/claimable_balances_test.go b/services/horizon/internal/db2/history/claimable_balances_test.go index cc03867e91..f38674866d 100644 --- a/services/horizon/internal/db2/history/claimable_balances_test.go +++ b/services/horizon/internal/db2/history/claimable_balances_test.go @@ -3,9 +3,9 @@ package history import ( "database/sql" "fmt" + "github.com/guregu/null" "testing" - "github.com/guregu/null" "github.com/stellar/go/services/horizon/internal/db2" "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/xdr" @@ -16,7 +16,7 @@ func TestRemoveClaimableBalance(t *testing.T) { defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { _ = q.SessionInterface.Rollback() }() @@ -69,7 +69,7 @@ func TestRemoveClaimableBalanceClaimants(t *testing.T) { defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { _ = q.SessionInterface.Rollback() }() @@ -112,7 +112,7 @@ func TestFindClaimableBalancesByDestination(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { _ = q.SessionInterface.Rollback() }() @@ -409,9 +409,7 @@ func TestFindClaimableBalance(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) - defer q.SessionInterface.Rollback() - tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { _ = q.SessionInterface.Rollback() }() @@ -460,9 +458,7 @@ func TestGetClaimableBalancesByID(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) - defer q.SessionInterface.Rollback() - tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { _ = q.SessionInterface.Rollback() }() diff --git a/services/horizon/internal/ingest/processor_runner.go b/services/horizon/internal/ingest/processor_runner.go index 2bc5657e21..764ff9cb5c 100644 --- a/services/horizon/internal/ingest/processor_runner.go +++ b/services/horizon/internal/ingest/processor_runner.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "fmt" - "github.com/stellar/go/support/db" "time" "github.com/stellar/go/ingest" @@ -407,10 +406,7 @@ func (s *ProcessorRunner) RunAllProcessorsOnLedger(ledger xdr.LedgerCloseMeta) ( groupChangeProcessors := buildChangeProcessor( s.historyQ, -<<<<<<< HEAD -======= s.session, ->>>>>>> 8f1836b1 (Use FastBatchInsertBuilder to insert to insert into claimable_balances and claimable_balance_claimants tables) &changeStatsProcessor, ledgerSource, ledger.LedgerSequence(), diff --git a/services/horizon/internal/ingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go index dfbb9aed6a..6ca0f7319e 100644 --- a/services/horizon/internal/ingest/processor_runner_test.go +++ b/services/horizon/internal/ingest/processor_runner_test.go @@ -200,7 +200,7 @@ func TestProcessorRunnerBuildChangeProcessor(t *testing.T) { } stats := &ingest.StatsChangeProcessor{} - processor := buildChangeProcessor(runner.historyQ, stats, ledgerSource, 123, "") + processor := buildChangeProcessor(runner.historyQ, &db.MockSession{}, stats, ledgerSource, 123, "") assert.IsType(t, &groupChangeProcessors{}, processor) assert.IsType(t, &statsChangeProcessor{}, processor.processors[0]) @@ -221,11 +221,7 @@ func TestProcessorRunnerBuildChangeProcessor(t *testing.T) { filters: &MockFilters{}, } -<<<<<<< HEAD - processor = buildChangeProcessor(runner.historyQ, stats, historyArchiveSource, 456, "") -======= processor = buildChangeProcessor(runner.historyQ, &db.MockSession{}, stats, historyArchiveSource, 456, "") ->>>>>>> 8f1836b1 (Use FastBatchInsertBuilder to insert to insert into claimable_balances and claimable_balance_claimants tables) assert.IsType(t, &groupChangeProcessors{}, processor) assert.IsType(t, &statsChangeProcessor{}, processor.processors[0]) diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go index 699694c28e..5806ffa85a 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go @@ -81,8 +81,10 @@ func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { } } - if err := p.InsertClaimableBalanceAndClaimants(ctx, cbsToInsert); err != nil { - return errors.Wrap(err, "error inserting claimable balance") + if len(cbsToInsert) > 0 { + if err := p.insertClaimableBalancesAndClaimants(ctx, cbsToInsert); err != nil { + return errors.Wrap(err, "error inserting claimable balance") + } } if len(cbIDsToDelete) > 0 { @@ -108,19 +110,18 @@ func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { return nil } -func (p *ClaimableBalancesChangeProcessor) InsertClaimableBalanceAndClaimants(ctx context.Context, claimableBalances []history.ClaimableBalance) error { - if len(claimableBalances) == 0 { - return nil - } - +func (p *ClaimableBalancesChangeProcessor) insertClaimableBalancesAndClaimants(ctx context.Context, + claimableBalances []history.ClaimableBalance) error { defer p.claimantsInsertBuilder.Reset() defer p.claimableBalanceInsertBuilder.Reset() for _, cb := range claimableBalances { + // Add claimable balance if err := p.claimableBalanceInsertBuilder.Add(cb); err != nil { - return errors.Wrap(err, "error executing insert") + return errors.Wrap(err, "error adding to ClaimableBalanceBatchInsertBuilder") } + // Add claimants for _, claimant := range cb.Claimants { claimant := history.ClaimableBalanceClaimant{ @@ -130,19 +131,19 @@ func (p *ClaimableBalancesChangeProcessor) InsertClaimableBalanceAndClaimants(ct } if err := p.claimantsInsertBuilder.Add(claimant); err != nil { - return errors.Wrap(err, "error adding to claimantsInsertBuilder") + return errors.Wrap(err, "error adding to ClaimableBalanceClaimantBatchInsertBuilder") } } } err := p.claimantsInsertBuilder.Exec(ctx, p.session) if err != nil { - return errors.Wrap(err, "error executing claimableBalanceInsertBuilder") + return errors.Wrap(err, "error executing ClaimableBalanceClaimantBatchInsertBuilder") } err = p.claimableBalanceInsertBuilder.Exec(ctx, p.session) if err != nil { - return errors.Wrap(err, "error executing claimantsInsertBuilder") + return errors.Wrap(err, "error executing ClaimableBalanceBatchInsertBuilder") } return nil } diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go index f7ca1c7b11..5a952d1dc3 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go @@ -7,7 +7,6 @@ import ( "testing" "github.com/guregu/null" - "github.com/stretchr/testify/suite" "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" @@ -183,7 +182,7 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) TestNewClaimableBalanc Ext: xdr.LedgerEntryExt{ V: 1, V1: &xdr.LedgerEntryExtensionV1{ - SponsoringId: nil, + SponsoringId: xdr.MustAddressPtr("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), }, }, } @@ -200,6 +199,7 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) TestNewClaimableBalanc Asset: cBalance.Asset, Amount: cBalance.Amount, LastModifiedLedger: uint32(lastModifiedLedgerSeq), + Sponsor: null.StringFrom("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), }, ).Return(nil).Once() @@ -209,42 +209,6 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) TestNewClaimableBalanc Post: &entry, }) s.Assert().NoError(err) - - // add sponsor - updated := xdr.LedgerEntry{ - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeClaimableBalance, - ClaimableBalance: &cBalance, - }, - LastModifiedLedgerSeq: lastModifiedLedgerSeq, - Ext: xdr.LedgerEntryExt{ - V: 1, - V1: &xdr.LedgerEntryExtensionV1{ - SponsoringId: xdr.MustAddressPtr("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), - }, - }, - } - - entry.LastModifiedLedgerSeq = entry.LastModifiedLedgerSeq - 1 - err = s.processor.ProcessChange(s.ctx, ingest.Change{ - Type: xdr.LedgerEntryTypeClaimableBalance, - Pre: &entry, - Post: &updated, - }) - s.Assert().NoError(err) - - // We use LedgerEntryChangesCache so all changes are squashed - s.mockClaimableBalanceBatchInsertBuilder.On( - "Add", - history.ClaimableBalance{ - BalanceID: id, - Claimants: []history.Claimant{}, - Asset: cBalance.Asset, - Amount: cBalance.Amount, - LastModifiedLedger: uint32(lastModifiedLedgerSeq), - Sponsor: null.StringFrom("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), - }, - ).Return(nil).Once() } func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) TestRemoveClaimableBalance() { diff --git a/services/horizon/internal/ingest/verify_test.go b/services/horizon/internal/ingest/verify_test.go index 53441bf00d..11b510fe54 100644 --- a/services/horizon/internal/ingest/verify_test.go +++ b/services/horizon/internal/ingest/verify_test.go @@ -271,13 +271,13 @@ func TestStateVerifierLockBusy(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{&db.Session{DB: tt.HorizonDB}} - tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { _ = q.SessionInterface.Rollback() }() checkpointLedger := uint32(63) - changeProcessor := buildChangeProcessor(q, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") + changeProcessor := buildChangeProcessor(q, q.SessionInterface, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") gen := randxdr.NewGenerator() var changes []xdr.LedgerEntryChange @@ -296,6 +296,8 @@ func TestStateVerifierLockBusy(t *testing.T) { } tt.Assert.NoError(changeProcessor.Commit(tt.Ctx)) + tt.Assert.NoError(q.SessionInterface.Commit()) + q.UpdateLastLedgerIngest(tt.Ctx, checkpointLedger) mockHistoryAdapter := &mockHistoryArchiveAdapter{} @@ -329,7 +331,7 @@ func TestStateVerifier(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{&db.Session{DB: tt.HorizonDB}} - tt.Assert.NoError(q.SessionInterface.BeginTx(&sql.TxOptions{})) + tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { _ = q.SessionInterface.Rollback() }() From a9959611c922ba8ed3099a0420dbc607402d22fc Mon Sep 17 00:00:00 2001 From: Urvi Date: Fri, 3 Nov 2023 14:56:37 -0700 Subject: [PATCH 08/14] Fix linter warning --- .../internal/db2/history/claimable_balances_test.go | 8 +++++--- .../mock_claimable_balance_batch_insert_builder.go | 2 +- services/horizon/internal/ingest/verify_test.go | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/services/horizon/internal/db2/history/claimable_balances_test.go b/services/horizon/internal/db2/history/claimable_balances_test.go index f38674866d..30f7ae0da7 100644 --- a/services/horizon/internal/db2/history/claimable_balances_test.go +++ b/services/horizon/internal/db2/history/claimable_balances_test.go @@ -3,9 +3,9 @@ package history import ( "database/sql" "fmt" - "github.com/guregu/null" "testing" + "github.com/guregu/null" "github.com/stellar/go/services/horizon/internal/db2" "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/xdr" @@ -264,8 +264,10 @@ func TestFindClaimableBalancesByDestinationWithLimit(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{}) - defer q.Rollback() + tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) + defer func() { + _ = q.SessionInterface.Rollback() + }() assetIssuer := "GA25GQLHJU3LPEJXEIAXK23AWEA5GWDUGRSHTQHDFT6HXHVMRULSQJUJ" asset1 := xdr.MustNewCreditAsset("ASSET1", assetIssuer) diff --git a/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go index fbd73bb6cd..d3ce104244 100644 --- a/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go @@ -2,8 +2,8 @@ package history import ( "context" - "github.com/stellar/go/support/db" + "github.com/stellar/go/support/db" "github.com/stretchr/testify/mock" ) diff --git a/services/horizon/internal/ingest/verify_test.go b/services/horizon/internal/ingest/verify_test.go index 11b510fe54..96a6bda95c 100644 --- a/services/horizon/internal/ingest/verify_test.go +++ b/services/horizon/internal/ingest/verify_test.go @@ -366,7 +366,7 @@ func TestStateVerifier(t *testing.T) { tt.Assert.NoError(changeProcessor.Commit(tt.Ctx)) tt.Assert.Equal(len(xdr.LedgerEntryTypeMap), len(coverage)) - q.SessionInterface.Commit() + tt.Assert.NoError(q.SessionInterface.Commit()) q.UpdateLastLedgerIngest(tt.Ctx, checkpointLedger) From 59431a2c3cb59732ca97749881d4ea11918e2417 Mon Sep 17 00:00:00 2001 From: Urvi Date: Mon, 6 Nov 2023 15:30:13 -0800 Subject: [PATCH 09/14] Addressing review comments --- .../actions/claimable_balance_test.go | 24 +++--- .../claimable_balance_batch_insert_builder.go | 27 +++---- ...e_balance_claimant_batch_insert_builder.go | 27 +++---- .../db2/history/claimable_balances.go | 5 +- .../db2/history/claimable_balances_test.go | 32 ++++---- ..._claimable_balance_batch_insert_builder.go | 9 +-- ...e_balance_claimant_batch_insert_builder.go | 10 +-- .../db2/history/mock_q_claimable_balances.go | 9 ++- .../internal/ingest/processor_runner_test.go | 81 +++++++++++++------ .../claimable_balances_change_processor.go | 10 +-- ...laimable_balances_change_processor_test.go | 23 ++---- .../horizon/internal/ingest/verify_test.go | 6 -- 12 files changed, 128 insertions(+), 135 deletions(-) diff --git a/services/horizon/internal/actions/claimable_balance_test.go b/services/horizon/internal/actions/claimable_balance_test.go index add2dac150..b645cc21d3 100644 --- a/services/horizon/internal/actions/claimable_balance_test.go +++ b/services/horizon/internal/actions/claimable_balance_test.go @@ -49,10 +49,10 @@ func TestGetClaimableBalanceByID(t *testing.T) { LastModifiedLedger: 123, } - balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(q.SessionInterface) for _, claimant := range cBalance.Claimants { claimant := history.ClaimableBalanceClaimant{ BalanceID: cBalance.BalanceID, @@ -62,8 +62,8 @@ func TestGetClaimableBalanceByID(t *testing.T) { tt.Assert.NoError(claimantsInsertBuilder.Add(claimant)) } - tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) - tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx)) + tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx)) handler := GetClaimableBalanceByIDHandler{} response, err := handler.GetResource(httptest.NewRecorder(), makeRequest( @@ -200,9 +200,9 @@ func TestGetClaimableBalances(t *testing.T) { hCBs = append(hCBs, cb) } - balanceInsertbuilder := q.NewClaimableBalanceBatchInsertBuilder() + balanceInsertbuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(q.SessionInterface) for _, cBalance := range hCBs { tt.Assert.NoError(balanceInsertbuilder.Add(cBalance)) @@ -217,8 +217,8 @@ func TestGetClaimableBalances(t *testing.T) { } } - tt.Assert.NoError(balanceInsertbuilder.Exec(tt.Ctx, q.SessionInterface)) - tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(balanceInsertbuilder.Exec(tt.Ctx)) + tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx)) handler := GetClaimableBalancesHandler{} response, err := handler.GetResourcePage(httptest.NewRecorder(), makeRequest( @@ -298,8 +298,8 @@ func TestGetClaimableBalances(t *testing.T) { tt.Assert.Len(response, 0) // new claimable balances are ingested, they should appear in the next pages - balanceInsertbuilder = q.NewClaimableBalanceBatchInsertBuilder() - claimantsInsertBuilder = q.NewClaimableBalanceClaimantBatchInsertBuilder() + balanceInsertbuilder = q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) + claimantsInsertBuilder = q.NewClaimableBalanceClaimantBatchInsertBuilder(q.SessionInterface) entriesMeta = []struct { id xdr.Hash @@ -339,8 +339,8 @@ func TestGetClaimableBalances(t *testing.T) { } } - tt.Assert.NoError(balanceInsertbuilder.Exec(tt.Ctx, q.SessionInterface)) - tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(balanceInsertbuilder.Exec(tt.Ctx)) + tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx)) response, err = handler.GetResourcePage(httptest.NewRecorder(), makeRequest( t, diff --git a/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go b/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go index 93a6abd9de..70105923a1 100644 --- a/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go @@ -4,30 +4,28 @@ import ( "context" "github.com/stellar/go/support/db" - "github.com/stellar/go/xdr" ) // ClaimableBalanceBatchInsertBuilder is used to insert claimable balance into the // claimable_balances table type ClaimableBalanceBatchInsertBuilder interface { Add(claimableBalance ClaimableBalance) error - Exec(ctx context.Context, session db.SessionInterface) error - Reset() + Exec(ctx context.Context) error } // ClaimableBalanceBatchInsertBuilder is a simple wrapper around db.FastBatchInsertBuilder type claimableBalanceBatchInsertBuilder struct { - encodingBuffer *xdr.EncodingBuffer - builder db.FastBatchInsertBuilder - table string + session db.SessionInterface + builder db.FastBatchInsertBuilder + table string } // NewClaimableBalanceBatchInsertBuilder constructs a new ClaimableBalanceBatchInsertBuilder instance -func (q *Q) NewClaimableBalanceBatchInsertBuilder() ClaimableBalanceBatchInsertBuilder { +func (q *Q) NewClaimableBalanceBatchInsertBuilder(session db.SessionInterface) ClaimableBalanceBatchInsertBuilder { return &claimableBalanceBatchInsertBuilder{ - encodingBuffer: xdr.NewEncodingBuffer(), - builder: db.FastBatchInsertBuilder{}, - table: "claimable_balances", + session: session, + builder: db.FastBatchInsertBuilder{}, + table: "claimable_balances", } } @@ -37,11 +35,6 @@ func (i *claimableBalanceBatchInsertBuilder) Add(claimableBalance ClaimableBalan } // Exec writes the batch of claimable balances to the database. -func (i *claimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { - return i.builder.Exec(ctx, session, i.table) -} - -// Reset clears out the current batch of claimable balances -func (i *claimableBalanceBatchInsertBuilder) Reset() { - i.builder.Reset() +func (i *claimableBalanceBatchInsertBuilder) Exec(ctx context.Context) error { + return i.builder.Exec(ctx, i.session, i.table) } diff --git a/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go b/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go index 37eaa67e35..1e87e6e181 100644 --- a/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go @@ -4,30 +4,28 @@ import ( "context" "github.com/stellar/go/support/db" - "github.com/stellar/go/xdr" ) // ClaimableBalanceClaimantBatchInsertBuilder is used to insert claimants into the // claimable_balance_claimants table type ClaimableBalanceClaimantBatchInsertBuilder interface { Add(claimableBalanceClaimant ClaimableBalanceClaimant) error - Exec(ctx context.Context, session db.SessionInterface) error - Reset() + Exec(ctx context.Context) error } // ClaimableBalanceClaimantBatchInsertBuilder is a simple wrapper around db.FastBatchInsertBuilder type claimableBalanceClaimantBatchInsertBuilder struct { - encodingBuffer *xdr.EncodingBuffer - builder db.FastBatchInsertBuilder - table string + session db.SessionInterface + builder db.FastBatchInsertBuilder + table string } // NewClaimableBalanceClaimantBatchInsertBuilder constructs a new ClaimableBalanceClaimantBatchInsertBuilder instance -func (q *Q) NewClaimableBalanceClaimantBatchInsertBuilder() ClaimableBalanceClaimantBatchInsertBuilder { +func (q *Q) NewClaimableBalanceClaimantBatchInsertBuilder(session db.SessionInterface) ClaimableBalanceClaimantBatchInsertBuilder { return &claimableBalanceClaimantBatchInsertBuilder{ - encodingBuffer: xdr.NewEncodingBuffer(), - builder: db.FastBatchInsertBuilder{}, - table: "claimable_balance_claimants", + session: session, + builder: db.FastBatchInsertBuilder{}, + table: "claimable_balance_claimants", } } @@ -37,11 +35,6 @@ func (i *claimableBalanceClaimantBatchInsertBuilder) Add(claimableBalanceClaiman } // Exec writes the batch of claimants to the database. -func (i *claimableBalanceClaimantBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { - return i.builder.Exec(ctx, session, i.table) -} - -// Reset clears out the current batch of claimants -func (i *claimableBalanceClaimantBatchInsertBuilder) Reset() { - i.builder.Reset() +func (i *claimableBalanceClaimantBatchInsertBuilder) Exec(ctx context.Context) error { + return i.builder.Exec(ctx, i.session, i.table) } diff --git a/services/horizon/internal/db2/history/claimable_balances.go b/services/horizon/internal/db2/history/claimable_balances.go index 5490bef11c..24e88cb512 100644 --- a/services/horizon/internal/db2/history/claimable_balances.go +++ b/services/horizon/internal/db2/history/claimable_balances.go @@ -11,6 +11,7 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/guregu/null" "github.com/stellar/go/services/horizon/internal/db2" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) @@ -134,8 +135,8 @@ type QClaimableBalances interface { RemoveClaimableBalanceClaimants(ctx context.Context, ids []string) (int64, error) GetClaimableBalancesByID(ctx context.Context, ids []string) ([]ClaimableBalance, error) CountClaimableBalances(ctx context.Context) (int, error) - NewClaimableBalanceClaimantBatchInsertBuilder() ClaimableBalanceClaimantBatchInsertBuilder - NewClaimableBalanceBatchInsertBuilder() ClaimableBalanceBatchInsertBuilder + NewClaimableBalanceClaimantBatchInsertBuilder(session db.SessionInterface) ClaimableBalanceClaimantBatchInsertBuilder + NewClaimableBalanceBatchInsertBuilder(session db.SessionInterface) ClaimableBalanceBatchInsertBuilder GetClaimantsByClaimableBalances(ctx context.Context, ids []string) (map[string][]ClaimableBalanceClaimant, error) } diff --git a/services/horizon/internal/db2/history/claimable_balances_test.go b/services/horizon/internal/db2/history/claimable_balances_test.go index 30f7ae0da7..217281bfb5 100644 --- a/services/horizon/internal/db2/history/claimable_balances_test.go +++ b/services/horizon/internal/db2/history/claimable_balances_test.go @@ -44,9 +44,9 @@ func TestRemoveClaimableBalance(t *testing.T) { Amount: 10, } - balanceBatchInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() + balanceBatchInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) tt.Assert.NoError(balanceBatchInsertBuilder.Add(cBalance)) - tt.Assert.NoError(balanceBatchInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(balanceBatchInsertBuilder.Exec(tt.Ctx)) r, err := q.FindClaimableBalanceByID(tt.Ctx, id) tt.Assert.NoError(err) @@ -97,9 +97,9 @@ func TestRemoveClaimableBalanceClaimants(t *testing.T) { Amount: 10, } - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(q.SessionInterface) tt.Assert.NoError(insertClaimants(claimantsInsertBuilder, cBalance)) - tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx)) removed, err := q.RemoveClaimableBalanceClaimants(tt.Ctx, []string{id}) tt.Assert.NoError(err) @@ -142,8 +142,8 @@ func TestFindClaimableBalancesByDestination(t *testing.T) { Amount: 10, } - balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(q.SessionInterface) tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) tt.Assert.NoError(insertClaimants(claimantsInsertBuilder, cBalance)) @@ -178,8 +178,8 @@ func TestFindClaimableBalancesByDestination(t *testing.T) { tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) tt.Assert.NoError(insertClaimants(claimantsInsertBuilder, cBalance)) - tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) - tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx)) + tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx)) query := ClaimableBalancesQuery{ PageQuery: db2.MustPageQuery("", false, "", 10), @@ -308,8 +308,8 @@ func TestFindClaimableBalancesByDestinationWithLimit(t *testing.T) { LastModifiedLedger: 123, Amount: 10, } - balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(q.SessionInterface) tt.Assert.NoError(balanceInsertBuilder.Add(cBalance1)) tt.Assert.NoError(insertClaimants(claimantsInsertBuilder, cBalance1)) @@ -342,8 +342,8 @@ func TestFindClaimableBalancesByDestinationWithLimit(t *testing.T) { tt.Assert.NoError(balanceInsertBuilder.Add(cBalance2)) tt.Assert.NoError(insertClaimants(claimantsInsertBuilder, cBalance2)) - tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) - tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx)) + tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx)) pageQuery := db2.MustPageQuery("", false, "", 1) @@ -439,9 +439,9 @@ func TestFindClaimableBalance(t *testing.T) { Amount: 10, } - balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) - tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx)) cb, err := q.FindClaimableBalanceByID(tt.Ctx, id) @@ -488,9 +488,9 @@ func TestGetClaimableBalancesByID(t *testing.T) { Amount: 10, } - balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) - tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx)) r, err := q.GetClaimableBalancesByID(tt.Ctx, []string{id}) tt.Assert.NoError(err) diff --git a/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go index d3ce104244..0c1a1e7049 100644 --- a/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/mock_claimable_balance_batch_insert_builder.go @@ -3,7 +3,6 @@ package history import ( "context" - "github.com/stellar/go/support/db" "github.com/stretchr/testify/mock" ) @@ -16,11 +15,7 @@ func (m *MockClaimableBalanceBatchInsertBuilder) Add(claimableBalance ClaimableB return a.Error(0) } -func (m *MockClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { - a := m.Called(ctx, session) +func (m *MockClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context) error { + a := m.Called(ctx) return a.Error(0) } - -func (m *MockClaimableBalanceBatchInsertBuilder) Reset() { - m.Called() -} diff --git a/services/horizon/internal/db2/history/mock_claimable_balance_claimant_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_claimable_balance_claimant_batch_insert_builder.go index 785df6fd16..3c8fb24d3c 100644 --- a/services/horizon/internal/db2/history/mock_claimable_balance_claimant_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/mock_claimable_balance_claimant_batch_insert_builder.go @@ -2,8 +2,6 @@ package history import ( "context" - "github.com/stellar/go/support/db" - "github.com/stretchr/testify/mock" ) @@ -16,11 +14,7 @@ func (m *MockClaimableBalanceClaimantBatchInsertBuilder) Add(claimableBalanceCla return a.Error(0) } -func (m *MockClaimableBalanceClaimantBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { - a := m.Called(ctx, session) +func (m *MockClaimableBalanceClaimantBatchInsertBuilder) Exec(ctx context.Context) error { + a := m.Called(ctx) return a.Error(0) } - -func (m *MockClaimableBalanceClaimantBatchInsertBuilder) Reset() { - m.Called() -} diff --git a/services/horizon/internal/db2/history/mock_q_claimable_balances.go b/services/horizon/internal/db2/history/mock_q_claimable_balances.go index 64b65cf1a3..f63a3e04e7 100644 --- a/services/horizon/internal/db2/history/mock_q_claimable_balances.go +++ b/services/horizon/internal/db2/history/mock_q_claimable_balances.go @@ -3,6 +3,7 @@ package history import ( "context" + "github.com/stellar/go/support/db" "github.com/stretchr/testify/mock" ) @@ -31,13 +32,13 @@ func (m *MockQClaimableBalances) RemoveClaimableBalanceClaimants(ctx context.Con return a.Get(0).(int64), a.Error(1) } -func (m *MockQClaimableBalances) NewClaimableBalanceClaimantBatchInsertBuilder() ClaimableBalanceClaimantBatchInsertBuilder { - a := m.Called() +func (m *MockQClaimableBalances) NewClaimableBalanceClaimantBatchInsertBuilder(session db.SessionInterface) ClaimableBalanceClaimantBatchInsertBuilder { + a := m.Called(session) return a.Get(0).(ClaimableBalanceClaimantBatchInsertBuilder) } -func (m *MockQClaimableBalances) NewClaimableBalanceBatchInsertBuilder() ClaimableBalanceBatchInsertBuilder { - a := m.Called() +func (m *MockQClaimableBalances) NewClaimableBalanceBatchInsertBuilder(session db.SessionInterface) ClaimableBalanceBatchInsertBuilder { + a := m.Called(session) return a.Get(0).(ClaimableBalanceBatchInsertBuilder) } diff --git a/services/horizon/internal/ingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go index 6ca0f7319e..18e5af6665 100644 --- a/services/horizon/internal/ingest/processor_runner_test.go +++ b/services/horizon/internal/ingest/processor_runner_test.go @@ -24,6 +24,8 @@ func TestProcessorRunnerRunHistoryArchiveIngestionGenesis(t *testing.T) { ctx := context.Background() maxBatchSize := 100000 + mockSession := &db.MockSession{} + q := &mockDBQ{} q.MockQAccounts.On("UpsertAccounts", ctx, []history.AccountEntry{ @@ -49,10 +51,14 @@ func TestProcessorRunnerRunHistoryArchiveIngestionGenesis(t *testing.T) { q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize). Return(mockAccountSignersBatchInsertBuilder).Once() - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). - Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() - q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). - Return(&history.MockClaimableBalanceBatchInsertBuilder{}).Once() + mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). + Return(mockClaimableBalanceBatchInsertBuilder).Once() + + mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). + Return(mockClaimantsBatchInsertBuilder).Once() + q.MockQAssetStats.On("InsertAssetStats", ctx, []history.ExpAssetStat{}, 100000). Return(nil) @@ -63,6 +69,7 @@ func TestProcessorRunnerRunHistoryArchiveIngestionGenesis(t *testing.T) { }, historyQ: q, filters: &MockFilters{}, + session: mockSession, } _, err := runner.RunGenesisStateIngestion() @@ -77,6 +84,7 @@ func TestProcessorRunnerRunHistoryArchiveIngestionHistoryArchive(t *testing.T) { NetworkPassphrase: network.PublicNetworkPassphrase, } + mockSession := &db.MockSession{} q := &mockDBQ{} defer mock.AssertExpectationsForObjects(t, q) historyAdapter := &mockHistoryArchiveAdapter{} @@ -117,10 +125,15 @@ func TestProcessorRunnerRunHistoryArchiveIngestionHistoryArchive(t *testing.T) { mockAccountSignersBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize). Return(mockAccountSignersBatchInsertBuilder).Once() - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). - Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() - q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). - Return(&history.MockClaimableBalanceBatchInsertBuilder{}).Once() + + mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). + Return(mockClaimableBalanceBatchInsertBuilder).Once() + + mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). + Return(mockClaimantsBatchInsertBuilder).Once() + q.MockQAssetStats.On("InsertAssetStats", ctx, []history.ExpAssetStat{}, 100000). Return(nil) @@ -130,6 +143,7 @@ func TestProcessorRunnerRunHistoryArchiveIngestionHistoryArchive(t *testing.T) { historyQ: q, historyAdapter: historyAdapter, filters: &MockFilters{}, + session: mockSession, } _, err := runner.RunHistoryArchiveIngestion(63, false, MaxSupportedProtocolVersion, bucketListHash) @@ -144,6 +158,7 @@ func TestProcessorRunnerRunHistoryArchiveIngestionProtocolVersionNotSupported(t NetworkPassphrase: network.PublicNetworkPassphrase, } + mockSession := &db.MockSession{} q := &mockDBQ{} defer mock.AssertExpectationsForObjects(t, q) historyAdapter := &mockHistoryArchiveAdapter{} @@ -155,10 +170,15 @@ func TestProcessorRunnerRunHistoryArchiveIngestionProtocolVersionNotSupported(t defer mock.AssertExpectationsForObjects(t, mockAccountSignersBatchInsertBuilder) q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize). Return(mockAccountSignersBatchInsertBuilder).Once() - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). - Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() - q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). - Return(&history.MockClaimableBalanceBatchInsertBuilder{}).Once() + + mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). + Return(mockClaimableBalanceBatchInsertBuilder).Once() + + mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). + Return(mockClaimantsBatchInsertBuilder).Once() + q.MockQAssetStats.On("InsertAssetStats", ctx, []history.ExpAssetStat{}, 100000). Return(nil) @@ -168,6 +188,7 @@ func TestProcessorRunnerRunHistoryArchiveIngestionProtocolVersionNotSupported(t historyQ: q, historyAdapter: historyAdapter, filters: &MockFilters{}, + session: mockSession, } _, err := runner.RunHistoryArchiveIngestion(100, false, 200, xdr.Hash{}) @@ -183,16 +204,22 @@ func TestProcessorRunnerBuildChangeProcessor(t *testing.T) { ctx := context.Background() maxBatchSize := 100000 + mockSession := &db.MockSession{} q := &mockDBQ{} defer mock.AssertExpectationsForObjects(t, q) // Twice = checking ledgerSource and historyArchiveSource q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize). Return(&history.MockAccountSignersBatchInsertBuilder{}).Twice() - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). - Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Twice() - q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). - Return(&history.MockClaimableBalanceBatchInsertBuilder{}).Twice() + + mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). + Return(mockClaimableBalanceBatchInsertBuilder).Twice() + + mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). + Return(mockClaimantsBatchInsertBuilder).Twice() + runner := ProcessorRunner{ ctx: ctx, historyQ: q, @@ -318,11 +345,13 @@ func TestProcessorRunnerWithFilterEnabled(t *testing.T) { q.MockQTransactions.On("NewTransactionFilteredTmpBatchInsertBuilder"). Return(mockTransactionsFilteredTmpBatchInsertBuilder) - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). - Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() + mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). + Return(mockClaimableBalanceBatchInsertBuilder).Once() - q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). - Return(&history.MockClaimableBalanceBatchInsertBuilder{}).Once() + mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). + Return(mockClaimantsBatchInsertBuilder).Once() q.On("DeleteTransactionsFilteredTmpOlderThan", ctx, mock.AnythingOfType("uint64")). Return(int64(0), nil) @@ -388,11 +417,13 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) { mockSession, ).Return(nil) - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). - Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() + mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). + Return(mockClaimableBalanceBatchInsertBuilder).Once() - q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). - Return(&history.MockClaimableBalanceBatchInsertBuilder{}).Once() + mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). + Return(mockClaimantsBatchInsertBuilder).Once() runner := ProcessorRunner{ ctx: ctx, @@ -510,7 +541,7 @@ func mockBatchBuilders(q *mockDBQ, mockSession *db.MockSession, ctx context.Cont q.MockQHistoryLiquidityPools.On("NewOperationLiquidityPoolBatchInsertBuilder"). Return(mockOperationLiquidityPoolBatchInsertBuilder) - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize). + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() q.On("NewTradeBatchInsertBuilder").Return(&history.MockTradeBatchInsertBuilder{}) diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go index 5806ffa85a..f799d77438 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go @@ -31,8 +31,8 @@ func NewClaimableBalancesChangeProcessor(Q history.QClaimableBalances, session d func (p *ClaimableBalancesChangeProcessor) reset() { p.cache = ingest.NewChangeCompactor() - p.claimantsInsertBuilder = p.qClaimableBalances.NewClaimableBalanceClaimantBatchInsertBuilder() - p.claimableBalanceInsertBuilder = p.qClaimableBalances.NewClaimableBalanceBatchInsertBuilder() + p.claimantsInsertBuilder = p.qClaimableBalances.NewClaimableBalanceClaimantBatchInsertBuilder(p.session) + p.claimableBalanceInsertBuilder = p.qClaimableBalances.NewClaimableBalanceBatchInsertBuilder(p.session) } func (p *ClaimableBalancesChangeProcessor) ProcessChange(ctx context.Context, change ingest.Change) error { @@ -112,8 +112,6 @@ func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { func (p *ClaimableBalancesChangeProcessor) insertClaimableBalancesAndClaimants(ctx context.Context, claimableBalances []history.ClaimableBalance) error { - defer p.claimantsInsertBuilder.Reset() - defer p.claimableBalanceInsertBuilder.Reset() for _, cb := range claimableBalances { @@ -136,12 +134,12 @@ func (p *ClaimableBalancesChangeProcessor) insertClaimableBalancesAndClaimants(c } } - err := p.claimantsInsertBuilder.Exec(ctx, p.session) + err := p.claimantsInsertBuilder.Exec(ctx) if err != nil { return errors.Wrap(err, "error executing ClaimableBalanceClaimantBatchInsertBuilder") } - err = p.claimableBalanceInsertBuilder.Exec(ctx, p.session) + err = p.claimableBalanceInsertBuilder.Exec(ctx) if err != nil { return errors.Wrap(err, "error executing ClaimableBalanceBatchInsertBuilder") } diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go index 5a952d1dc3..625a64f74e 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go @@ -37,15 +37,11 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteState) SetupTest() { s.mockQ = &history.MockQClaimableBalances{} s.session = &db.MockSession{} s.mockQ. - On("NewClaimableBalanceClaimantBatchInsertBuilder"). + On("NewClaimableBalanceClaimantBatchInsertBuilder", s.session). Return(s.mockClaimantsBatchInsertBuilder).Once() s.mockQ. - On("NewClaimableBalanceBatchInsertBuilder"). + On("NewClaimableBalanceBatchInsertBuilder", s.session). Return(s.mockClaimableBalanceBatchInsertBuilder).Once() - - s.mockClaimantsBatchInsertBuilder.On("Reset").Return(nil).Once() - s.mockClaimableBalanceBatchInsertBuilder.On("Reset").Return(nil).Once() - s.processor = NewClaimableBalancesChangeProcessor(s.mockQ, s.session) } @@ -91,7 +87,7 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteState) TestCreatesClaimableBal LastModifiedLedger: uint32(lastModifiedLedgerSeq), }).Return(nil).Once() - s.mockClaimableBalanceBatchInsertBuilder.On("Exec", s.ctx, s.session).Return(nil).Once() + s.mockClaimableBalanceBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() s.mockClaimantsBatchInsertBuilder.On("Add", history.ClaimableBalanceClaimant{ BalanceID: id, @@ -99,7 +95,7 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteState) TestCreatesClaimableBal LastModifiedLedger: uint32(lastModifiedLedgerSeq), }).Return(nil).Once() - s.mockClaimantsBatchInsertBuilder.On("Exec", s.ctx, s.session).Return(nil).Once() + s.mockClaimantsBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() err = s.processor.ProcessChange(s.ctx, ingest.Change{ Type: xdr.LedgerEntryTypeClaimableBalance, @@ -136,17 +132,14 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) SetupTest() { s.mockQ = &history.MockQClaimableBalances{} s.session = &db.MockSession{} s.mockQ. - On("NewClaimableBalanceClaimantBatchInsertBuilder"). + On("NewClaimableBalanceClaimantBatchInsertBuilder", &db.MockSession{}). Return(s.mockClaimantsBatchInsertBuilder).Twice() s.mockQ. - On("NewClaimableBalanceBatchInsertBuilder"). + On("NewClaimableBalanceBatchInsertBuilder", &db.MockSession{}). Return(s.mockClaimableBalanceBatchInsertBuilder).Twice() - s.mockClaimantsBatchInsertBuilder.On("Reset").Return(nil).Once() - s.mockClaimableBalanceBatchInsertBuilder.On("Reset").Return(nil).Once() - - s.mockClaimantsBatchInsertBuilder.On("Exec", s.ctx, s.session).Return(nil).Once() - s.mockClaimableBalanceBatchInsertBuilder.On("Exec", s.ctx, s.session).Return(nil).Once() + s.mockClaimantsBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() + s.mockClaimableBalanceBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() s.processor = NewClaimableBalancesChangeProcessor(s.mockQ, s.session) } diff --git a/services/horizon/internal/ingest/verify_test.go b/services/horizon/internal/ingest/verify_test.go index 96a6bda95c..d2137aff36 100644 --- a/services/horizon/internal/ingest/verify_test.go +++ b/services/horizon/internal/ingest/verify_test.go @@ -272,9 +272,6 @@ func TestStateVerifierLockBusy(t *testing.T) { q := &history.Q{&db.Session{DB: tt.HorizonDB}} tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) - defer func() { - _ = q.SessionInterface.Rollback() - }() checkpointLedger := uint32(63) changeProcessor := buildChangeProcessor(q, q.SessionInterface, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") @@ -332,9 +329,6 @@ func TestStateVerifier(t *testing.T) { q := &history.Q{&db.Session{DB: tt.HorizonDB}} tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) - defer func() { - _ = q.SessionInterface.Rollback() - }() checkpointLedger := uint32(63) changeProcessor := buildChangeProcessor(q, q.SessionInterface, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") From 6ac24e0bcaf36cb4a856666e8b934781ce9e7401 Mon Sep 17 00:00:00 2001 From: Urvi Date: Mon, 6 Nov 2023 17:31:57 -0800 Subject: [PATCH 10/14] Addressing review comments --- .../internal/ingest/processor_runner_test.go | 37 ++++++--- .../claimable_balances_change_processor.go | 80 ++++++++----------- ...laimable_balances_change_processor_test.go | 16 ++-- 3 files changed, 68 insertions(+), 65 deletions(-) diff --git a/services/horizon/internal/ingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go index 18e5af6665..c97d30c423 100644 --- a/services/horizon/internal/ingest/processor_runner_test.go +++ b/services/horizon/internal/ingest/processor_runner_test.go @@ -53,11 +53,14 @@ func TestProcessorRunnerRunHistoryArchiveIngestionGenesis(t *testing.T) { mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). - Return(mockClaimableBalanceBatchInsertBuilder).Once() + Return(mockClaimableBalanceBatchInsertBuilder).Twice() mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). - Return(mockClaimantsBatchInsertBuilder).Once() + Return(mockClaimantsBatchInsertBuilder).Twice() + + mockClaimantsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() + mockClaimableBalanceBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() q.MockQAssetStats.On("InsertAssetStats", ctx, []history.ExpAssetStat{}, 100000). Return(nil) @@ -128,11 +131,14 @@ func TestProcessorRunnerRunHistoryArchiveIngestionHistoryArchive(t *testing.T) { mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). - Return(mockClaimableBalanceBatchInsertBuilder).Once() + Return(mockClaimableBalanceBatchInsertBuilder).Twice() mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). - Return(mockClaimantsBatchInsertBuilder).Once() + Return(mockClaimantsBatchInsertBuilder).Twice() + + mockClaimantsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() + mockClaimableBalanceBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() q.MockQAssetStats.On("InsertAssetStats", ctx, []history.ExpAssetStat{}, 100000). Return(nil) @@ -179,6 +185,9 @@ func TestProcessorRunnerRunHistoryArchiveIngestionProtocolVersionNotSupported(t q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). Return(mockClaimantsBatchInsertBuilder).Once() + mockClaimantsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() + mockClaimableBalanceBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() + q.MockQAssetStats.On("InsertAssetStats", ctx, []history.ExpAssetStat{}, 100000). Return(nil) @@ -353,6 +362,9 @@ func TestProcessorRunnerWithFilterEnabled(t *testing.T) { q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). Return(mockClaimantsBatchInsertBuilder).Once() + mockClaimantsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() + mockClaimableBalanceBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() + q.On("DeleteTransactionsFilteredTmpOlderThan", ctx, mock.AnythingOfType("uint64")). Return(int64(0), nil) @@ -417,14 +429,6 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) { mockSession, ).Return(nil) - mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} - q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). - Return(mockClaimableBalanceBatchInsertBuilder).Once() - - mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). - Return(mockClaimantsBatchInsertBuilder).Once() - runner := ProcessorRunner{ ctx: ctx, config: config, @@ -541,8 +545,15 @@ func mockBatchBuilders(q *mockDBQ, mockSession *db.MockSession, ctx context.Cont q.MockQHistoryLiquidityPools.On("NewOperationLiquidityPoolBatchInsertBuilder"). Return(mockOperationLiquidityPoolBatchInsertBuilder) + mockClaimableBalanceClaimantBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} + mockClaimableBalanceClaimantBatchInsertBuilder.On("Exec", ctx).Return(nil) q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). - Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() + Return(mockClaimableBalanceClaimantBatchInsertBuilder) + + mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). + Return(mockClaimableBalanceBatchInsertBuilder) + mockClaimableBalanceBatchInsertBuilder.On("Exec", ctx).Return(nil) q.On("NewTradeBatchInsertBuilder").Return(&history.MockTradeBatchInsertBuilder{}) diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go index f799d77438..c98bde9821 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go @@ -2,6 +2,7 @@ package processors import ( "context" + "fmt" "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" @@ -50,27 +51,43 @@ func (p *ClaimableBalancesChangeProcessor) ProcessChange(ctx context.Context, ch if err != nil { return errors.Wrap(err, "error in Commit") } - p.reset() } return nil } func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { + defer p.reset() var ( - cbsToInsert []history.ClaimableBalance cbIDsToDelete []string ) changes := p.cache.GetChanges() for _, change := range changes { - if change.Post != nil { + switch { + case change.Pre == nil && change.Post != nil: // Created - row, err := p.ledgerEntryToRow(change.Post) + cb, err := p.ledgerEntryToRow(change.Post) if err != nil { return err } - cbsToInsert = append(cbsToInsert, row) - } else { + // Add claimable balance + if err := p.claimableBalanceInsertBuilder.Add(cb); err != nil { + return errors.Wrap(err, "error adding to ClaimableBalanceBatchInsertBuilder") + } + + // Add claimants + for _, claimant := range cb.Claimants { + claimant := history.ClaimableBalanceClaimant{ + BalanceID: cb.BalanceID, + Destination: claimant.Destination, + LastModifiedLedger: cb.LastModifiedLedger, + } + + if err := p.claimantsInsertBuilder.Add(claimant); err != nil { + return errors.Wrap(err, "error adding to ClaimableBalanceClaimantBatchInsertBuilder") + } + } + case change.Pre != nil && change.Post == nil: // Removed cBalance := change.Pre.Data.MustClaimableBalance() id, err := p.encodingBuffer.MarshalHex(cBalance.BalanceId) @@ -78,13 +95,20 @@ func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { return err } cbIDsToDelete = append(cbIDsToDelete, id) + default: + // claimable balance can only be created or removed + return fmt.Errorf("invalid change entry for a claimable balance was detected") } } - if len(cbsToInsert) > 0 { - if err := p.insertClaimableBalancesAndClaimants(ctx, cbsToInsert); err != nil { - return errors.Wrap(err, "error inserting claimable balance") - } + err := p.claimantsInsertBuilder.Exec(ctx) + if err != nil { + return errors.Wrap(err, "error executing ClaimableBalanceClaimantBatchInsertBuilder") + } + + err = p.claimableBalanceInsertBuilder.Exec(ctx) + if err != nil { + return errors.Wrap(err, "error executing ClaimableBalanceBatchInsertBuilder") } if len(cbIDsToDelete) > 0 { @@ -110,42 +134,6 @@ func (p *ClaimableBalancesChangeProcessor) Commit(ctx context.Context) error { return nil } -func (p *ClaimableBalancesChangeProcessor) insertClaimableBalancesAndClaimants(ctx context.Context, - claimableBalances []history.ClaimableBalance) error { - - for _, cb := range claimableBalances { - - // Add claimable balance - if err := p.claimableBalanceInsertBuilder.Add(cb); err != nil { - return errors.Wrap(err, "error adding to ClaimableBalanceBatchInsertBuilder") - } - - // Add claimants - for _, claimant := range cb.Claimants { - claimant := history.ClaimableBalanceClaimant{ - BalanceID: cb.BalanceID, - Destination: claimant.Destination, - LastModifiedLedger: cb.LastModifiedLedger, - } - - if err := p.claimantsInsertBuilder.Add(claimant); err != nil { - return errors.Wrap(err, "error adding to ClaimableBalanceClaimantBatchInsertBuilder") - } - } - } - - err := p.claimantsInsertBuilder.Exec(ctx) - if err != nil { - return errors.Wrap(err, "error executing ClaimableBalanceClaimantBatchInsertBuilder") - } - - err = p.claimableBalanceInsertBuilder.Exec(ctx) - if err != nil { - return errors.Wrap(err, "error executing ClaimableBalanceBatchInsertBuilder") - } - return nil -} - func buildClaimants(claimants []xdr.Claimant) history.Claimants { hClaimants := history.Claimants{} for _, c := range claimants { diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go index 625a64f74e..ccf1a85de2 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go @@ -38,10 +38,14 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteState) SetupTest() { s.session = &db.MockSession{} s.mockQ. On("NewClaimableBalanceClaimantBatchInsertBuilder", s.session). - Return(s.mockClaimantsBatchInsertBuilder).Once() + Return(s.mockClaimantsBatchInsertBuilder) s.mockQ. On("NewClaimableBalanceBatchInsertBuilder", s.session). - Return(s.mockClaimableBalanceBatchInsertBuilder).Once() + Return(s.mockClaimableBalanceBatchInsertBuilder) + + s.mockClaimantsBatchInsertBuilder.On("Exec", s.ctx).Return(nil) + s.mockClaimableBalanceBatchInsertBuilder.On("Exec", s.ctx).Return(nil) + s.processor = NewClaimableBalancesChangeProcessor(s.mockQ, s.session) } @@ -133,13 +137,13 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) SetupTest() { s.session = &db.MockSession{} s.mockQ. On("NewClaimableBalanceClaimantBatchInsertBuilder", &db.MockSession{}). - Return(s.mockClaimantsBatchInsertBuilder).Twice() + Return(s.mockClaimantsBatchInsertBuilder) s.mockQ. On("NewClaimableBalanceBatchInsertBuilder", &db.MockSession{}). - Return(s.mockClaimableBalanceBatchInsertBuilder).Twice() + Return(s.mockClaimableBalanceBatchInsertBuilder) - s.mockClaimantsBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() - s.mockClaimableBalanceBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() + s.mockClaimantsBatchInsertBuilder.On("Exec", s.ctx).Return(nil) + s.mockClaimableBalanceBatchInsertBuilder.On("Exec", s.ctx).Return(nil) s.processor = NewClaimableBalancesChangeProcessor(s.mockQ, s.session) } From 64221094837531c22bcf16d2525abc32ff438dda Mon Sep 17 00:00:00 2001 From: Urvi Date: Tue, 7 Nov 2023 10:11:29 -0800 Subject: [PATCH 11/14] Addressing review comments --- .../actions/claimable_balance_test.go | 20 +++++++-------- .../claimable_balance_batch_insert_builder.go | 4 +-- ...e_balance_claimant_batch_insert_builder.go | 4 +-- .../db2/history/claimable_balances.go | 5 ++-- .../db2/history/claimable_balances_test.go | 20 +++++++-------- .../db2/history/mock_q_claimable_balances.go | 9 +++---- .../internal/ingest/processor_runner_test.go | 25 +++++++++---------- .../claimable_balances_change_processor.go | 4 +-- ...laimable_balances_change_processor_test.go | 10 +++----- 9 files changed, 48 insertions(+), 53 deletions(-) diff --git a/services/horizon/internal/actions/claimable_balance_test.go b/services/horizon/internal/actions/claimable_balance_test.go index b645cc21d3..a5052e86da 100644 --- a/services/horizon/internal/actions/claimable_balance_test.go +++ b/services/horizon/internal/actions/claimable_balance_test.go @@ -19,11 +19,11 @@ func TestGetClaimableBalanceByID(t *testing.T) { tt := test.Start(t) defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) - q := &history.Q{tt.HorizonSession()} + q := &history.Q{SessionInterface: tt.HorizonSession()} - tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) + tt.Assert.NoError(q.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { - _ = q.SessionInterface.Rollback() + _ = q.Rollback() }() accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" @@ -49,10 +49,10 @@ func TestGetClaimableBalanceByID(t *testing.T) { LastModifiedLedger: 123, } - balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(q.SessionInterface) + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() for _, claimant := range cBalance.Claimants { claimant := history.ClaimableBalanceClaimant{ BalanceID: cBalance.BalanceID, @@ -156,7 +156,7 @@ func TestGetClaimableBalances(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{tt.HorizonSession()} - tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) + tt.Assert.NoError(q.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { _ = q.SessionInterface.Rollback() }() @@ -200,9 +200,9 @@ func TestGetClaimableBalances(t *testing.T) { hCBs = append(hCBs, cb) } - balanceInsertbuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) + balanceInsertbuilder := q.NewClaimableBalanceBatchInsertBuilder() - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(q.SessionInterface) + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() for _, cBalance := range hCBs { tt.Assert.NoError(balanceInsertbuilder.Add(cBalance)) @@ -298,8 +298,8 @@ func TestGetClaimableBalances(t *testing.T) { tt.Assert.Len(response, 0) // new claimable balances are ingested, they should appear in the next pages - balanceInsertbuilder = q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) - claimantsInsertBuilder = q.NewClaimableBalanceClaimantBatchInsertBuilder(q.SessionInterface) + balanceInsertbuilder = q.NewClaimableBalanceBatchInsertBuilder() + claimantsInsertBuilder = q.NewClaimableBalanceClaimantBatchInsertBuilder() entriesMeta = []struct { id xdr.Hash diff --git a/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go b/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go index 70105923a1..515cc1643c 100644 --- a/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/claimable_balance_batch_insert_builder.go @@ -21,9 +21,9 @@ type claimableBalanceBatchInsertBuilder struct { } // NewClaimableBalanceBatchInsertBuilder constructs a new ClaimableBalanceBatchInsertBuilder instance -func (q *Q) NewClaimableBalanceBatchInsertBuilder(session db.SessionInterface) ClaimableBalanceBatchInsertBuilder { +func (q *Q) NewClaimableBalanceBatchInsertBuilder() ClaimableBalanceBatchInsertBuilder { return &claimableBalanceBatchInsertBuilder{ - session: session, + session: q, builder: db.FastBatchInsertBuilder{}, table: "claimable_balances", } diff --git a/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go b/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go index 1e87e6e181..2f4dc6733c 100644 --- a/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/claimable_balance_claimant_batch_insert_builder.go @@ -21,9 +21,9 @@ type claimableBalanceClaimantBatchInsertBuilder struct { } // NewClaimableBalanceClaimantBatchInsertBuilder constructs a new ClaimableBalanceClaimantBatchInsertBuilder instance -func (q *Q) NewClaimableBalanceClaimantBatchInsertBuilder(session db.SessionInterface) ClaimableBalanceClaimantBatchInsertBuilder { +func (q *Q) NewClaimableBalanceClaimantBatchInsertBuilder() ClaimableBalanceClaimantBatchInsertBuilder { return &claimableBalanceClaimantBatchInsertBuilder{ - session: session, + session: q, builder: db.FastBatchInsertBuilder{}, table: "claimable_balance_claimants", } diff --git a/services/horizon/internal/db2/history/claimable_balances.go b/services/horizon/internal/db2/history/claimable_balances.go index 24e88cb512..5490bef11c 100644 --- a/services/horizon/internal/db2/history/claimable_balances.go +++ b/services/horizon/internal/db2/history/claimable_balances.go @@ -11,7 +11,6 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/guregu/null" "github.com/stellar/go/services/horizon/internal/db2" - "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) @@ -135,8 +134,8 @@ type QClaimableBalances interface { RemoveClaimableBalanceClaimants(ctx context.Context, ids []string) (int64, error) GetClaimableBalancesByID(ctx context.Context, ids []string) ([]ClaimableBalance, error) CountClaimableBalances(ctx context.Context) (int, error) - NewClaimableBalanceClaimantBatchInsertBuilder(session db.SessionInterface) ClaimableBalanceClaimantBatchInsertBuilder - NewClaimableBalanceBatchInsertBuilder(session db.SessionInterface) ClaimableBalanceBatchInsertBuilder + NewClaimableBalanceClaimantBatchInsertBuilder() ClaimableBalanceClaimantBatchInsertBuilder + NewClaimableBalanceBatchInsertBuilder() ClaimableBalanceBatchInsertBuilder GetClaimantsByClaimableBalances(ctx context.Context, ids []string) (map[string][]ClaimableBalanceClaimant, error) } diff --git a/services/horizon/internal/db2/history/claimable_balances_test.go b/services/horizon/internal/db2/history/claimable_balances_test.go index 217281bfb5..96310016c4 100644 --- a/services/horizon/internal/db2/history/claimable_balances_test.go +++ b/services/horizon/internal/db2/history/claimable_balances_test.go @@ -44,7 +44,7 @@ func TestRemoveClaimableBalance(t *testing.T) { Amount: 10, } - balanceBatchInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) + balanceBatchInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() tt.Assert.NoError(balanceBatchInsertBuilder.Add(cBalance)) tt.Assert.NoError(balanceBatchInsertBuilder.Exec(tt.Ctx)) @@ -97,7 +97,7 @@ func TestRemoveClaimableBalanceClaimants(t *testing.T) { Amount: 10, } - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(q.SessionInterface) + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() tt.Assert.NoError(insertClaimants(claimantsInsertBuilder, cBalance)) tt.Assert.NoError(claimantsInsertBuilder.Exec(tt.Ctx)) @@ -112,9 +112,9 @@ func TestFindClaimableBalancesByDestination(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) + tt.Assert.NoError(q.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { - _ = q.SessionInterface.Rollback() + _ = q.Rollback() }() dest1 := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" @@ -142,8 +142,8 @@ func TestFindClaimableBalancesByDestination(t *testing.T) { Amount: 10, } - balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(q.SessionInterface) + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) tt.Assert.NoError(insertClaimants(claimantsInsertBuilder, cBalance)) @@ -308,8 +308,8 @@ func TestFindClaimableBalancesByDestinationWithLimit(t *testing.T) { LastModifiedLedger: 123, Amount: 10, } - balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) - claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder(q.SessionInterface) + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() + claimantsInsertBuilder := q.NewClaimableBalanceClaimantBatchInsertBuilder() tt.Assert.NoError(balanceInsertBuilder.Add(cBalance1)) tt.Assert.NoError(insertClaimants(claimantsInsertBuilder, cBalance1)) @@ -439,7 +439,7 @@ func TestFindClaimableBalance(t *testing.T) { Amount: 10, } - balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx)) @@ -488,7 +488,7 @@ func TestGetClaimableBalancesByID(t *testing.T) { Amount: 10, } - balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder(q.SessionInterface) + balanceInsertBuilder := q.NewClaimableBalanceBatchInsertBuilder() tt.Assert.NoError(balanceInsertBuilder.Add(cBalance)) tt.Assert.NoError(balanceInsertBuilder.Exec(tt.Ctx)) diff --git a/services/horizon/internal/db2/history/mock_q_claimable_balances.go b/services/horizon/internal/db2/history/mock_q_claimable_balances.go index f63a3e04e7..64b65cf1a3 100644 --- a/services/horizon/internal/db2/history/mock_q_claimable_balances.go +++ b/services/horizon/internal/db2/history/mock_q_claimable_balances.go @@ -3,7 +3,6 @@ package history import ( "context" - "github.com/stellar/go/support/db" "github.com/stretchr/testify/mock" ) @@ -32,13 +31,13 @@ func (m *MockQClaimableBalances) RemoveClaimableBalanceClaimants(ctx context.Con return a.Get(0).(int64), a.Error(1) } -func (m *MockQClaimableBalances) NewClaimableBalanceClaimantBatchInsertBuilder(session db.SessionInterface) ClaimableBalanceClaimantBatchInsertBuilder { - a := m.Called(session) +func (m *MockQClaimableBalances) NewClaimableBalanceClaimantBatchInsertBuilder() ClaimableBalanceClaimantBatchInsertBuilder { + a := m.Called() return a.Get(0).(ClaimableBalanceClaimantBatchInsertBuilder) } -func (m *MockQClaimableBalances) NewClaimableBalanceBatchInsertBuilder(session db.SessionInterface) ClaimableBalanceBatchInsertBuilder { - a := m.Called(session) +func (m *MockQClaimableBalances) NewClaimableBalanceBatchInsertBuilder() ClaimableBalanceBatchInsertBuilder { + a := m.Called() return a.Get(0).(ClaimableBalanceBatchInsertBuilder) } diff --git a/services/horizon/internal/ingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go index c97d30c423..edcdd8d97c 100644 --- a/services/horizon/internal/ingest/processor_runner_test.go +++ b/services/horizon/internal/ingest/processor_runner_test.go @@ -52,11 +52,11 @@ func TestProcessorRunnerRunHistoryArchiveIngestionGenesis(t *testing.T) { Return(mockAccountSignersBatchInsertBuilder).Once() mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} - q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). Return(mockClaimableBalanceBatchInsertBuilder).Twice() mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). Return(mockClaimantsBatchInsertBuilder).Twice() mockClaimantsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() @@ -130,11 +130,11 @@ func TestProcessorRunnerRunHistoryArchiveIngestionHistoryArchive(t *testing.T) { Return(mockAccountSignersBatchInsertBuilder).Once() mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} - q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). Return(mockClaimableBalanceBatchInsertBuilder).Twice() mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). Return(mockClaimantsBatchInsertBuilder).Twice() mockClaimantsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() @@ -178,11 +178,11 @@ func TestProcessorRunnerRunHistoryArchiveIngestionProtocolVersionNotSupported(t Return(mockAccountSignersBatchInsertBuilder).Once() mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} - q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). Return(mockClaimableBalanceBatchInsertBuilder).Once() mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). Return(mockClaimantsBatchInsertBuilder).Once() mockClaimantsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() @@ -213,7 +213,6 @@ func TestProcessorRunnerBuildChangeProcessor(t *testing.T) { ctx := context.Background() maxBatchSize := 100000 - mockSession := &db.MockSession{} q := &mockDBQ{} defer mock.AssertExpectationsForObjects(t, q) @@ -222,11 +221,11 @@ func TestProcessorRunnerBuildChangeProcessor(t *testing.T) { Return(&history.MockAccountSignersBatchInsertBuilder{}).Twice() mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} - q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). Return(mockClaimableBalanceBatchInsertBuilder).Twice() mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). Return(mockClaimantsBatchInsertBuilder).Twice() runner := ProcessorRunner{ @@ -355,11 +354,11 @@ func TestProcessorRunnerWithFilterEnabled(t *testing.T) { Return(mockTransactionsFilteredTmpBatchInsertBuilder) mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} - q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). Return(mockClaimableBalanceBatchInsertBuilder).Once() mockClaimantsBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). Return(mockClaimantsBatchInsertBuilder).Once() mockClaimantsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() @@ -547,11 +546,11 @@ func mockBatchBuilders(q *mockDBQ, mockSession *db.MockSession, ctx context.Cont mockClaimableBalanceClaimantBatchInsertBuilder := &history.MockClaimableBalanceClaimantBatchInsertBuilder{} mockClaimableBalanceClaimantBatchInsertBuilder.On("Exec", ctx).Return(nil) - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", mockSession). + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder"). Return(mockClaimableBalanceClaimantBatchInsertBuilder) mockClaimableBalanceBatchInsertBuilder := &history.MockClaimableBalanceBatchInsertBuilder{} - q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder", mockSession). + q.MockQClaimableBalances.On("NewClaimableBalanceBatchInsertBuilder"). Return(mockClaimableBalanceBatchInsertBuilder) mockClaimableBalanceBatchInsertBuilder.On("Exec", ctx).Return(nil) diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go index c98bde9821..0727b415b4 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go @@ -32,8 +32,8 @@ func NewClaimableBalancesChangeProcessor(Q history.QClaimableBalances, session d func (p *ClaimableBalancesChangeProcessor) reset() { p.cache = ingest.NewChangeCompactor() - p.claimantsInsertBuilder = p.qClaimableBalances.NewClaimableBalanceClaimantBatchInsertBuilder(p.session) - p.claimableBalanceInsertBuilder = p.qClaimableBalances.NewClaimableBalanceBatchInsertBuilder(p.session) + p.claimantsInsertBuilder = p.qClaimableBalances.NewClaimableBalanceClaimantBatchInsertBuilder() + p.claimableBalanceInsertBuilder = p.qClaimableBalances.NewClaimableBalanceBatchInsertBuilder() } func (p *ClaimableBalancesChangeProcessor) ProcessChange(ctx context.Context, change ingest.Change) error { diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go index ccf1a85de2..a6c70569c2 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go @@ -35,12 +35,11 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteState) SetupTest() { s.mockClaimableBalanceBatchInsertBuilder = &history.MockClaimableBalanceBatchInsertBuilder{} s.mockQ = &history.MockQClaimableBalances{} - s.session = &db.MockSession{} s.mockQ. - On("NewClaimableBalanceClaimantBatchInsertBuilder", s.session). + On("NewClaimableBalanceClaimantBatchInsertBuilder"). Return(s.mockClaimantsBatchInsertBuilder) s.mockQ. - On("NewClaimableBalanceBatchInsertBuilder", s.session). + On("NewClaimableBalanceBatchInsertBuilder"). Return(s.mockClaimableBalanceBatchInsertBuilder) s.mockClaimantsBatchInsertBuilder.On("Exec", s.ctx).Return(nil) @@ -134,12 +133,11 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) SetupTest() { s.mockClaimantsBatchInsertBuilder = &history.MockClaimableBalanceClaimantBatchInsertBuilder{} s.mockClaimableBalanceBatchInsertBuilder = &history.MockClaimableBalanceBatchInsertBuilder{} s.mockQ = &history.MockQClaimableBalances{} - s.session = &db.MockSession{} s.mockQ. - On("NewClaimableBalanceClaimantBatchInsertBuilder", &db.MockSession{}). + On("NewClaimableBalanceClaimantBatchInsertBuilder"). Return(s.mockClaimantsBatchInsertBuilder) s.mockQ. - On("NewClaimableBalanceBatchInsertBuilder", &db.MockSession{}). + On("NewClaimableBalanceBatchInsertBuilder"). Return(s.mockClaimableBalanceBatchInsertBuilder) s.mockClaimantsBatchInsertBuilder.On("Exec", s.ctx).Return(nil) From f30845ea51ac6e2e57eeeec40509114762ffe9aa Mon Sep 17 00:00:00 2001 From: Urvi Date: Tue, 7 Nov 2023 10:37:51 -0800 Subject: [PATCH 12/14] Addressing review comments --- .../actions/claimable_balance_test.go | 2 +- .../db2/history/claimable_balances_test.go | 20 +++++++++---------- .../horizon/internal/ingest/verify_test.go | 12 +++++------ 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/services/horizon/internal/actions/claimable_balance_test.go b/services/horizon/internal/actions/claimable_balance_test.go index a5052e86da..fa922b930b 100644 --- a/services/horizon/internal/actions/claimable_balance_test.go +++ b/services/horizon/internal/actions/claimable_balance_test.go @@ -158,7 +158,7 @@ func TestGetClaimableBalances(t *testing.T) { tt.Assert.NoError(q.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { - _ = q.SessionInterface.Rollback() + _ = q.Rollback() }() entriesMeta := []struct { diff --git a/services/horizon/internal/db2/history/claimable_balances_test.go b/services/horizon/internal/db2/history/claimable_balances_test.go index 96310016c4..ca32975c62 100644 --- a/services/horizon/internal/db2/history/claimable_balances_test.go +++ b/services/horizon/internal/db2/history/claimable_balances_test.go @@ -16,9 +16,9 @@ func TestRemoveClaimableBalance(t *testing.T) { defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) + tt.Assert.NoError(q.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { - _ = q.SessionInterface.Rollback() + _ = q.Rollback() }() accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" @@ -69,9 +69,9 @@ func TestRemoveClaimableBalanceClaimants(t *testing.T) { defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) + tt.Assert.NoError(q.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { - _ = q.SessionInterface.Rollback() + _ = q.Rollback() }() accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" @@ -264,9 +264,9 @@ func TestFindClaimableBalancesByDestinationWithLimit(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) + tt.Assert.NoError(q.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { - _ = q.SessionInterface.Rollback() + _ = q.Rollback() }() assetIssuer := "GA25GQLHJU3LPEJXEIAXK23AWEA5GWDUGRSHTQHDFT6HXHVMRULSQJUJ" @@ -411,9 +411,9 @@ func TestFindClaimableBalance(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) + tt.Assert.NoError(q.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { - _ = q.SessionInterface.Rollback() + _ = q.Rollback() }() accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" @@ -460,9 +460,9 @@ func TestGetClaimableBalancesByID(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) + tt.Assert.NoError(q.BeginTx(tt.Ctx, &sql.TxOptions{})) defer func() { - _ = q.SessionInterface.Rollback() + _ = q.Rollback() }() accountID := "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML" diff --git a/services/horizon/internal/ingest/verify_test.go b/services/horizon/internal/ingest/verify_test.go index d2137aff36..2d213f2a0e 100644 --- a/services/horizon/internal/ingest/verify_test.go +++ b/services/horizon/internal/ingest/verify_test.go @@ -271,10 +271,10 @@ func TestStateVerifierLockBusy(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{&db.Session{DB: tt.HorizonDB}} - tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) + tt.Assert.NoError(q.BeginTx(tt.Ctx, &sql.TxOptions{})) checkpointLedger := uint32(63) - changeProcessor := buildChangeProcessor(q, q.SessionInterface, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") + changeProcessor := buildChangeProcessor(q, q, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") gen := randxdr.NewGenerator() var changes []xdr.LedgerEntryChange @@ -293,7 +293,7 @@ func TestStateVerifierLockBusy(t *testing.T) { } tt.Assert.NoError(changeProcessor.Commit(tt.Ctx)) - tt.Assert.NoError(q.SessionInterface.Commit()) + tt.Assert.NoError(q.Commit()) q.UpdateLastLedgerIngest(tt.Ctx, checkpointLedger) @@ -328,10 +328,10 @@ func TestStateVerifier(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &history.Q{&db.Session{DB: tt.HorizonDB}} - tt.Assert.NoError(q.SessionInterface.BeginTx(tt.Ctx, &sql.TxOptions{})) + tt.Assert.NoError(q.BeginTx(tt.Ctx, &sql.TxOptions{})) checkpointLedger := uint32(63) - changeProcessor := buildChangeProcessor(q, q.SessionInterface, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") + changeProcessor := buildChangeProcessor(q, q, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") mockChangeReader := &ingest.MockChangeReader{} gen := randxdr.NewGenerator() @@ -360,7 +360,7 @@ func TestStateVerifier(t *testing.T) { tt.Assert.NoError(changeProcessor.Commit(tt.Ctx)) tt.Assert.Equal(len(xdr.LedgerEntryTypeMap), len(coverage)) - tt.Assert.NoError(q.SessionInterface.Commit()) + tt.Assert.NoError(q.Commit()) q.UpdateLastLedgerIngest(tt.Ctx, checkpointLedger) From d99c8169a8839bb382f36b3687b313e5f38d62ae Mon Sep 17 00:00:00 2001 From: Urvi Date: Tue, 7 Nov 2023 11:11:22 -0800 Subject: [PATCH 13/14] Files missed in previous commit --- services/horizon/internal/ingest/processor_runner.go | 2 +- .../ingest/processors/claimable_balances_change_processor.go | 5 +---- .../processors/claimable_balances_change_processor_test.go | 4 ++-- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/services/horizon/internal/ingest/processor_runner.go b/services/horizon/internal/ingest/processor_runner.go index 764ff9cb5c..2f46aa0471 100644 --- a/services/horizon/internal/ingest/processor_runner.go +++ b/services/horizon/internal/ingest/processor_runner.go @@ -128,7 +128,7 @@ func buildChangeProcessor( processors.NewAssetStatsProcessor(historyQ, networkPassphrase, useLedgerCache), processors.NewSignersProcessor(historyQ, useLedgerCache), processors.NewTrustLinesProcessor(historyQ), - processors.NewClaimableBalancesChangeProcessor(historyQ, session), + processors.NewClaimableBalancesChangeProcessor(historyQ), processors.NewLiquidityPoolsChangeProcessor(historyQ, ledgerSequence), }) } diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go index 0727b415b4..5d985e691e 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor.go @@ -6,7 +6,6 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) @@ -17,14 +16,12 @@ type ClaimableBalancesChangeProcessor struct { cache *ingest.ChangeCompactor claimantsInsertBuilder history.ClaimableBalanceClaimantBatchInsertBuilder claimableBalanceInsertBuilder history.ClaimableBalanceBatchInsertBuilder - session db.SessionInterface } -func NewClaimableBalancesChangeProcessor(Q history.QClaimableBalances, session db.SessionInterface) *ClaimableBalancesChangeProcessor { +func NewClaimableBalancesChangeProcessor(Q history.QClaimableBalances) *ClaimableBalancesChangeProcessor { p := &ClaimableBalancesChangeProcessor{ encodingBuffer: xdr.NewEncodingBuffer(), qClaimableBalances: Q, - session: session, } p.reset() return p diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go index a6c70569c2..2a340434f5 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go @@ -45,7 +45,7 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteState) SetupTest() { s.mockClaimantsBatchInsertBuilder.On("Exec", s.ctx).Return(nil) s.mockClaimableBalanceBatchInsertBuilder.On("Exec", s.ctx).Return(nil) - s.processor = NewClaimableBalancesChangeProcessor(s.mockQ, s.session) + s.processor = NewClaimableBalancesChangeProcessor(s.mockQ) } func (s *ClaimableBalancesChangeProcessorTestSuiteState) TearDownTest() { @@ -143,7 +143,7 @@ func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) SetupTest() { s.mockClaimantsBatchInsertBuilder.On("Exec", s.ctx).Return(nil) s.mockClaimableBalanceBatchInsertBuilder.On("Exec", s.ctx).Return(nil) - s.processor = NewClaimableBalancesChangeProcessor(s.mockQ, s.session) + s.processor = NewClaimableBalancesChangeProcessor(s.mockQ) } func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) TearDownTest() { From 527172d8a7f31d9f1d5c3335ccb65fc18ea2b03c Mon Sep 17 00:00:00 2001 From: Urvi Date: Tue, 7 Nov 2023 12:22:17 -0800 Subject: [PATCH 14/14] Fix linter warnings --- services/horizon/internal/ingest/processor_runner.go | 3 --- services/horizon/internal/ingest/processor_runner_test.go | 4 ++-- .../processors/claimable_balances_change_processor_test.go | 3 --- services/horizon/internal/ingest/verify_test.go | 4 ++-- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/services/horizon/internal/ingest/processor_runner.go b/services/horizon/internal/ingest/processor_runner.go index 2f46aa0471..cff8960c1d 100644 --- a/services/horizon/internal/ingest/processor_runner.go +++ b/services/horizon/internal/ingest/processor_runner.go @@ -109,7 +109,6 @@ func (s *ProcessorRunner) DisableMemoryStatsLogging() { func buildChangeProcessor( historyQ history.IngestionQ, - session db.SessionInterface, changeStats *ingest.StatsChangeProcessor, source ingestionSource, ledgerSequence uint32, @@ -240,7 +239,6 @@ func (s *ProcessorRunner) RunHistoryArchiveIngestion( changeStats := ingest.StatsChangeProcessor{} changeProcessor := buildChangeProcessor( s.historyQ, - s.session, &changeStats, historyArchiveSource, checkpointLedger, @@ -406,7 +404,6 @@ func (s *ProcessorRunner) RunAllProcessorsOnLedger(ledger xdr.LedgerCloseMeta) ( groupChangeProcessors := buildChangeProcessor( s.historyQ, - s.session, &changeStatsProcessor, ledgerSource, ledger.LedgerSequence(), diff --git a/services/horizon/internal/ingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go index edcdd8d97c..63b8caa161 100644 --- a/services/horizon/internal/ingest/processor_runner_test.go +++ b/services/horizon/internal/ingest/processor_runner_test.go @@ -235,7 +235,7 @@ func TestProcessorRunnerBuildChangeProcessor(t *testing.T) { } stats := &ingest.StatsChangeProcessor{} - processor := buildChangeProcessor(runner.historyQ, &db.MockSession{}, stats, ledgerSource, 123, "") + processor := buildChangeProcessor(runner.historyQ, stats, ledgerSource, 123, "") assert.IsType(t, &groupChangeProcessors{}, processor) assert.IsType(t, &statsChangeProcessor{}, processor.processors[0]) @@ -256,7 +256,7 @@ func TestProcessorRunnerBuildChangeProcessor(t *testing.T) { filters: &MockFilters{}, } - processor = buildChangeProcessor(runner.historyQ, &db.MockSession{}, stats, historyArchiveSource, 456, "") + processor = buildChangeProcessor(runner.historyQ, stats, historyArchiveSource, 456, "") assert.IsType(t, &groupChangeProcessors{}, processor) assert.IsType(t, &statsChangeProcessor{}, processor.processors[0]) diff --git a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go index 2a340434f5..524de095f7 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_change_processor_test.go @@ -10,7 +10,6 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" - "github.com/stellar/go/support/db" "github.com/stellar/go/xdr" "github.com/stretchr/testify/suite" ) @@ -26,7 +25,6 @@ type ClaimableBalancesChangeProcessorTestSuiteState struct { mockQ *history.MockQClaimableBalances mockClaimantsBatchInsertBuilder *history.MockClaimableBalanceClaimantBatchInsertBuilder mockClaimableBalanceBatchInsertBuilder *history.MockClaimableBalanceBatchInsertBuilder - session db.SessionInterface } func (s *ClaimableBalancesChangeProcessorTestSuiteState) SetupTest() { @@ -125,7 +123,6 @@ type ClaimableBalancesChangeProcessorTestSuiteLedger struct { mockQ *history.MockQClaimableBalances mockClaimantsBatchInsertBuilder *history.MockClaimableBalanceClaimantBatchInsertBuilder mockClaimableBalanceBatchInsertBuilder *history.MockClaimableBalanceBatchInsertBuilder - session db.SessionInterface } func (s *ClaimableBalancesChangeProcessorTestSuiteLedger) SetupTest() { diff --git a/services/horizon/internal/ingest/verify_test.go b/services/horizon/internal/ingest/verify_test.go index 2d213f2a0e..1fbb6b8f8c 100644 --- a/services/horizon/internal/ingest/verify_test.go +++ b/services/horizon/internal/ingest/verify_test.go @@ -274,7 +274,7 @@ func TestStateVerifierLockBusy(t *testing.T) { tt.Assert.NoError(q.BeginTx(tt.Ctx, &sql.TxOptions{})) checkpointLedger := uint32(63) - changeProcessor := buildChangeProcessor(q, q, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") + changeProcessor := buildChangeProcessor(q, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") gen := randxdr.NewGenerator() var changes []xdr.LedgerEntryChange @@ -331,7 +331,7 @@ func TestStateVerifier(t *testing.T) { tt.Assert.NoError(q.BeginTx(tt.Ctx, &sql.TxOptions{})) checkpointLedger := uint32(63) - changeProcessor := buildChangeProcessor(q, q, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") + changeProcessor := buildChangeProcessor(q, &ingest.StatsChangeProcessor{}, ledgerSource, checkpointLedger, "") mockChangeReader := &ingest.MockChangeReader{} gen := randxdr.NewGenerator()