diff --git a/services/horizon/docker/docker-compose.yml b/services/horizon/docker/docker-compose.yml index 309df946a1..dbaa8994fe 100644 --- a/services/horizon/docker/docker-compose.yml +++ b/services/horizon/docker/docker-compose.yml @@ -2,7 +2,7 @@ version: '3' services: horizon-postgres: platform: linux/amd64 - image: postgres:12-bullseye + image: postgres:postgres:12-bullseye restart: on-failure environment: - POSTGRES_HOST_AUTH_METHOD=trust diff --git a/services/horizon/docker/verify-range/Dockerfile b/services/horizon/docker/verify-range/Dockerfile index 56f8c84796..0143dd2cfa 100644 --- a/services/horizon/docker/verify-range/Dockerfile +++ b/services/horizon/docker/verify-range/Dockerfile @@ -6,7 +6,7 @@ ENV STELLAR_CORE_VERSION=${STELLAR_CORE_VERSION:-*} ENV DEBIAN_FRONTEND=noninteractive ADD dependencies / -RUN ["chmod", "+x", "dependencies"] +RUN ["chmod", "+x", "/dependencies"] RUN /dependencies ADD stellar-core.cfg / diff --git a/services/horizon/internal/action_offers_test.go b/services/horizon/internal/action_offers_test.go index 13458db9fe..464757c94e 100644 --- a/services/horizon/internal/action_offers_test.go +++ b/services/horizon/internal/action_offers_test.go @@ -24,7 +24,9 @@ func TestOfferActions_Show(t *testing.T) { ht.Assert.NoError(err) ledgerCloseTime := time.Now().Unix() - _, err = q.InsertLedger(ctx, xdr.LedgerHeaderHistoryEntry{ + ht.Assert.NoError(q.Begin(ctx)) + ledgerBatch := q.NewLedgerBatchInsertBuilder() + err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ LedgerSeq: 100, ScpValue: xdr.StellarValue{ @@ -33,6 +35,8 @@ func TestOfferActions_Show(t *testing.T) { }, }, 0, 0, 0, 0, 0) ht.Assert.NoError(err) + ht.Assert.NoError(ledgerBatch.Exec(ht.Ctx, q)) + ht.Assert.NoError(q.Commit()) issuer := xdr.MustAddress("GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H") nativeAsset := xdr.MustNewNativeAsset() diff --git a/services/horizon/internal/actions/account_test.go b/services/horizon/internal/actions/account_test.go index ed3d529575..b8c9d0d03b 100644 --- a/services/horizon/internal/actions/account_test.go +++ b/services/horizon/internal/actions/account_test.go @@ -228,7 +228,9 @@ func TestAccountInfo(t *testing.T) { assert.NoError(t, err) ledgerFourCloseTime := time.Now().Unix() - _, err = q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{ + assert.NoError(t, q.Begin(tt.Ctx)) + ledgerBatch := q.NewLedgerBatchInsertBuilder() + err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ LedgerSeq: 4, ScpValue: xdr.StellarValue{ @@ -237,6 +239,8 @@ func TestAccountInfo(t *testing.T) { }, }, 0, 0, 0, 0, 0) assert.NoError(t, err) + assert.NoError(t, ledgerBatch.Exec(tt.Ctx, q)) + assert.NoError(t, q.Commit()) account, err := AccountInfo(tt.Ctx, &history.Q{tt.HorizonSession()}, accountID) tt.Assert.NoError(err) @@ -408,7 +412,9 @@ func TestGetAccountsHandlerPageResultsByAsset(t *testing.T) { err := q.UpsertAccounts(tt.Ctx, []history.AccountEntry{account1, account2}) assert.NoError(t, err) ledgerCloseTime := time.Now().Unix() - _, err = q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{ + assert.NoError(t, q.Begin(tt.Ctx)) + ledgerBatch := q.NewLedgerBatchInsertBuilder() + err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ LedgerSeq: 1234, ScpValue: xdr.StellarValue{ @@ -417,6 +423,8 @@ func TestGetAccountsHandlerPageResultsByAsset(t *testing.T) { }, }, 0, 0, 0, 0, 0) assert.NoError(t, err) + assert.NoError(t, ledgerBatch.Exec(tt.Ctx, q)) + assert.NoError(t, q.Commit()) for _, row := range accountSigners { _, err = q.CreateAccountSigner(tt.Ctx, row.Account, row.Signer, row.Weight, nil) @@ -511,7 +519,9 @@ func TestGetAccountsHandlerPageResultsByLiquidityPool(t *testing.T) { assert.NoError(t, err) ledgerCloseTime := time.Now().Unix() - _, err = q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{ + assert.NoError(t, q.Begin(tt.Ctx)) + ledgerBatch := q.NewLedgerBatchInsertBuilder() + err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ LedgerSeq: 1234, ScpValue: xdr.StellarValue{ @@ -520,6 +530,9 @@ func TestGetAccountsHandlerPageResultsByLiquidityPool(t *testing.T) { }, }, 0, 0, 0, 0, 0) assert.NoError(t, err) + assert.NoError(t, ledgerBatch.Exec(tt.Ctx, q)) + assert.NoError(t, q.Commit()) + var assetType, code, issuer string usd.MustExtract(&assetType, &code, &issuer) params := map[string]string{ diff --git a/services/horizon/internal/actions/offer_test.go b/services/horizon/internal/actions/offer_test.go index 41663c65d5..091dfba14c 100644 --- a/services/horizon/internal/actions/offer_test.go +++ b/services/horizon/internal/actions/offer_test.go @@ -79,7 +79,9 @@ func TestGetOfferByIDHandler(t *testing.T) { handler := GetOfferByID{} ledgerCloseTime := time.Now().Unix() - _, err := q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{ + assert.NoError(t, q.Begin(tt.Ctx)) + ledgerBatch := q.NewLedgerBatchInsertBuilder() + err := ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ LedgerSeq: 3, ScpValue: xdr.StellarValue{ @@ -87,7 +89,9 @@ func TestGetOfferByIDHandler(t *testing.T) { }, }, }, 0, 0, 0, 0, 0) - tt.Assert.NoError(err) + assert.NoError(t, err) + assert.NoError(t, ledgerBatch.Exec(tt.Ctx, q)) + assert.NoError(t, q.Commit()) err = q.UpsertOffers(tt.Ctx, []history.Offer{eurOffer, usdOffer}) tt.Assert.NoError(err) @@ -186,7 +190,9 @@ func TestGetOffersHandler(t *testing.T) { handler := GetOffersHandler{} ledgerCloseTime := time.Now().Unix() - _, err := q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{ + assert.NoError(t, q.Begin(tt.Ctx)) + ledgerBatch := q.NewLedgerBatchInsertBuilder() + err := ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ LedgerSeq: 3, ScpValue: xdr.StellarValue{ @@ -194,7 +200,9 @@ func TestGetOffersHandler(t *testing.T) { }, }, }, 0, 0, 0, 0, 0) - tt.Assert.NoError(err) + assert.NoError(t, err) + assert.NoError(t, ledgerBatch.Exec(tt.Ctx, q)) + assert.NoError(t, q.Commit()) err = q.UpsertOffers(tt.Ctx, []history.Offer{eurOffer, twoEurOffer, usdOffer}) tt.Assert.NoError(err) diff --git a/services/horizon/internal/actions/operation_test.go b/services/horizon/internal/actions/operation_test.go index 099f8d1886..9aa033fd24 100644 --- a/services/horizon/internal/actions/operation_test.go +++ b/services/horizon/internal/actions/operation_test.go @@ -36,17 +36,21 @@ func TestInvokeHostFnDetailsInPaymentOperations(t *testing.T) { opID1 := toid.New(sequence, txIndex, 1).ToInt64() ledgerCloseTime := time.Now().Unix() - _, err := q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{ - Header: xdr.LedgerHeader{ - LedgerSeq: xdr.Uint32(sequence), - ScpValue: xdr.StellarValue{ - CloseTime: xdr.TimePoint(ledgerCloseTime), + ledgerBatch := q.NewLedgerBatchInsertBuilder() + err := ledgerBatch.Add( + xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(sequence), + ScpValue: xdr.StellarValue{ + CloseTime: xdr.TimePoint(ledgerCloseTime), + }, }, - }, - }, 1, 0, 1, 0, 0) + }, 1, 0, 1, 0, 0) tt.Assert.NoError(err) + tt.Assert.NoError(q.Begin(tt.Ctx)) + tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q)) - transactionBuilder := q.NewTransactionBatchInsertBuilder(1) + transactionBuilder := q.NewTransactionBatchInsertBuilder() firstTransaction := buildLedgerTransaction(tt.T, testTransaction{ index: uint32(txIndex), envelopeXDR: "AAAAACiSTRmpH6bHC6Ekna5e82oiGY5vKDEEUgkq9CB//t+rAAAAyAEXUhsAADDRAAAAAAAAAAAAAAABAAAAAAAAAAsBF1IbAABX4QAAAAAAAAAA", @@ -55,11 +59,13 @@ func TestInvokeHostFnDetailsInPaymentOperations(t *testing.T) { metaXDR: "AAAAAQAAAAAAAAAA", hash: "19aaa18db88605aedec04659fb45e06f240b022eb2d429e05133e4d53cd945ba", }) - err = transactionBuilder.Add(tt.Ctx, firstTransaction, uint32(sequence)) + err = transactionBuilder.Add(firstTransaction, uint32(sequence)) tt.Assert.NoError(err) + tt.Assert.NoError(transactionBuilder.Exec(tt.Ctx, q)) + + operationBuilder := q.NewOperationBatchInsertBuilder() - operationBuilder := q.NewOperationBatchInsertBuilder(1) - err = operationBuilder.Add(tt.Ctx, + err = operationBuilder.Add( opID1, txID, 1, @@ -118,6 +124,8 @@ func TestInvokeHostFnDetailsInPaymentOperations(t *testing.T) { null.String{}, true) tt.Assert.NoError(err) + tt.Assert.NoError(operationBuilder.Exec(tt.Ctx, q)) + tt.Assert.NoError(q.Commit()) records, err := handler.GetResourcePage( httptest.NewRecorder(), diff --git a/services/horizon/internal/actions/transaction_test.go b/services/horizon/internal/actions/transaction_test.go index e029edef3a..b76cf1b0bf 100644 --- a/services/horizon/internal/actions/transaction_test.go +++ b/services/horizon/internal/actions/transaction_test.go @@ -149,6 +149,7 @@ func checkOuterHashResponse( } func TestFeeBumpTransactionPage(t *testing.T) { + tt := test.Start(t) defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) diff --git a/services/horizon/internal/actions_account_test.go b/services/horizon/internal/actions_account_test.go index 541300c3a6..1fa2b12b14 100644 --- a/services/horizon/internal/actions_account_test.go +++ b/services/horizon/internal/actions_account_test.go @@ -18,12 +18,17 @@ func TestAccountActions_InvalidID(t *testing.T) { ht.Assert.NoError(err) err = q.UpdateIngestVersion(ht.Ctx, ingest.CurrentVersion) ht.Assert.NoError(err) - _, err = q.InsertLedger(ht.Ctx, xdr.LedgerHeaderHistoryEntry{ + + ht.Assert.NoError(q.Begin(ht.Ctx)) + ledgerBatch := q.NewLedgerBatchInsertBuilder() + err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ LedgerSeq: 100, }, }, 0, 0, 0, 0, 0) ht.Assert.NoError(err) + ht.Assert.NoError(ledgerBatch.Exec(ht.Ctx, q)) + ht.Assert.NoError(q.Commit()) // existing account w := ht.Get( diff --git a/services/horizon/internal/actions_data_test.go b/services/horizon/internal/actions_data_test.go index 3de82a915d..1142a4000a 100644 --- a/services/horizon/internal/actions_data_test.go +++ b/services/horizon/internal/actions_data_test.go @@ -44,12 +44,16 @@ func TestDataActions_Show(t *testing.T) { ht.Assert.NoError(err) err = q.UpdateIngestVersion(ht.Ctx, ingest.CurrentVersion) ht.Assert.NoError(err) - _, err = q.InsertLedger(ht.Ctx, xdr.LedgerHeaderHistoryEntry{ + ht.Assert.NoError(q.Begin(ht.Ctx)) + ledgerBatch := q.NewLedgerBatchInsertBuilder() + err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ LedgerSeq: 100, }, }, 0, 0, 0, 0, 0) ht.Assert.NoError(err) + ht.Assert.NoError(ledgerBatch.Exec(ht.Ctx, q)) + ht.Assert.NoError(q.Commit()) err = q.UpsertAccountData(ht.Ctx, []history.Data{data1, data2}) assert.NoError(t, err) diff --git a/services/horizon/internal/actions_trade_test.go b/services/horizon/internal/actions_trade_test.go index 1aa7b157fb..72991c5da2 100644 --- a/services/horizon/internal/actions_trade_test.go +++ b/services/horizon/internal/actions_trade_test.go @@ -820,8 +820,11 @@ func IngestTestTrade( return err } - batch := q.NewTradeBatchInsertBuilder(0) - batch.Add(ctx, history.InsertTrade{ + if err = q.Begin(ctx); err != nil { + return err + } + batch := q.NewTradeBatchInsertBuilder() + err = batch.Add(history.InsertTrade{ HistoryOperationID: opCounter, Order: 0, CounterAssetID: assets[assetBought.String()].ID, @@ -839,7 +842,10 @@ func IngestTestTrade( Type: history.OrderbookTradeType, }) - err = batch.Exec(ctx) + if err != nil { + return err + } + err = batch.Exec(ctx, q) if err != nil { return err } @@ -849,6 +855,10 @@ func IngestTestTrade( return err } + if err := q.Commit(); err != nil { + return err + } + return nil } diff --git a/services/horizon/internal/db2/history/account_loader.go b/services/horizon/internal/db2/history/account_loader.go new file mode 100644 index 0000000000..f3946b0448 --- /dev/null +++ b/services/horizon/internal/db2/history/account_loader.go @@ -0,0 +1,214 @@ +package history + +import ( + "context" + "database/sql/driver" + "fmt" + "sort" + "strings" + + "github.com/lib/pq" + + "github.com/stellar/go/support/collections/set" + "github.com/stellar/go/support/db" + "github.com/stellar/go/support/errors" + "github.com/stellar/go/support/ordered" +) + +// FutureAccountID represents a future history account. +// A FutureAccountID is created by an AccountLoader and +// the account id is available after calling Exec() on +// the AccountLoader. +type FutureAccountID struct { + address string + loader *AccountLoader +} + +const loaderLookupBatchSize = 50000 + +// Value implements the database/sql/driver Valuer interface. +func (a FutureAccountID) Value() (driver.Value, error) { + return a.loader.GetNow(a.address) +} + +// AccountLoader will map account addresses to their history +// account ids. If there is no existing mapping for a given address, +// the AccountLoader will insert into the history_accounts table to +// establish a mapping. +type AccountLoader struct { + sealed bool + set set.Set[string] + ids map[string]int64 +} + +var errSealed = errors.New("cannot register more entries to loader after calling Exec()") + +// NewAccountLoader will construct a new AccountLoader instance. +func NewAccountLoader() *AccountLoader { + return &AccountLoader{ + sealed: false, + set: set.Set[string]{}, + ids: map[string]int64{}, + } +} + +// GetFuture registers the given account address into the loader and +// returns a FutureAccountID which will hold the history account id for +// the address after Exec() is called. +func (a *AccountLoader) GetFuture(address string) FutureAccountID { + if a.sealed { + panic(errSealed) + } + + a.set.Add(address) + return FutureAccountID{ + address: address, + loader: a, + } +} + +// GetNow returns the history account id for the given address. +// GetNow should only be called on values which were registered by +// GetFuture() calls. Also, Exec() must be called before any GetNow +// call can succeed. +func (a *AccountLoader) GetNow(address string) (int64, error) { + if !a.sealed { + return 0, fmt.Errorf(`invalid account loader state, + Exec was not called yet to properly seal and resolve %v id`, address) + } + if internalID, ok := a.ids[address]; !ok { + return 0, fmt.Errorf(`account loader address %q was not found`, address) + } else { + return internalID, nil + } +} + +func (a *AccountLoader) lookupKeys(ctx context.Context, q *Q, addresses []string) error { + for i := 0; i < len(addresses); i += loaderLookupBatchSize { + end := ordered.Min(len(addresses), i+loaderLookupBatchSize) + + var accounts []Account + if err := q.AccountsByAddresses(ctx, &accounts, addresses[i:end]); err != nil { + return errors.Wrap(err, "could not select accounts") + } + + for _, account := range accounts { + a.ids[account.Address] = account.ID + } + } + return nil +} + +// Exec will look up all the history account ids for the addresses registered in the loader. +// If there are no history account ids for a given set of addresses, Exec will insert rows +// into the history_accounts table to establish a mapping between address and history account id. +func (a *AccountLoader) Exec(ctx context.Context, session db.SessionInterface) error { + a.sealed = true + if len(a.set) == 0 { + return nil + } + q := &Q{session} + addresses := make([]string, 0, len(a.set)) + for address := range a.set { + addresses = append(addresses, address) + } + + if err := a.lookupKeys(ctx, q, addresses); err != nil { + return err + } + + insert := 0 + for _, address := range addresses { + if _, ok := a.ids[address]; ok { + continue + } + addresses[insert] = address + insert++ + } + if insert == 0 { + return nil + } + addresses = addresses[:insert] + // sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock + // https://github.com/stellar/go/issues/2370 + sort.Strings(addresses) + + err := bulkInsert( + ctx, + q, + "history_accounts", + []string{"address"}, + []bulkInsertField{ + { + name: "address", + dbType: "character varying(64)", + objects: addresses, + }, + }, + ) + if err != nil { + return err + } + + return a.lookupKeys(ctx, q, addresses) +} + +type bulkInsertField struct { + name string + dbType string + objects []string +} + +func bulkInsert(ctx context.Context, q *Q, table string, conflictFields []string, fields []bulkInsertField) error { + unnestPart := make([]string, 0, len(fields)) + insertFieldsPart := make([]string, 0, len(fields)) + pqArrays := make([]interface{}, 0, len(fields)) + + for _, field := range fields { + unnestPart = append( + unnestPart, + fmt.Sprintf("unnest(?::%s[]) /* %s */", field.dbType, field.name), + ) + insertFieldsPart = append( + insertFieldsPart, + field.name, + ) + pqArrays = append( + pqArrays, + pq.Array(field.objects), + ) + } + + sql := ` + WITH r AS + (SELECT ` + strings.Join(unnestPart, ",") + `) + INSERT INTO ` + table + ` + (` + strings.Join(insertFieldsPart, ",") + `) + SELECT * from r + ON CONFLICT (` + strings.Join(conflictFields, ",") + `) DO NOTHING` + + _, err := q.ExecRaw( + context.WithValue(ctx, &db.QueryTypeContextKey, db.UpsertQueryType), + sql, + pqArrays..., + ) + return err +} + +// AccountLoaderStub is a stub wrapper around AccountLoader which allows +// you to manually configure the mapping of addresses to history account ids +type AccountLoaderStub struct { + Loader *AccountLoader +} + +// NewAccountLoaderStub returns a new AccountLoaderStub instance +func NewAccountLoaderStub() AccountLoaderStub { + return AccountLoaderStub{Loader: NewAccountLoader()} +} + +// Insert updates the wrapped AccountLoader so that the given account +// address is mapped to the provided history account id +func (a AccountLoaderStub) Insert(address string, id int64) { + a.Loader.sealed = true + a.Loader.ids[address] = id +} diff --git a/services/horizon/internal/db2/history/account_loader_test.go b/services/horizon/internal/db2/history/account_loader_test.go new file mode 100644 index 0000000000..54d2c7a143 --- /dev/null +++ b/services/horizon/internal/db2/history/account_loader_test.go @@ -0,0 +1,52 @@ +package history + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/stellar/go/keypair" + "github.com/stellar/go/services/horizon/internal/test" +) + +func TestAccountLoader(t *testing.T) { + tt := test.Start(t) + defer tt.Finish() + test.ResetHorizonDB(t, tt.HorizonDB) + session := tt.HorizonSession() + + var addresses []string + for i := 0; i < 100; i++ { + addresses = append(addresses, keypair.MustRandom().Address()) + } + + loader := NewAccountLoader() + for _, address := range addresses { + future := loader.GetFuture(address) + _, err := future.Value() + assert.Error(t, err) + assert.Contains(t, err.Error(), `invalid account loader state,`) + duplicateFuture := loader.GetFuture(address) + assert.Equal(t, future, duplicateFuture) + } + + assert.NoError(t, loader.Exec(context.Background(), session)) + assert.Panics(t, func() { + loader.GetFuture(keypair.MustRandom().Address()) + }) + + q := &Q{session} + for _, address := range addresses { + internalId, err := loader.GetNow(address) + assert.NoError(t, err) + var account Account + assert.NoError(t, q.AccountByAddress(context.Background(), &account, address)) + assert.Equal(t, account.ID, internalId) + assert.Equal(t, account.Address, address) + } + + _, err := loader.GetNow("not present") + assert.Error(t, err) + assert.Contains(t, err.Error(), `was not found`) +} diff --git a/services/horizon/internal/db2/history/asset_loader.go b/services/horizon/internal/db2/history/asset_loader.go new file mode 100644 index 0000000000..b5ee9a8326 --- /dev/null +++ b/services/horizon/internal/db2/history/asset_loader.go @@ -0,0 +1,219 @@ +package history + +import ( + "context" + "database/sql/driver" + "fmt" + "sort" + "strings" + + sq "github.com/Masterminds/squirrel" + + "github.com/stellar/go/support/collections/set" + "github.com/stellar/go/support/db" + "github.com/stellar/go/support/errors" + "github.com/stellar/go/support/ordered" + "github.com/stellar/go/xdr" +) + +type AssetKey struct { + Type string + Code string + Issuer string +} + +func (key AssetKey) String() string { + if key.Type == xdr.AssetTypeToString[xdr.AssetTypeAssetTypeNative] { + return key.Type + } + return key.Type + "/" + key.Code + "/" + key.Issuer +} + +// AssetKeyFromXDR constructs an AssetKey from an xdr asset +func AssetKeyFromXDR(asset xdr.Asset) AssetKey { + return AssetKey{ + Type: xdr.AssetTypeToString[asset.Type], + Code: strings.TrimRight(asset.GetCode(), "\x00"), + Issuer: asset.GetIssuer(), + } +} + +// FutureAssetID represents a future history asset. +// A FutureAssetID is created by an AssetLoader and +// the asset id is available after calling Exec() on +// the AssetLoader. +type FutureAssetID struct { + asset AssetKey + loader *AssetLoader +} + +// Value implements the database/sql/driver Valuer interface. +func (a FutureAssetID) Value() (driver.Value, error) { + return a.loader.GetNow(a.asset) +} + +// AssetLoader will map assets to their history +// asset ids. If there is no existing mapping for a given sset, +// the AssetLoader will insert into the history_assets table to +// establish a mapping. +type AssetLoader struct { + sealed bool + set set.Set[AssetKey] + ids map[AssetKey]int64 +} + +// NewAssetLoader will construct a new AssetLoader instance. +func NewAssetLoader() *AssetLoader { + return &AssetLoader{ + sealed: false, + set: set.Set[AssetKey]{}, + ids: map[AssetKey]int64{}, + } +} + +// GetFuture registers the given asset into the loader and +// returns a FutureAssetID which will hold the history asset id for +// the asset after Exec() is called. +func (a *AssetLoader) GetFuture(asset AssetKey) FutureAssetID { + if a.sealed { + panic(errSealed) + } + a.set.Add(asset) + return FutureAssetID{ + asset: asset, + loader: a, + } +} + +// GetNow returns the history asset id for the given asset. +// GetNow should only be called on values which were registered by +// GetFuture() calls. Also, Exec() must be called before any GetNow +// call can succeed. +func (a *AssetLoader) GetNow(asset AssetKey) (int64, error) { + if !a.sealed { + return 0, fmt.Errorf(`invalid asset loader state, + Exec was not called yet to properly seal and resolve %v id`, asset) + } + if internalID, ok := a.ids[asset]; !ok { + return 0, fmt.Errorf(`asset loader id %v was not found`, asset) + } else { + return internalID, nil + } +} + +func (a *AssetLoader) lookupKeys(ctx context.Context, q *Q, keys []AssetKey) error { + var rows []Asset + for i := 0; i < len(keys); i += loaderLookupBatchSize { + end := ordered.Min(len(keys), i+loaderLookupBatchSize) + subset := keys[i:end] + keyStrings := make([]string, 0, len(subset)) + for _, key := range subset { + keyStrings = append(keyStrings, key.Type+"/"+key.Code+"/"+key.Issuer) + } + err := q.Select(ctx, &rows, sq.Select("*").From("history_assets").Where(sq.Eq{ + "concat(asset_type, '/', asset_code, '/', asset_issuer)": keyStrings, + })) + if err != nil { + return errors.Wrap(err, "could not select assets") + } + + for _, row := range rows { + a.ids[AssetKey{ + Type: row.Type, + Code: row.Code, + Issuer: row.Issuer, + }] = row.ID + } + } + return nil +} + +// Exec will look up all the history asset ids for the assets registered in the loader. +// If there are no history asset ids for a given set of assets, Exec will insert rows +// into the history_assets table. +func (a *AssetLoader) Exec(ctx context.Context, session db.SessionInterface) error { + a.sealed = true + if len(a.set) == 0 { + return nil + } + q := &Q{session} + keys := make([]AssetKey, 0, len(a.set)) + for key := range a.set { + keys = append(keys, key) + } + + if err := a.lookupKeys(ctx, q, keys); err != nil { + return err + } + + assetTypes := make([]string, 0, len(a.set)-len(a.ids)) + assetCodes := make([]string, 0, len(a.set)-len(a.ids)) + assetIssuers := make([]string, 0, len(a.set)-len(a.ids)) + // sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock + // https://github.com/stellar/go/issues/2370 + sort.Slice(keys, func(i, j int) bool { + return keys[i].String() < keys[j].String() + }) + insert := 0 + for _, key := range keys { + if _, ok := a.ids[key]; ok { + continue + } + assetTypes = append(assetTypes, key.Type) + assetCodes = append(assetCodes, key.Code) + assetIssuers = append(assetIssuers, key.Issuer) + keys[insert] = key + insert++ + } + if insert == 0 { + return nil + } + keys = keys[:insert] + + err := bulkInsert( + ctx, + q, + "history_assets", + []string{"asset_code", "asset_type", "asset_issuer"}, + []bulkInsertField{ + { + name: "asset_code", + dbType: "character varying(12)", + objects: assetCodes, + }, + { + name: "asset_issuer", + dbType: "character varying(56)", + objects: assetIssuers, + }, + { + name: "asset_type", + dbType: "character varying(64)", + objects: assetTypes, + }, + }, + ) + if err != nil { + return err + } + + return a.lookupKeys(ctx, q, keys) +} + +// AssetLoaderStub is a stub wrapper around AssetLoader which allows +// you to manually configure the mapping of assets to history asset ids +type AssetLoaderStub struct { + Loader *AssetLoader +} + +// NewAssetLoaderStub returns a new AssetLoaderStub instance +func NewAssetLoaderStub() AssetLoaderStub { + return AssetLoaderStub{Loader: NewAssetLoader()} +} + +// Insert updates the wrapped AssetLoaderStub so that the given asset +// address is mapped to the provided history asset id +func (a AssetLoaderStub) Insert(asset AssetKey, id int64) { + a.Loader.sealed = true + a.Loader.ids[asset] = id +} diff --git a/services/horizon/internal/db2/history/asset_loader_test.go b/services/horizon/internal/db2/history/asset_loader_test.go new file mode 100644 index 0000000000..d67163d764 --- /dev/null +++ b/services/horizon/internal/db2/history/asset_loader_test.go @@ -0,0 +1,102 @@ +package history + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/stellar/go/keypair" + "github.com/stellar/go/services/horizon/internal/test" + "github.com/stellar/go/xdr" +) + +func TestAssetKeyToString(t *testing.T) { + num4key := AssetKey{ + Type: "credit_alphanum4", + Code: "USD", + Issuer: "A1B2C3", + } + + num12key := AssetKey{ + Type: "credit_alphanum12", + Code: "USDABC", + Issuer: "A1B2C3", + } + + nativekey := AssetKey{ + Type: "native", + } + + assert.Equal(t, num4key.String(), "credit_alphanum4/USD/A1B2C3") + assert.Equal(t, num12key.String(), "credit_alphanum12/USDABC/A1B2C3") + assert.Equal(t, nativekey.String(), "native") +} + +func TestAssetLoader(t *testing.T) { + tt := test.Start(t) + defer tt.Finish() + test.ResetHorizonDB(t, tt.HorizonDB) + session := tt.HorizonSession() + + var keys []AssetKey + for i := 0; i < 100; i++ { + var key AssetKey + if i == 0 { + key = AssetKeyFromXDR(xdr.Asset{Type: xdr.AssetTypeAssetTypeNative}) + } else if i%2 == 0 { + code := [4]byte{0, 0, 0, 0} + copy(code[:], fmt.Sprintf("ab%d", i)) + key = AssetKeyFromXDR(xdr.Asset{ + Type: xdr.AssetTypeAssetTypeCreditAlphanum4, + AlphaNum4: &xdr.AlphaNum4{ + AssetCode: code, + Issuer: xdr.MustAddress(keypair.MustRandom().Address())}}) + } else { + code := [12]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + copy(code[:], fmt.Sprintf("abcdef%d", i)) + key = AssetKeyFromXDR(xdr.Asset{ + Type: xdr.AssetTypeAssetTypeCreditAlphanum12, + AlphaNum12: &xdr.AlphaNum12{ + AssetCode: code, + Issuer: xdr.MustAddress(keypair.MustRandom().Address())}}) + + } + keys = append(keys, key) + } + + loader := NewAssetLoader() + for _, key := range keys { + future := loader.GetFuture(key) + _, err := future.Value() + assert.Error(t, err) + assert.Contains(t, err.Error(), `invalid asset loader state,`) + duplicateFuture := loader.GetFuture(key) + assert.Equal(t, future, duplicateFuture) + } + + assert.NoError(t, loader.Exec(context.Background(), session)) + assert.Panics(t, func() { + loader.GetFuture(AssetKey{Type: "invalid"}) + }) + + q := &Q{session} + for _, key := range keys { + internalID, err := loader.GetNow(key) + assert.NoError(t, err) + var assetXDR xdr.Asset + if key.Type == "native" { + assetXDR = xdr.MustNewNativeAsset() + } else { + assetXDR = xdr.MustNewCreditAsset(key.Code, key.Issuer) + } + assetID, err := q.GetAssetID(context.Background(), assetXDR) + assert.NoError(t, err) + assert.Equal(t, assetID, internalID) + } + + _, err := loader.GetNow(AssetKey{}) + assert.Error(t, err) + assert.Contains(t, err.Error(), `was not found`) +} diff --git a/services/horizon/internal/db2/history/claimable_balance_loader.go b/services/horizon/internal/db2/history/claimable_balance_loader.go new file mode 100644 index 0000000000..dd7dee4ea5 --- /dev/null +++ b/services/horizon/internal/db2/history/claimable_balance_loader.go @@ -0,0 +1,147 @@ +package history + +import ( + "context" + "database/sql/driver" + "fmt" + "sort" + + "github.com/stellar/go/support/collections/set" + "github.com/stellar/go/support/db" + "github.com/stellar/go/support/errors" + "github.com/stellar/go/support/ordered" +) + +// FutureClaimableBalanceID represents a future history claimable balance. +// A FutureClaimableBalanceID is created by a ClaimableBalanceLoader and +// the claimable balance id is available after calling Exec() on +// the ClaimableBalanceLoader. +type FutureClaimableBalanceID struct { + id string + loader *ClaimableBalanceLoader +} + +// Value implements the database/sql/driver Valuer interface. +func (a FutureClaimableBalanceID) Value() (driver.Value, error) { + return a.loader.getNow(a.id) +} + +// ClaimableBalanceLoader will map claimable balance ids to their internal +// history ids. If there is no existing mapping for a given claimable balance id, +// the ClaimableBalanceLoader will insert into the history_claimable_balances table to +// establish a mapping. +type ClaimableBalanceLoader struct { + sealed bool + set set.Set[string] + ids map[string]int64 +} + +// NewClaimableBalanceLoader will construct a new ClaimableBalanceLoader instance. +func NewClaimableBalanceLoader() *ClaimableBalanceLoader { + return &ClaimableBalanceLoader{ + sealed: false, + set: set.Set[string]{}, + ids: map[string]int64{}, + } +} + +// GetFuture registers the given claimable balance into the loader and +// returns a FutureClaimableBalanceID which will hold the internal history id for +// the claimable balance after Exec() is called. +func (a *ClaimableBalanceLoader) GetFuture(id string) FutureClaimableBalanceID { + if a.sealed { + panic(errSealed) + } + + a.set.Add(id) + return FutureClaimableBalanceID{ + id: id, + loader: a, + } +} + +// getNow returns the internal history id for the given claimable balance. +// getNow should only be called on values which were registered by +// GetFuture() calls. Also, Exec() must be called before any getNow +// call can succeed. +func (a *ClaimableBalanceLoader) getNow(id string) (int64, error) { + if !a.sealed { + return 0, fmt.Errorf(`invalid claimable balance loader state, + Exec was not called yet to properly seal and resolve %v id`, id) + } + if internalID, ok := a.ids[id]; !ok { + return 0, fmt.Errorf(`claimable balance loader id %q was not found`, id) + } else { + return internalID, nil + } +} + +func (a *ClaimableBalanceLoader) lookupKeys(ctx context.Context, q *Q, ids []string) error { + for i := 0; i < len(ids); i += loaderLookupBatchSize { + end := ordered.Min(len(ids), i+loaderLookupBatchSize) + + cbs, err := q.ClaimableBalancesByIDs(ctx, ids[i:end]) + if err != nil { + return errors.Wrap(err, "could not select claimable balances") + } + + for _, cb := range cbs { + a.ids[cb.BalanceID] = cb.InternalID + } + } + return nil +} + +// Exec will look up all the internal history ids for the claimable balances registered in the loader. +// If there are no internal ids for a given set of claimable balances, Exec will insert rows +// into the history_claimable_balances table. +func (a *ClaimableBalanceLoader) Exec(ctx context.Context, session db.SessionInterface) error { + a.sealed = true + if len(a.set) == 0 { + return nil + } + q := &Q{session} + ids := make([]string, 0, len(a.set)) + for id := range a.set { + ids = append(ids, id) + } + + if err := a.lookupKeys(ctx, q, ids); err != nil { + return err + } + + insert := 0 + for _, id := range ids { + if _, ok := a.ids[id]; ok { + continue + } + ids[insert] = id + insert++ + } + if insert == 0 { + return nil + } + ids = ids[:insert] + // sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock + // https://github.com/stellar/go/issues/2370 + sort.Strings(ids) + + err := bulkInsert( + ctx, + q, + "history_claimable_balances", + []string{"claimable_balance_id"}, + []bulkInsertField{ + { + name: "claimable_balance_id", + dbType: "text", + objects: ids, + }, + }, + ) + if err != nil { + return err + } + + return a.lookupKeys(ctx, q, ids) +} diff --git a/services/horizon/internal/db2/history/claimable_balance_loader_test.go b/services/horizon/internal/db2/history/claimable_balance_loader_test.go new file mode 100644 index 0000000000..4dd7324521 --- /dev/null +++ b/services/horizon/internal/db2/history/claimable_balance_loader_test.go @@ -0,0 +1,62 @@ +package history + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/stellar/go/services/horizon/internal/test" + "github.com/stellar/go/xdr" +) + +func TestClaimableBalanceLoader(t *testing.T) { + tt := test.Start(t) + defer tt.Finish() + test.ResetHorizonDB(t, tt.HorizonDB) + session := tt.HorizonSession() + + var ids []string + for i := 0; i < 100; i++ { + balanceID := xdr.ClaimableBalanceId{ + Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0, + V0: &xdr.Hash{byte(i)}, + } + id, err := xdr.MarshalHex(balanceID) + tt.Assert.NoError(err) + ids = append(ids, id) + } + + loader := NewClaimableBalanceLoader() + var futures []FutureClaimableBalanceID + for _, id := range ids { + future := loader.GetFuture(id) + futures = append(futures, future) + _, err := future.Value() + assert.Error(t, err) + assert.Contains(t, err.Error(), `invalid claimable balance loader state,`) + duplicateFuture := loader.GetFuture(id) + assert.Equal(t, future, duplicateFuture) + } + + assert.NoError(t, loader.Exec(context.Background(), session)) + assert.Panics(t, func() { + loader.GetFuture("not-present") + }) + + q := &Q{session} + for i, id := range ids { + future := futures[i] + internalID, err := future.Value() + assert.NoError(t, err) + cb, err := q.ClaimableBalanceByID(context.Background(), id) + assert.NoError(t, err) + assert.Equal(t, cb.BalanceID, id) + assert.Equal(t, cb.InternalID, internalID) + } + + futureCb := &FutureClaimableBalanceID{id: "not-present", loader: loader} + _, err := futureCb.Value() + assert.Error(t, err) + assert.Contains(t, err.Error(), `was not found`) +} diff --git a/services/horizon/internal/db2/history/effect.go b/services/horizon/internal/db2/history/effect.go index c905479c6c..13a9c52519 100644 --- a/services/horizon/internal/db2/history/effect.go +++ b/services/horizon/internal/db2/history/effect.go @@ -246,7 +246,7 @@ func (q *EffectsQ) Select(ctx context.Context, dest interface{}) error { // QEffects defines history_effects related queries. type QEffects interface { QCreateAccountsHistory - NewEffectBatchInsertBuilder(maxBatchSize int) EffectBatchInsertBuilder + NewEffectBatchInsertBuilder() EffectBatchInsertBuilder } var selectEffect = sq.Select("heff.*, hacc.address"). diff --git a/services/horizon/internal/db2/history/effect_batch_insert_builder.go b/services/horizon/internal/db2/history/effect_batch_insert_builder.go index 8b2522cf9e..e4ae333cb2 100644 --- a/services/horizon/internal/db2/history/effect_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/effect_batch_insert_builder.go @@ -4,6 +4,7 @@ import ( "context" "github.com/guregu/null" + "github.com/stellar/go/support/db" ) @@ -11,52 +12,49 @@ import ( // history_effects table type EffectBatchInsertBuilder interface { Add( - ctx context.Context, - accountID int64, + accountID FutureAccountID, muxedAccount null.String, operationID int64, order uint32, effectType EffectType, details []byte, ) error - Exec(ctx context.Context) error + Exec(ctx context.Context, session db.SessionInterface) error } // effectBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder type effectBatchInsertBuilder struct { - builder db.BatchInsertBuilder + table string + builder db.FastBatchInsertBuilder } // NewEffectBatchInsertBuilder constructs a new EffectBatchInsertBuilder instance -func (q *Q) NewEffectBatchInsertBuilder(maxBatchSize int) EffectBatchInsertBuilder { +func (q *Q) NewEffectBatchInsertBuilder() EffectBatchInsertBuilder { return &effectBatchInsertBuilder{ - builder: db.BatchInsertBuilder{ - Table: q.GetTable("history_effects"), - MaxBatchSize: maxBatchSize, - }, + table: "history_effects", + builder: db.FastBatchInsertBuilder{}, } } // Add adds a effect to the batch func (i *effectBatchInsertBuilder) Add( - ctx context.Context, - accountID int64, + accountID FutureAccountID, muxedAccount null.String, operationID int64, order uint32, effectType EffectType, details []byte, ) error { - return i.builder.Row(ctx, map[string]interface{}{ + return i.builder.Row(map[string]interface{}{ "history_account_id": accountID, "address_muxed": muxedAccount, "history_operation_id": operationID, - "\"order\"": order, + "order": order, "type": effectType, - "details": details, + "details": string(details), }) } -func (i *effectBatchInsertBuilder) Exec(ctx context.Context) error { - return i.builder.Exec(ctx) +func (i *effectBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + return i.builder.Exec(ctx, session, i.table) } diff --git a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go index bd57eb4414..dc02148a7d 100644 --- a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go +++ b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/guregu/null" + "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/toid" ) @@ -14,21 +15,21 @@ func TestAddEffect(t *testing.T) { defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} + tt.Assert.NoError(q.Begin(tt.Ctx)) address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY" muxedAddres := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26" - accounIDs, err := q.CreateAccounts(tt.Ctx, []string{address}, 1) - tt.Assert.NoError(err) + accountLoader := NewAccountLoader() - builder := q.NewEffectBatchInsertBuilder(2) + builder := q.NewEffectBatchInsertBuilder() sequence := int32(56) details, err := json.Marshal(map[string]string{ "amount": "1000.0000000", "asset_type": "native", }) - err = builder.Add(tt.Ctx, - accounIDs[address], + err = builder.Add( + accountLoader.GetFuture(address), null.StringFrom(muxedAddres), toid.New(sequence, 1, 1).ToInt64(), 1, @@ -37,8 +38,9 @@ func TestAddEffect(t *testing.T) { ) tt.Assert.NoError(err) - err = builder.Exec(tt.Ctx) - tt.Assert.NoError(err) + tt.Assert.NoError(accountLoader.Exec(tt.Ctx, q)) + tt.Assert.NoError(builder.Exec(tt.Ctx, q)) + tt.Assert.NoError(q.Commit()) effects := []Effect{} tt.Assert.NoError(q.Effects().Select(tt.Ctx, &effects)) diff --git a/services/horizon/internal/db2/history/effect_test.go b/services/horizon/internal/db2/history/effect_test.go index 6430049ab4..498d5e92df 100644 --- a/services/horizon/internal/db2/history/effect_test.go +++ b/services/horizon/internal/db2/history/effect_test.go @@ -6,6 +6,7 @@ import ( "testing" "github.com/guregu/null" + "github.com/stellar/go/protocols/horizon/effects" "github.com/stellar/go/services/horizon/internal/db2" "github.com/stellar/go/services/horizon/internal/test" @@ -17,45 +18,43 @@ func TestEffectsForLiquidityPool(t *testing.T) { defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} + tt.Assert.NoError(q.Begin(tt.Ctx)) // Insert Effect address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY" muxedAddres := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26" - accountIDs, err := q.CreateAccounts(tt.Ctx, []string{address}, 1) - tt.Assert.NoError(err) + accountLoader := NewAccountLoader() - builder := q.NewEffectBatchInsertBuilder(2) + builder := q.NewEffectBatchInsertBuilder() sequence := int32(56) details, err := json.Marshal(map[string]string{ "amount": "1000.0000000", "asset_type": "native", }) + tt.Assert.NoError(err) opID := toid.New(sequence, 1, 1).ToInt64() - err = builder.Add(tt.Ctx, - accountIDs[address], + tt.Assert.NoError(builder.Add( + accountLoader.GetFuture(address), null.StringFrom(muxedAddres), opID, 1, 3, details, - ) - tt.Assert.NoError(err) + )) - err = builder.Exec(tt.Ctx) - tt.Assert.NoError(err) + tt.Assert.NoError(accountLoader.Exec(tt.Ctx, q)) + tt.Assert.NoError(builder.Exec(tt.Ctx, q)) // Insert Liquidity Pool history liquidityPoolID := "abcde" - toInternalID, err := q.CreateHistoryLiquidityPools(tt.Ctx, []string{liquidityPoolID}, 2) - tt.Assert.NoError(err) - operationBuilder := q.NewOperationLiquidityPoolBatchInsertBuilder(2) - tt.Assert.NoError(err) - internalID, ok := toInternalID[liquidityPoolID] - tt.Assert.True(ok) - err = operationBuilder.Add(tt.Ctx, opID, internalID) - tt.Assert.NoError(err) - err = operationBuilder.Exec(tt.Ctx) - tt.Assert.NoError(err) + lpLoader := NewLiquidityPoolLoader() + + operationBuilder := q.NewOperationLiquidityPoolBatchInsertBuilder() + tt.Assert.NoError(operationBuilder.Add(opID, lpLoader.GetFuture(liquidityPoolID))) + tt.Assert.NoError(lpLoader.Exec(tt.Ctx, q)) + tt.Assert.NoError(operationBuilder.Exec(tt.Ctx, q)) + + tt.Assert.NoError(q.Commit()) var result []Effect err = q.Effects().ForLiquidityPool(tt.Ctx, db2.PageQuery{ @@ -75,13 +74,13 @@ func TestEffectsForTrustlinesSponsorshipEmptyAssetType(t *testing.T) { defer tt.Finish() test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} + tt.Assert.NoError(q.Begin(tt.Ctx)) address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY" muxedAddres := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26" - accountIDs, err := q.CreateAccounts(tt.Ctx, []string{address}, 1) - tt.Assert.NoError(err) + accountLoader := NewAccountLoader() - builder := q.NewEffectBatchInsertBuilder(1) + builder := q.NewEffectBatchInsertBuilder() sequence := int32(56) tests := []struct { effectType EffectType @@ -141,26 +140,24 @@ func TestEffectsForTrustlinesSponsorshipEmptyAssetType(t *testing.T) { for i, test := range tests { var bytes []byte - bytes, err = json.Marshal(test.details) + bytes, err := json.Marshal(test.details) tt.Require.NoError(err) - err = builder.Add(tt.Ctx, - accountIDs[address], + tt.Require.NoError(builder.Add( + accountLoader.GetFuture(address), null.StringFrom(muxedAddres), opID, uint32(i), test.effectType, bytes, - ) - tt.Require.NoError(err) + )) } - - err = builder.Exec(tt.Ctx) - tt.Require.NoError(err) + tt.Require.NoError(accountLoader.Exec(tt.Ctx, q)) + tt.Require.NoError(builder.Exec(tt.Ctx, q)) + tt.Assert.NoError(q.Commit()) var results []Effect - err = q.Effects().Select(tt.Ctx, &results) - tt.Require.NoError(err) + tt.Require.NoError(q.Effects().Select(tt.Ctx, &results)) tt.Require.Len(results, len(tests)) for i, test := range tests { diff --git a/services/horizon/internal/db2/history/fee_bump_scenario.go b/services/horizon/internal/db2/history/fee_bump_scenario.go index bdd28135ee..da6563c732 100644 --- a/services/horizon/internal/db2/history/fee_bump_scenario.go +++ b/services/horizon/internal/db2/history/fee_bump_scenario.go @@ -10,12 +10,13 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/guregu/null" + "github.com/stretchr/testify/assert" + "github.com/stellar/go/ingest" "github.com/stellar/go/network" "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/toid" "github.com/stellar/go/xdr" - "github.com/stretchr/testify/assert" ) func ledgerToMap(ledger Ledger) map[string]interface{} { @@ -247,29 +248,31 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture { hash: "edba3051b2f2d9b713e8a08709d631eccb72c59864ff3c564c68792271bb24a7", }) ctx := context.Background() - insertBuilder := q.NewTransactionBatchInsertBuilder(2) - prefilterInsertBuilder := q.NewTransactionFilteredTmpBatchInsertBuilder(2) + tt.Assert.NoError(q.Begin(ctx)) + + insertBuilder := q.NewTransactionBatchInsertBuilder() + prefilterInsertBuilder := q.NewTransactionFilteredTmpBatchInsertBuilder() // include both fee bump and normal transaction in the same batch // to make sure both kinds of transactions can be inserted using a single exec statement - tt.Assert.NoError(insertBuilder.Add(ctx, feeBumpTransaction, sequence)) - tt.Assert.NoError(insertBuilder.Add(ctx, normalTransaction, sequence)) - tt.Assert.NoError(insertBuilder.Exec(ctx)) + tt.Assert.NoError(insertBuilder.Add(feeBumpTransaction, sequence)) + tt.Assert.NoError(insertBuilder.Add(normalTransaction, sequence)) + tt.Assert.NoError(insertBuilder.Exec(ctx, q)) - tt.Assert.NoError(prefilterInsertBuilder.Add(ctx, feeBumpTransaction, sequence)) - tt.Assert.NoError(prefilterInsertBuilder.Add(ctx, normalTransaction, sequence)) - tt.Assert.NoError(prefilterInsertBuilder.Exec(ctx)) + tt.Assert.NoError(prefilterInsertBuilder.Add(feeBumpTransaction, sequence)) + tt.Assert.NoError(prefilterInsertBuilder.Add(normalTransaction, sequence)) + tt.Assert.NoError(prefilterInsertBuilder.Exec(ctx, q)) account := fixture.Envelope.SourceAccount().ToAccountId() feeBumpAccount := fixture.Envelope.FeeBumpAccount().ToAccountId() - opBuilder := q.NewOperationBatchInsertBuilder(1) + opBuilder := q.NewOperationBatchInsertBuilder() details, err := json.Marshal(map[string]string{ "bump_to": "98", }) + tt.Assert.NoError(err) tt.Assert.NoError(opBuilder.Add( - ctx, toid.New(fixture.Ledger.Sequence, 1, 1).ToInt64(), toid.New(fixture.Ledger.Sequence, 1, 0).ToInt64(), 1, @@ -279,26 +282,28 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture { null.String{}, false, )) - tt.Assert.NoError(opBuilder.Exec(ctx)) + tt.Assert.NoError(opBuilder.Exec(ctx, q)) - effectBuilder := q.NewEffectBatchInsertBuilder(2) + effectBuilder := q.NewEffectBatchInsertBuilder() details, err = json.Marshal(map[string]interface{}{"new_seq": 98}) tt.Assert.NoError(err) - accounIDs, err := q.CreateAccounts(ctx, []string{account.Address()}, 1) - tt.Assert.NoError(err) + accountLoader := NewAccountLoader() err = effectBuilder.Add( - ctx, - accounIDs[account.Address()], + accountLoader.GetFuture(account.Address()), null.String{}, toid.New(fixture.Ledger.Sequence, 1, 1).ToInt64(), 1, EffectSequenceBumped, details, ) + tt.Assert.NoError(err) - tt.Assert.NoError(effectBuilder.Exec(ctx)) + tt.Assert.NoError(accountLoader.Exec(ctx, q.SessionInterface)) + tt.Assert.NoError(effectBuilder.Exec(ctx, q.SessionInterface)) + + tt.Assert.NoError(q.Commit()) fixture.Transaction = Transaction{ TransactionWithoutLedger: TransactionWithoutLedger{ diff --git a/services/horizon/internal/db2/history/history_claimable_balances.go b/services/horizon/internal/db2/history/history_claimable_balances.go index b67294f4a6..5d2076f3fd 100644 --- a/services/horizon/internal/db2/history/history_claimable_balances.go +++ b/services/horizon/internal/db2/history/history_claimable_balances.go @@ -13,8 +13,8 @@ import ( // QHistoryClaimableBalances defines account related queries. type QHistoryClaimableBalances interface { CreateHistoryClaimableBalances(ctx context.Context, ids []string, batchSize int) (map[string]int64, error) - NewOperationClaimableBalanceBatchInsertBuilder(maxBatchSize int) OperationClaimableBalanceBatchInsertBuilder - NewTransactionClaimableBalanceBatchInsertBuilder(maxBatchSize int) TransactionClaimableBalanceBatchInsertBuilder + NewOperationClaimableBalanceBatchInsertBuilder() OperationClaimableBalanceBatchInsertBuilder + NewTransactionClaimableBalanceBatchInsertBuilder() TransactionClaimableBalanceBatchInsertBuilder } // CreateHistoryClaimableBalances creates rows in the history_claimable_balances table for a given list of ids. @@ -92,63 +92,61 @@ func (q *Q) ClaimableBalanceByID(ctx context.Context, id string) (dest HistoryCl } type OperationClaimableBalanceBatchInsertBuilder interface { - Add(ctx context.Context, operationID, internalID int64) error - Exec(ctx context.Context) error + Add(operationID int64, claimableBalance FutureClaimableBalanceID) error + Exec(ctx context.Context, session db.SessionInterface) error } type operationClaimableBalanceBatchInsertBuilder struct { - builder db.BatchInsertBuilder + table string + builder db.FastBatchInsertBuilder } -func (q *Q) NewOperationClaimableBalanceBatchInsertBuilder(maxBatchSize int) OperationClaimableBalanceBatchInsertBuilder { +func (q *Q) NewOperationClaimableBalanceBatchInsertBuilder() OperationClaimableBalanceBatchInsertBuilder { return &operationClaimableBalanceBatchInsertBuilder{ - builder: db.BatchInsertBuilder{ - Table: q.GetTable("history_operation_claimable_balances"), - MaxBatchSize: maxBatchSize, - }, + table: "history_operation_claimable_balances", + builder: db.FastBatchInsertBuilder{}, } } // Add adds a new operation claimable balance to the batch -func (i *operationClaimableBalanceBatchInsertBuilder) Add(ctx context.Context, operationID, internalID int64) error { - return i.builder.Row(ctx, map[string]interface{}{ +func (i *operationClaimableBalanceBatchInsertBuilder) Add(operationID int64, claimableBalance FutureClaimableBalanceID) error { + return i.builder.Row(map[string]interface{}{ "history_operation_id": operationID, - "history_claimable_balance_id": internalID, + "history_claimable_balance_id": claimableBalance, }) } // Exec flushes all pending operation claimable balances to the db -func (i *operationClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context) error { - return i.builder.Exec(ctx) +func (i *operationClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + return i.builder.Exec(ctx, session, i.table) } type TransactionClaimableBalanceBatchInsertBuilder interface { - Add(ctx context.Context, transactionID, internalID int64) error - Exec(ctx context.Context) error + Add(transactionID int64, claimableBalance FutureClaimableBalanceID) error + Exec(ctx context.Context, session db.SessionInterface) error } type transactionClaimableBalanceBatchInsertBuilder struct { - builder db.BatchInsertBuilder + table string + builder db.FastBatchInsertBuilder } -func (q *Q) NewTransactionClaimableBalanceBatchInsertBuilder(maxBatchSize int) TransactionClaimableBalanceBatchInsertBuilder { +func (q *Q) NewTransactionClaimableBalanceBatchInsertBuilder() TransactionClaimableBalanceBatchInsertBuilder { return &transactionClaimableBalanceBatchInsertBuilder{ - builder: db.BatchInsertBuilder{ - Table: q.GetTable("history_transaction_claimable_balances"), - MaxBatchSize: maxBatchSize, - }, + table: "history_transaction_claimable_balances", + builder: db.FastBatchInsertBuilder{}, } } // Add adds a new transaction claimable balance to the batch -func (i *transactionClaimableBalanceBatchInsertBuilder) Add(ctx context.Context, transactionID, internalID int64) error { - return i.builder.Row(ctx, map[string]interface{}{ +func (i *transactionClaimableBalanceBatchInsertBuilder) Add(transactionID int64, claimableBalance FutureClaimableBalanceID) error { + return i.builder.Row(map[string]interface{}{ "history_transaction_id": transactionID, - "history_claimable_balance_id": internalID, + "history_claimable_balance_id": claimableBalance, }) } // Exec flushes all pending transaction claimable balances to the db -func (i *transactionClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context) error { - return i.builder.Exec(ctx) +func (i *transactionClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + return i.builder.Exec(ctx, session, i.table) } diff --git a/services/horizon/internal/db2/history/history_liquidity_pools.go b/services/horizon/internal/db2/history/history_liquidity_pools.go index 0601d91be7..bb13ac59f9 100644 --- a/services/horizon/internal/db2/history/history_liquidity_pools.go +++ b/services/horizon/internal/db2/history/history_liquidity_pools.go @@ -5,6 +5,7 @@ import ( "sort" sq "github.com/Masterminds/squirrel" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" ) @@ -12,8 +13,8 @@ import ( // QHistoryLiquidityPools defines account related queries. type QHistoryLiquidityPools interface { CreateHistoryLiquidityPools(ctx context.Context, poolIDs []string, batchSize int) (map[string]int64, error) - NewOperationLiquidityPoolBatchInsertBuilder(maxBatchSize int) OperationLiquidityPoolBatchInsertBuilder - NewTransactionLiquidityPoolBatchInsertBuilder(maxBatchSize int) TransactionLiquidityPoolBatchInsertBuilder + NewOperationLiquidityPoolBatchInsertBuilder() OperationLiquidityPoolBatchInsertBuilder + NewTransactionLiquidityPoolBatchInsertBuilder() TransactionLiquidityPoolBatchInsertBuilder } // CreateHistoryLiquidityPools creates rows in the history_liquidity_pools table for a given list of ids. @@ -101,63 +102,61 @@ func (q *Q) LiquidityPoolByID(ctx context.Context, poolID string) (dest HistoryL } type OperationLiquidityPoolBatchInsertBuilder interface { - Add(ctx context.Context, operationID, internalID int64) error - Exec(ctx context.Context) error + Add(operationID int64, lp FutureLiquidityPoolID) error + Exec(ctx context.Context, session db.SessionInterface) error } type operationLiquidityPoolBatchInsertBuilder struct { - builder db.BatchInsertBuilder + table string + builder db.FastBatchInsertBuilder } -func (q *Q) NewOperationLiquidityPoolBatchInsertBuilder(maxBatchSize int) OperationLiquidityPoolBatchInsertBuilder { +func (q *Q) NewOperationLiquidityPoolBatchInsertBuilder() OperationLiquidityPoolBatchInsertBuilder { return &operationLiquidityPoolBatchInsertBuilder{ - builder: db.BatchInsertBuilder{ - Table: q.GetTable("history_operation_liquidity_pools"), - MaxBatchSize: maxBatchSize, - }, + table: "history_operation_liquidity_pools", + builder: db.FastBatchInsertBuilder{}, } } // Add adds a new operation claimable balance to the batch -func (i *operationLiquidityPoolBatchInsertBuilder) Add(ctx context.Context, operationID, internalID int64) error { - return i.builder.Row(ctx, map[string]interface{}{ +func (i *operationLiquidityPoolBatchInsertBuilder) Add(operationID int64, lp FutureLiquidityPoolID) error { + return i.builder.Row(map[string]interface{}{ "history_operation_id": operationID, - "history_liquidity_pool_id": internalID, + "history_liquidity_pool_id": lp, }) } // Exec flushes all pending operation claimable balances to the db -func (i *operationLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context) error { - return i.builder.Exec(ctx) +func (i *operationLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + return i.builder.Exec(ctx, session, i.table) } type TransactionLiquidityPoolBatchInsertBuilder interface { - Add(ctx context.Context, transactionID, internalID int64) error - Exec(ctx context.Context) error + Add(transactionID int64, lp FutureLiquidityPoolID) error + Exec(ctx context.Context, session db.SessionInterface) error } type transactionLiquidityPoolBatchInsertBuilder struct { - builder db.BatchInsertBuilder + table string + builder db.FastBatchInsertBuilder } -func (q *Q) NewTransactionLiquidityPoolBatchInsertBuilder(maxBatchSize int) TransactionLiquidityPoolBatchInsertBuilder { +func (q *Q) NewTransactionLiquidityPoolBatchInsertBuilder() TransactionLiquidityPoolBatchInsertBuilder { return &transactionLiquidityPoolBatchInsertBuilder{ - builder: db.BatchInsertBuilder{ - Table: q.GetTable("history_transaction_liquidity_pools"), - MaxBatchSize: maxBatchSize, - }, + table: "history_transaction_liquidity_pools", + builder: db.FastBatchInsertBuilder{}, } } // Add adds a new transaction claimable balance to the batch -func (i *transactionLiquidityPoolBatchInsertBuilder) Add(ctx context.Context, transactionID, internalID int64) error { - return i.builder.Row(ctx, map[string]interface{}{ +func (i *transactionLiquidityPoolBatchInsertBuilder) Add(transactionID int64, lp FutureLiquidityPoolID) error { + return i.builder.Row(map[string]interface{}{ "history_transaction_id": transactionID, - "history_liquidity_pool_id": internalID, + "history_liquidity_pool_id": lp, }) } // Exec flushes all pending transaction claimable balances to the db -func (i *transactionLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context) error { - return i.builder.Exec(ctx) +func (i *transactionLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + return i.builder.Exec(ctx, session, i.table) } diff --git a/services/horizon/internal/db2/history/ledger.go b/services/horizon/internal/db2/history/ledger.go index 7d367a8464..ca89534702 100644 --- a/services/horizon/internal/db2/history/ledger.go +++ b/services/horizon/internal/db2/history/ledger.go @@ -10,6 +10,7 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/guregu/null" "github.com/stellar/go/services/horizon/internal/db2" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/support/ordered" "github.com/stellar/go/toid" @@ -90,27 +91,46 @@ func (q *LedgersQ) Select(ctx context.Context, dest interface{}) error { // QLedgers defines ingestion ledger related queries. type QLedgers interface { - InsertLedger( - ctx context.Context, + NewLedgerBatchInsertBuilder() LedgerBatchInsertBuilder +} + +// LedgerBatchInsertBuilder is used to insert ledgers into the +// history_ledgers table +type LedgerBatchInsertBuilder interface { + Add( ledger xdr.LedgerHeaderHistoryEntry, successTxsCount int, failedTxsCount int, opCount int, txSetOpCount int, ingestVersion int, - ) (int64, error) + ) error + Exec(ctx context.Context, session db.SessionInterface) error +} + +// ledgerBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder +type ledgerBatchInsertBuilder struct { + builder db.FastBatchInsertBuilder + table string +} + +// NewLedgerBatchInsertBuilder constructs a new EffectBatchInsertBuilder instance +func (q *Q) NewLedgerBatchInsertBuilder() LedgerBatchInsertBuilder { + return &ledgerBatchInsertBuilder{ + table: "history_ledgers", + builder: db.FastBatchInsertBuilder{}, + } } -// InsertLedger creates a row in the history_ledgers table. -// Returns number of rows affected and error. -func (q *Q) InsertLedger(ctx context.Context, +// Add adds a effect to the batch +func (i *ledgerBatchInsertBuilder) Add( ledger xdr.LedgerHeaderHistoryEntry, successTxsCount int, failedTxsCount int, opCount int, txSetOpCount int, ingestVersion int, -) (int64, error) { +) error { m, err := ledgerHeaderToMap( ledger, successTxsCount, @@ -120,16 +140,14 @@ func (q *Q) InsertLedger(ctx context.Context, ingestVersion, ) if err != nil { - return 0, err + return err } - sql := sq.Insert("history_ledgers").SetMap(m) - result, err := q.Exec(ctx, sql) - if err != nil { - return 0, err - } + return i.builder.Row(m) +} - return result.RowsAffected() +func (i *ledgerBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + return i.builder.Exec(ctx, session, i.table) } // GetLedgerGaps obtains ingestion gaps in the history_ledgers table. diff --git a/services/horizon/internal/db2/history/ledger_test.go b/services/horizon/internal/db2/history/ledger_test.go index 4bf6d7b058..4fe9125fbe 100644 --- a/services/horizon/internal/db2/history/ledger_test.go +++ b/services/horizon/internal/db2/history/ledger_test.go @@ -119,7 +119,8 @@ func TestInsertLedger(t *testing.T) { tt.Assert.NoError(err) expectedLedger.LedgerHeaderXDR = null.NewString(ledgerHeaderBase64, true) - rowsAffected, err := q.InsertLedger(tt.Ctx, + ledgerBatch := q.NewLedgerBatchInsertBuilder() + err = ledgerBatch.Add( ledgerEntry, 12, 3, @@ -128,7 +129,9 @@ func TestInsertLedger(t *testing.T) { int(expectedLedger.ImporterVersion), ) tt.Assert.NoError(err) - tt.Assert.Equal(rowsAffected, int64(1)) + tt.Assert.NoError(q.Begin(tt.Ctx)) + tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(q.Commit()) err = q.LedgerBySequence(tt.Ctx, &ledgerFromDB, 69859) tt.Assert.NoError(err) @@ -204,7 +207,8 @@ func insertLedgerWithSequence(tt *test.T, q *Q, seq uint32) { ledgerHeaderBase64, err := xdr.MarshalBase64(ledgerEntry.Header) tt.Assert.NoError(err) expectedLedger.LedgerHeaderXDR = null.NewString(ledgerHeaderBase64, true) - rowsAffected, err := q.InsertLedger(tt.Ctx, + ledgerBatch := q.NewLedgerBatchInsertBuilder() + err = ledgerBatch.Add( ledgerEntry, 12, 3, @@ -213,7 +217,9 @@ func insertLedgerWithSequence(tt *test.T, q *Q, seq uint32) { int(expectedLedger.ImporterVersion), ) tt.Assert.NoError(err) - tt.Assert.Equal(rowsAffected, int64(1)) + tt.Assert.NoError(q.Begin(tt.Ctx)) + tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q.SessionInterface)) + tt.Assert.NoError(q.Commit()) } func TestGetLedgerGaps(t *testing.T) { diff --git a/services/horizon/internal/db2/history/liquidity_pool_loader.go b/services/horizon/internal/db2/history/liquidity_pool_loader.go new file mode 100644 index 0000000000..cf89ae67b4 --- /dev/null +++ b/services/horizon/internal/db2/history/liquidity_pool_loader.go @@ -0,0 +1,165 @@ +package history + +import ( + "context" + "database/sql/driver" + "fmt" + "sort" + + "github.com/stellar/go/support/collections/set" + "github.com/stellar/go/support/db" + "github.com/stellar/go/support/errors" + "github.com/stellar/go/support/ordered" +) + +// FutureLiquidityPoolID represents a future history liquidity pool. +// A FutureLiquidityPoolID is created by an LiquidityPoolLoader and +// the liquidity pool id is available after calling Exec() on +// the LiquidityPoolLoader. +type FutureLiquidityPoolID struct { + id string + loader *LiquidityPoolLoader +} + +// Value implements the database/sql/driver Valuer interface. +func (a FutureLiquidityPoolID) Value() (driver.Value, error) { + return a.loader.GetNow(a.id) +} + +// LiquidityPoolLoader will map liquidity pools to their internal +// history ids. If there is no existing mapping for a given liquidity pool, +// the LiquidityPoolLoader will insert into the history_liquidity_pools table to +// establish a mapping. +type LiquidityPoolLoader struct { + sealed bool + set set.Set[string] + ids map[string]int64 +} + +// NewLiquidityPoolLoader will construct a new LiquidityPoolLoader instance. +func NewLiquidityPoolLoader() *LiquidityPoolLoader { + return &LiquidityPoolLoader{ + sealed: false, + set: set.Set[string]{}, + ids: map[string]int64{}, + } +} + +// GetFuture registers the given liquidity pool into the loader and +// returns a FutureLiquidityPoolID which will hold the internal history id for +// the liquidity pool after Exec() is called. +func (a *LiquidityPoolLoader) GetFuture(id string) FutureLiquidityPoolID { + if a.sealed { + panic(errSealed) + } + + a.set.Add(id) + return FutureLiquidityPoolID{ + id: id, + loader: a, + } +} + +// GetNow returns the internal history id for the given liquidity pool. +// GetNow should only be called on values which were registered by +// GetFuture() calls. Also, Exec() must be called before any GetNow +// call can succeed. +func (a *LiquidityPoolLoader) GetNow(id string) (int64, error) { + if !a.sealed { + return 0, fmt.Errorf(`invalid liquidity pool loader state, + Exec was not called yet to properly seal and resolve %v id`, id) + } + if internalID, ok := a.ids[id]; !ok { + return 0, fmt.Errorf(`liquidity pool loader id %q was not found`, id) + } else { + return internalID, nil + } +} + +func (a *LiquidityPoolLoader) lookupKeys(ctx context.Context, q *Q, ids []string) error { + for i := 0; i < len(ids); i += loaderLookupBatchSize { + end := ordered.Min(len(ids), i+loaderLookupBatchSize) + + lps, err := q.LiquidityPoolsByIDs(ctx, ids[i:end]) + if err != nil { + return errors.Wrap(err, "could not select accounts") + } + + for _, lp := range lps { + a.ids[lp.PoolID] = lp.InternalID + } + } + return nil +} + +// Exec will look up all the internal history ids for the liquidity pools registered in the loader. +// If there are no internal history ids for a given set of liquidity pools, Exec will insert rows +// into the history_liquidity_pools table. +func (a *LiquidityPoolLoader) Exec(ctx context.Context, session db.SessionInterface) error { + a.sealed = true + if len(a.set) == 0 { + return nil + } + q := &Q{session} + ids := make([]string, 0, len(a.set)) + for id := range a.set { + ids = append(ids, id) + } + + if err := a.lookupKeys(ctx, q, ids); err != nil { + return err + } + + insert := 0 + for _, id := range ids { + if _, ok := a.ids[id]; ok { + continue + } + ids[insert] = id + insert++ + } + if insert == 0 { + return nil + } + ids = ids[:insert] + // sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock + // https://github.com/stellar/go/issues/2370 + sort.Strings(ids) + + err := bulkInsert( + ctx, + q, + "history_liquidity_pools", + []string{"liquidity_pool_id"}, + []bulkInsertField{ + { + name: "liquidity_pool_id", + dbType: "text", + objects: ids, + }, + }, + ) + if err != nil { + return err + } + + return a.lookupKeys(ctx, q, ids) +} + +// LiquidityPoolLoaderStub is a stub wrapper around LiquidityPoolLoader which allows +// you to manually configure the mapping of liquidity pools to history liquidity ppol ids +type LiquidityPoolLoaderStub struct { + Loader *LiquidityPoolLoader +} + +// NewLiquidityPoolLoaderStub returns a new LiquidityPoolLoader instance +func NewLiquidityPoolLoaderStub() LiquidityPoolLoaderStub { + return LiquidityPoolLoaderStub{Loader: NewLiquidityPoolLoader()} +} + +// Insert updates the wrapped LiquidityPoolLoader so that the given liquidity pool +// is mapped to the provided history liquidity pool id +func (a LiquidityPoolLoaderStub) Insert(lp string, id int64) { + a.Loader.sealed = true + a.Loader.ids[lp] = id +} diff --git a/services/horizon/internal/db2/history/liquidity_pool_loader_test.go b/services/horizon/internal/db2/history/liquidity_pool_loader_test.go new file mode 100644 index 0000000000..6e5b4addf7 --- /dev/null +++ b/services/horizon/internal/db2/history/liquidity_pool_loader_test.go @@ -0,0 +1,55 @@ +package history + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/stellar/go/services/horizon/internal/test" + "github.com/stellar/go/xdr" +) + +func TestLiquidityPoolLoader(t *testing.T) { + tt := test.Start(t) + defer tt.Finish() + test.ResetHorizonDB(t, tt.HorizonDB) + session := tt.HorizonSession() + + var ids []string + for i := 0; i < 100; i++ { + poolID := xdr.PoolId{byte(i)} + id, err := xdr.MarshalHex(poolID) + tt.Assert.NoError(err) + ids = append(ids, id) + } + + loader := NewLiquidityPoolLoader() + for _, id := range ids { + future := loader.GetFuture(id) + _, err := future.Value() + assert.Error(t, err) + assert.Contains(t, err.Error(), `invalid liquidity pool loader state,`) + duplicateFuture := loader.GetFuture(id) + assert.Equal(t, future, duplicateFuture) + } + + assert.NoError(t, loader.Exec(context.Background(), session)) + assert.Panics(t, func() { + loader.GetFuture("not-present") + }) + + q := &Q{session} + for _, id := range ids { + internalID, err := loader.GetNow(id) + assert.NoError(t, err) + lp, err := q.LiquidityPoolByID(context.Background(), id) + assert.NoError(t, err) + assert.Equal(t, lp.PoolID, id) + assert.Equal(t, lp.InternalID, internalID) + } + + _, err := loader.GetNow("not present") + assert.Error(t, err) + assert.Contains(t, err.Error(), `was not found`) +} diff --git a/services/horizon/internal/db2/history/main.go b/services/horizon/internal/db2/history/main.go index c91c7c3a7d..4210fe10fb 100644 --- a/services/horizon/internal/db2/history/main.go +++ b/services/horizon/internal/db2/history/main.go @@ -275,11 +275,11 @@ type IngestionQ interface { // QParticipants // Copy the small interfaces with shared methods directly, otherwise error: // duplicate method CreateAccounts - NewTransactionParticipantsBatchInsertBuilder(maxBatchSize int) TransactionParticipantsBatchInsertBuilder - NewOperationParticipantBatchInsertBuilder(maxBatchSize int) OperationParticipantBatchInsertBuilder + NewTransactionParticipantsBatchInsertBuilder() TransactionParticipantsBatchInsertBuilder + NewOperationParticipantBatchInsertBuilder() OperationParticipantBatchInsertBuilder QSigners //QTrades - NewTradeBatchInsertBuilder(maxBatchSize int) TradeBatchInsertBuilder + NewTradeBatchInsertBuilder() TradeBatchInsertBuilder RebuildTradeAggregationTimes(ctx context.Context, from, to strtime.Millis, roundingSlippageFilter int) error RebuildTradeAggregationBuckets(ctx context.Context, fromLedger, toLedger uint32, roundingSlippageFilter int) error ReapLookupTables(ctx context.Context, offsets map[string]int64) (map[string]int64, map[string]int64, error) diff --git a/services/horizon/internal/db2/history/mock_effect_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_effect_batch_insert_builder.go index 48ee96e306..35168a775a 100644 --- a/services/horizon/internal/db2/history/mock_effect_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/mock_effect_batch_insert_builder.go @@ -3,6 +3,8 @@ package history import ( "context" + "github.com/stellar/go/support/db" + "github.com/guregu/null" "github.com/stretchr/testify/mock" ) @@ -13,15 +15,15 @@ type MockEffectBatchInsertBuilder struct { } // Add mock -func (m *MockEffectBatchInsertBuilder) Add(ctx context.Context, - accountID int64, +func (m *MockEffectBatchInsertBuilder) Add( + accountID FutureAccountID, muxedAccount null.String, operationID int64, order uint32, effectType EffectType, details []byte, ) error { - a := m.Called(ctx, + a := m.Called( accountID, muxedAccount, operationID, @@ -33,7 +35,7 @@ func (m *MockEffectBatchInsertBuilder) Add(ctx context.Context, } // Exec mock -func (m *MockEffectBatchInsertBuilder) Exec(ctx context.Context) error { - a := m.Called(ctx) +func (m *MockEffectBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + a := m.Called(ctx, session) return a.Error(0) } diff --git a/services/horizon/internal/db2/history/mock_operation_participant_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_operation_participant_batch_insert_builder.go index 014763f989..481a731043 100644 --- a/services/horizon/internal/db2/history/mock_operation_participant_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/mock_operation_participant_batch_insert_builder.go @@ -2,6 +2,9 @@ package history import ( "context" + + "github.com/stellar/go/support/db" + "github.com/stretchr/testify/mock" ) @@ -11,13 +14,13 @@ type MockOperationParticipantBatchInsertBuilder struct { } // Add mock -func (m *MockOperationParticipantBatchInsertBuilder) Add(ctx context.Context, operationID int64, accountID int64) error { - a := m.Called(ctx, operationID, accountID) +func (m *MockOperationParticipantBatchInsertBuilder) Add(operationID int64, account FutureAccountID) error { + a := m.Called(operationID, account) return a.Error(0) } // Exec mock -func (m *MockOperationParticipantBatchInsertBuilder) Exec(ctx context.Context) error { - a := m.Called(ctx) +func (m *MockOperationParticipantBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + a := m.Called(ctx, session) return a.Error(0) } diff --git a/services/horizon/internal/db2/history/mock_operations_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_operations_batch_insert_builder.go index d8321a79b7..6a63efcdc1 100644 --- a/services/horizon/internal/db2/history/mock_operations_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/mock_operations_batch_insert_builder.go @@ -4,6 +4,7 @@ import ( "context" "github.com/guregu/null" + "github.com/stellar/go/support/db" "github.com/stellar/go/xdr" "github.com/stretchr/testify/mock" ) @@ -14,7 +15,7 @@ type MockOperationsBatchInsertBuilder struct { } // Add mock -func (m *MockOperationsBatchInsertBuilder) Add(ctx context.Context, +func (m *MockOperationsBatchInsertBuilder) Add( id int64, transactionID int64, applicationOrder uint32, @@ -24,7 +25,7 @@ func (m *MockOperationsBatchInsertBuilder) Add(ctx context.Context, sourceAccountMuxed null.String, isPayment bool, ) error { - a := m.Called(ctx, + a := m.Called( id, transactionID, applicationOrder, @@ -38,7 +39,7 @@ func (m *MockOperationsBatchInsertBuilder) Add(ctx context.Context, } // Exec mock -func (m *MockOperationsBatchInsertBuilder) Exec(ctx context.Context) error { - a := m.Called(ctx) +func (m *MockOperationsBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + a := m.Called(ctx, session) return a.Error(0) } diff --git a/services/horizon/internal/db2/history/mock_q_effects.go b/services/horizon/internal/db2/history/mock_q_effects.go index d8bdd97765..615e4699fa 100644 --- a/services/horizon/internal/db2/history/mock_q_effects.go +++ b/services/horizon/internal/db2/history/mock_q_effects.go @@ -10,8 +10,8 @@ type MockQEffects struct { mock.Mock } -func (m *MockQEffects) NewEffectBatchInsertBuilder(maxBatchSize int) EffectBatchInsertBuilder { - a := m.Called(maxBatchSize) +func (m *MockQEffects) NewEffectBatchInsertBuilder() EffectBatchInsertBuilder { + a := m.Called() return a.Get(0).(EffectBatchInsertBuilder) } diff --git a/services/horizon/internal/db2/history/mock_q_history_claimable_balances.go b/services/horizon/internal/db2/history/mock_q_history_claimable_balances.go index 9b7fe1d3b6..6607456af2 100644 --- a/services/horizon/internal/db2/history/mock_q_history_claimable_balances.go +++ b/services/horizon/internal/db2/history/mock_q_history_claimable_balances.go @@ -3,6 +3,8 @@ package history import ( "context" + "github.com/stellar/go/support/db" + "github.com/stretchr/testify/mock" ) @@ -16,8 +18,8 @@ func (m *MockQHistoryClaimableBalances) CreateHistoryClaimableBalances(ctx conte return a.Get(0).(map[string]int64), a.Error(1) } -func (m *MockQHistoryClaimableBalances) NewTransactionClaimableBalanceBatchInsertBuilder(maxBatchSize int) TransactionClaimableBalanceBatchInsertBuilder { - a := m.Called(maxBatchSize) +func (m *MockQHistoryClaimableBalances) NewTransactionClaimableBalanceBatchInsertBuilder() TransactionClaimableBalanceBatchInsertBuilder { + a := m.Called() return a.Get(0).(TransactionClaimableBalanceBatchInsertBuilder) } @@ -27,19 +29,19 @@ type MockTransactionClaimableBalanceBatchInsertBuilder struct { mock.Mock } -func (m *MockTransactionClaimableBalanceBatchInsertBuilder) Add(ctx context.Context, transactionID, accountID int64) error { - a := m.Called(ctx, transactionID, accountID) +func (m *MockTransactionClaimableBalanceBatchInsertBuilder) Add(transactionID int64, claimableBalance FutureClaimableBalanceID) error { + a := m.Called(transactionID, claimableBalance) return a.Error(0) } -func (m *MockTransactionClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context) error { - a := m.Called(ctx) +func (m *MockTransactionClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + a := m.Called(ctx, session) return a.Error(0) } // NewOperationClaimableBalanceBatchInsertBuilder mock -func (m *MockQHistoryClaimableBalances) NewOperationClaimableBalanceBatchInsertBuilder(maxBatchSize int) OperationClaimableBalanceBatchInsertBuilder { - a := m.Called(maxBatchSize) +func (m *MockQHistoryClaimableBalances) NewOperationClaimableBalanceBatchInsertBuilder() OperationClaimableBalanceBatchInsertBuilder { + a := m.Called() return a.Get(0).(OperationClaimableBalanceBatchInsertBuilder) } @@ -49,12 +51,12 @@ type MockOperationClaimableBalanceBatchInsertBuilder struct { mock.Mock } -func (m *MockOperationClaimableBalanceBatchInsertBuilder) Add(ctx context.Context, transactionID, accountID int64) error { - a := m.Called(ctx, transactionID, accountID) +func (m *MockOperationClaimableBalanceBatchInsertBuilder) Add(operationID int64, claimableBalance FutureClaimableBalanceID) error { + a := m.Called(operationID, claimableBalance) return a.Error(0) } -func (m *MockOperationClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context) error { - a := m.Called(ctx) +func (m *MockOperationClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + a := m.Called(ctx, session) return a.Error(0) } diff --git a/services/horizon/internal/db2/history/mock_q_history_liquidity_pools.go b/services/horizon/internal/db2/history/mock_q_history_liquidity_pools.go index 08f4920de2..bf000a22e9 100644 --- a/services/horizon/internal/db2/history/mock_q_history_liquidity_pools.go +++ b/services/horizon/internal/db2/history/mock_q_history_liquidity_pools.go @@ -3,6 +3,8 @@ package history import ( "context" + "github.com/stellar/go/support/db" + "github.com/stretchr/testify/mock" ) @@ -16,8 +18,8 @@ func (m *MockQHistoryLiquidityPools) CreateHistoryLiquidityPools(ctx context.Con return a.Get(0).(map[string]int64), a.Error(1) } -func (m *MockQHistoryLiquidityPools) NewTransactionLiquidityPoolBatchInsertBuilder(maxBatchSize int) TransactionLiquidityPoolBatchInsertBuilder { - a := m.Called(maxBatchSize) +func (m *MockQHistoryLiquidityPools) NewTransactionLiquidityPoolBatchInsertBuilder() TransactionLiquidityPoolBatchInsertBuilder { + a := m.Called() return a.Get(0).(TransactionLiquidityPoolBatchInsertBuilder) } @@ -27,19 +29,19 @@ type MockTransactionLiquidityPoolBatchInsertBuilder struct { mock.Mock } -func (m *MockTransactionLiquidityPoolBatchInsertBuilder) Add(ctx context.Context, transactionID, accountID int64) error { - a := m.Called(ctx, transactionID, accountID) +func (m *MockTransactionLiquidityPoolBatchInsertBuilder) Add(transactionID int64, lp FutureLiquidityPoolID) error { + a := m.Called(transactionID, lp) return a.Error(0) } -func (m *MockTransactionLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context) error { - a := m.Called(ctx) +func (m *MockTransactionLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + a := m.Called(ctx, session) return a.Error(0) } // NewOperationLiquidityPoolBatchInsertBuilder mock -func (m *MockQHistoryLiquidityPools) NewOperationLiquidityPoolBatchInsertBuilder(maxBatchSize int) OperationLiquidityPoolBatchInsertBuilder { - a := m.Called(maxBatchSize) +func (m *MockQHistoryLiquidityPools) NewOperationLiquidityPoolBatchInsertBuilder() OperationLiquidityPoolBatchInsertBuilder { + a := m.Called() return a.Get(0).(OperationLiquidityPoolBatchInsertBuilder) } @@ -49,12 +51,12 @@ type MockOperationLiquidityPoolBatchInsertBuilder struct { mock.Mock } -func (m *MockOperationLiquidityPoolBatchInsertBuilder) Add(ctx context.Context, transactionID, accountID int64) error { - a := m.Called(ctx, transactionID, accountID) +func (m *MockOperationLiquidityPoolBatchInsertBuilder) Add(operationID int64, lp FutureLiquidityPoolID) error { + a := m.Called(operationID, lp) return a.Error(0) } -func (m *MockOperationLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context) error { - a := m.Called(ctx) +func (m *MockOperationLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + a := m.Called(ctx, session) return a.Error(0) } diff --git a/services/horizon/internal/db2/history/mock_q_ledgers.go b/services/horizon/internal/db2/history/mock_q_ledgers.go index 16d3ef5524..f02cd7517c 100644 --- a/services/horizon/internal/db2/history/mock_q_ledgers.go +++ b/services/horizon/internal/db2/history/mock_q_ledgers.go @@ -3,23 +3,38 @@ package history import ( "context" - "github.com/stretchr/testify/mock" - + "github.com/stellar/go/support/db" "github.com/stellar/go/xdr" + + "github.com/stretchr/testify/mock" ) type MockQLedgers struct { mock.Mock } -func (m *MockQLedgers) InsertLedger(ctx context.Context, +func (m *MockQLedgers) NewLedgerBatchInsertBuilder() LedgerBatchInsertBuilder { + a := m.Called() + return a.Get(0).(LedgerBatchInsertBuilder) +} + +type MockLedgersBatchInsertBuilder struct { + mock.Mock +} + +func (m *MockLedgersBatchInsertBuilder) Add( ledger xdr.LedgerHeaderHistoryEntry, successTxsCount int, failedTxsCount int, opCount int, txSetOpCount int, ingestVersion int, -) (int64, error) { - a := m.Called(ctx, ledger, successTxsCount, failedTxsCount, opCount, txSetOpCount, ingestVersion) - return a.Get(0).(int64), a.Error(1) +) error { + a := m.Called(ledger, successTxsCount, failedTxsCount, opCount, txSetOpCount, ingestVersion) + return a.Error(0) +} + +func (m *MockLedgersBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + a := m.Called(ctx, session) + return a.Error(0) } diff --git a/services/horizon/internal/db2/history/mock_q_operations.go b/services/horizon/internal/db2/history/mock_q_operations.go index 08a97c6da9..6c2741ad3e 100644 --- a/services/horizon/internal/db2/history/mock_q_operations.go +++ b/services/horizon/internal/db2/history/mock_q_operations.go @@ -8,7 +8,7 @@ type MockQOperations struct { } // NewOperationBatchInsertBuilder mock -func (m *MockQOperations) NewOperationBatchInsertBuilder(maxBatchSize int) OperationBatchInsertBuilder { - a := m.Called(maxBatchSize) +func (m *MockQOperations) NewOperationBatchInsertBuilder() OperationBatchInsertBuilder { + a := m.Called() return a.Get(0).(OperationBatchInsertBuilder) } diff --git a/services/horizon/internal/db2/history/mock_q_participants.go b/services/horizon/internal/db2/history/mock_q_participants.go index 9365e06db3..b871199190 100644 --- a/services/horizon/internal/db2/history/mock_q_participants.go +++ b/services/horizon/internal/db2/history/mock_q_participants.go @@ -2,7 +2,10 @@ package history import ( "context" + "github.com/stretchr/testify/mock" + + "github.com/stellar/go/support/db" ) // MockQParticipants is a mock implementation of the QParticipants interface @@ -15,8 +18,8 @@ func (m *MockQParticipants) CreateAccounts(ctx context.Context, addresses []stri return a.Get(0).(map[string]int64), a.Error(1) } -func (m *MockQParticipants) NewTransactionParticipantsBatchInsertBuilder(maxBatchSize int) TransactionParticipantsBatchInsertBuilder { - a := m.Called(maxBatchSize) +func (m *MockQParticipants) NewTransactionParticipantsBatchInsertBuilder() TransactionParticipantsBatchInsertBuilder { + a := m.Called() return a.Get(0).(TransactionParticipantsBatchInsertBuilder) } @@ -26,18 +29,19 @@ type MockTransactionParticipantsBatchInsertBuilder struct { mock.Mock } -func (m *MockTransactionParticipantsBatchInsertBuilder) Add(ctx context.Context, transactionID, accountID int64) error { - a := m.Called(ctx, transactionID, accountID) +func (m *MockTransactionParticipantsBatchInsertBuilder) Add(transactionID int64, accountID FutureAccountID) error { + a := m.Called(transactionID, accountID) return a.Error(0) } -func (m *MockTransactionParticipantsBatchInsertBuilder) Exec(ctx context.Context) error { - a := m.Called(ctx) +func (m *MockTransactionParticipantsBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + a := m.Called(ctx, session) return a.Error(0) } // NewOperationParticipantBatchInsertBuilder mock -func (m *MockQParticipants) NewOperationParticipantBatchInsertBuilder(maxBatchSize int) OperationParticipantBatchInsertBuilder { - a := m.Called(maxBatchSize) - return a.Get(0).(OperationParticipantBatchInsertBuilder) +func (m *MockQParticipants) NewOperationParticipantBatchInsertBuilder() OperationParticipantBatchInsertBuilder { + a := m.Called() + v := a.Get(0) + return v.(OperationParticipantBatchInsertBuilder) } diff --git a/services/horizon/internal/db2/history/mock_q_trades.go b/services/horizon/internal/db2/history/mock_q_trades.go index d05e0e6a3d..2080f14a8d 100644 --- a/services/horizon/internal/db2/history/mock_q_trades.go +++ b/services/horizon/internal/db2/history/mock_q_trades.go @@ -3,6 +3,7 @@ package history import ( "context" + "github.com/stellar/go/support/db" "github.com/stellar/go/xdr" "github.com/stretchr/testify/mock" @@ -27,8 +28,8 @@ func (m *MockQTrades) CreateHistoryLiquidityPools(ctx context.Context, poolIDs [ return a.Get(0).(map[string]int64), a.Error(1) } -func (m *MockQTrades) NewTradeBatchInsertBuilder(maxBatchSize int) TradeBatchInsertBuilder { - a := m.Called(maxBatchSize) +func (m *MockQTrades) NewTradeBatchInsertBuilder() TradeBatchInsertBuilder { + a := m.Called() return a.Get(0).(TradeBatchInsertBuilder) } @@ -41,12 +42,12 @@ type MockTradeBatchInsertBuilder struct { mock.Mock } -func (m *MockTradeBatchInsertBuilder) Add(ctx context.Context, entries ...InsertTrade) error { - a := m.Called(ctx, entries) +func (m *MockTradeBatchInsertBuilder) Add(entries ...InsertTrade) error { + a := m.Called(entries) return a.Error(0) } -func (m *MockTradeBatchInsertBuilder) Exec(ctx context.Context) error { - a := m.Called(ctx) +func (m *MockTradeBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + a := m.Called(ctx, session) return a.Error(0) } diff --git a/services/horizon/internal/db2/history/mock_q_transactions.go b/services/horizon/internal/db2/history/mock_q_transactions.go index 3bf308128f..064d0e34c4 100644 --- a/services/horizon/internal/db2/history/mock_q_transactions.go +++ b/services/horizon/internal/db2/history/mock_q_transactions.go @@ -7,12 +7,12 @@ type MockQTransactions struct { mock.Mock } -func (m *MockQTransactions) NewTransactionBatchInsertBuilder(maxBatchSize int) TransactionBatchInsertBuilder { - a := m.Called(maxBatchSize) +func (m *MockQTransactions) NewTransactionBatchInsertBuilder() TransactionBatchInsertBuilder { + a := m.Called() return a.Get(0).(TransactionBatchInsertBuilder) } -func (m *MockQTransactions) NewTransactionFilteredTmpBatchInsertBuilder(maxBatchSize int) TransactionBatchInsertBuilder { - a := m.Called(maxBatchSize) +func (m *MockQTransactions) NewTransactionFilteredTmpBatchInsertBuilder() TransactionBatchInsertBuilder { + a := m.Called() return a.Get(0).(TransactionBatchInsertBuilder) } diff --git a/services/horizon/internal/db2/history/mock_transactions_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_transactions_batch_insert_builder.go index 8e2608d553..db16097a03 100644 --- a/services/horizon/internal/db2/history/mock_transactions_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/mock_transactions_batch_insert_builder.go @@ -6,18 +6,19 @@ import ( "github.com/stretchr/testify/mock" "github.com/stellar/go/ingest" + "github.com/stellar/go/support/db" ) type MockTransactionsBatchInsertBuilder struct { mock.Mock } -func (m *MockTransactionsBatchInsertBuilder) Add(ctx context.Context, transaction ingest.LedgerTransaction, sequence uint32) error { - a := m.Called(ctx, transaction, sequence) +func (m *MockTransactionsBatchInsertBuilder) Add(transaction ingest.LedgerTransaction, sequence uint32) error { + a := m.Called(transaction, sequence) return a.Error(0) } -func (m *MockTransactionsBatchInsertBuilder) Exec(ctx context.Context) error { - a := m.Called(ctx) +func (m *MockTransactionsBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + a := m.Called(ctx, session) return a.Error(0) } diff --git a/services/horizon/internal/db2/history/operation.go b/services/horizon/internal/db2/history/operation.go index 94064e8f28..803c19791f 100644 --- a/services/horizon/internal/db2/history/operation.go +++ b/services/horizon/internal/db2/history/operation.go @@ -383,7 +383,7 @@ func validateTransactionForOperation(transaction Transaction, operation Operatio // QOperations defines history_operation related queries. type QOperations interface { - NewOperationBatchInsertBuilder(maxBatchSize int) OperationBatchInsertBuilder + NewOperationBatchInsertBuilder() OperationBatchInsertBuilder } var selectOperation = sq.Select( diff --git a/services/horizon/internal/db2/history/operation_batch_insert_builder.go b/services/horizon/internal/db2/history/operation_batch_insert_builder.go index b2a9db6c88..e786ec97f7 100644 --- a/services/horizon/internal/db2/history/operation_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/operation_batch_insert_builder.go @@ -12,7 +12,6 @@ import ( // history_operations table type OperationBatchInsertBuilder interface { Add( - ctx context.Context, id int64, transactionID int64, applicationOrder uint32, @@ -22,27 +21,25 @@ type OperationBatchInsertBuilder interface { sourceAcccountMuxed null.String, isPayment bool, ) error - Exec(ctx context.Context) error + Exec(ctx context.Context, session db.SessionInterface) error } // operationBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder type operationBatchInsertBuilder struct { - builder db.BatchInsertBuilder + builder db.FastBatchInsertBuilder + table string } // NewOperationBatchInsertBuilder constructs a new TransactionBatchInsertBuilder instance -func (q *Q) NewOperationBatchInsertBuilder(maxBatchSize int) OperationBatchInsertBuilder { +func (q *Q) NewOperationBatchInsertBuilder() OperationBatchInsertBuilder { return &operationBatchInsertBuilder{ - builder: db.BatchInsertBuilder{ - Table: q.GetTable("history_operations"), - MaxBatchSize: maxBatchSize, - }, + table: "history_operations", + builder: db.FastBatchInsertBuilder{}, } } // Add adds a transaction's operations to the batch func (i *operationBatchInsertBuilder) Add( - ctx context.Context, id int64, transactionID int64, applicationOrder uint32, @@ -52,12 +49,12 @@ func (i *operationBatchInsertBuilder) Add( sourceAccountMuxed null.String, isPayment bool, ) error { - return i.builder.Row(ctx, map[string]interface{}{ + return i.builder.Row(map[string]interface{}{ "id": id, "transaction_id": transactionID, "application_order": applicationOrder, "type": operationType, - "details": details, + "details": string(details), "source_account": sourceAccount, "source_account_muxed": sourceAccountMuxed, "is_payment": isPayment, @@ -65,6 +62,6 @@ func (i *operationBatchInsertBuilder) Add( } -func (i *operationBatchInsertBuilder) Exec(ctx context.Context) error { - return i.builder.Exec(ctx) +func (i *operationBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + return i.builder.Exec(ctx, session, i.table) } diff --git a/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go b/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go index 5b4b596e57..0c06d545f3 100644 --- a/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go +++ b/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go @@ -16,9 +16,11 @@ func TestAddOperation(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - txBatch := q.NewTransactionBatchInsertBuilder(0) + tt.Assert.NoError(q.Begin(tt.Ctx)) - builder := q.NewOperationBatchInsertBuilder(1) + txBatch := q.NewTransactionBatchInsertBuilder() + + builder := q.NewOperationBatchInsertBuilder() transactionHash := "2a805712c6d10f9e74bb0ccf54ae92a2b4b1e586451fe8133a2433816f6b567c" transactionResult := "AAAAAAAAAGQAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAA=" @@ -35,8 +37,8 @@ func TestAddOperation(t *testing.T) { ) sequence := int32(56) - tt.Assert.NoError(txBatch.Add(tt.Ctx, transaction, uint32(sequence))) - tt.Assert.NoError(txBatch.Exec(tt.Ctx)) + tt.Assert.NoError(txBatch.Add(transaction, uint32(sequence))) + tt.Assert.NoError(txBatch.Exec(tt.Ctx, q)) details, err := json.Marshal(map[string]string{ "to": "GANFZDRBCNTUXIODCJEYMACPMCSZEVE4WZGZ3CZDZ3P2SXK4KH75IK6Y", @@ -48,7 +50,7 @@ func TestAddOperation(t *testing.T) { sourceAccount := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY" sourceAccountMuxed := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26" - err = builder.Add(tt.Ctx, + err = builder.Add( toid.New(sequence, 1, 1).ToInt64(), toid.New(sequence, 1, 0).ToInt64(), 1, @@ -60,8 +62,9 @@ func TestAddOperation(t *testing.T) { ) tt.Assert.NoError(err) - err = builder.Exec(tt.Ctx) + err = builder.Exec(tt.Ctx, q) tt.Assert.NoError(err) + tt.Assert.NoError(q.Commit()) ops := []Operation{} err = q.Select(tt.Ctx, &ops, selectOperation) diff --git a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder.go b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder.go index 6b2b3afb56..8882141426 100644 --- a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder.go @@ -10,40 +10,37 @@ import ( // history_operations table type OperationParticipantBatchInsertBuilder interface { Add( - ctx context.Context, operationID int64, - accountID int64, + accountID FutureAccountID, ) error - Exec(ctx context.Context) error + Exec(ctx context.Context, session db.SessionInterface) error } // operationParticipantBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder type operationParticipantBatchInsertBuilder struct { - builder db.BatchInsertBuilder + table string + builder db.FastBatchInsertBuilder } -// NewOperationParticipantBatchInsertBuilder constructs a new TransactionBatchInsertBuilder instance -func (q *Q) NewOperationParticipantBatchInsertBuilder(maxBatchSize int) OperationParticipantBatchInsertBuilder { +// NewOperationParticipantBatchInsertBuilder constructs a new OperationParticipantBatchInsertBuilder instance +func (q *Q) NewOperationParticipantBatchInsertBuilder() OperationParticipantBatchInsertBuilder { return &operationParticipantBatchInsertBuilder{ - builder: db.BatchInsertBuilder{ - Table: q.GetTable("history_operation_participants"), - MaxBatchSize: maxBatchSize, - }, + table: "history_operation_participants", + builder: db.FastBatchInsertBuilder{}, } } // Add adds an operation participant to the batch func (i *operationParticipantBatchInsertBuilder) Add( - ctx context.Context, operationID int64, - accountID int64, + accountID FutureAccountID, ) error { - return i.builder.Row(ctx, map[string]interface{}{ + return i.builder.Row(map[string]interface{}{ "history_operation_id": operationID, "history_account_id": accountID, }) } -func (i *operationParticipantBatchInsertBuilder) Exec(ctx context.Context) error { - return i.builder.Exec(ctx) +func (i *operationParticipantBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + return i.builder.Exec(ctx, session, i.table) } diff --git a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go index 51a0c4800d..7e823064f2 100644 --- a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go +++ b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go @@ -4,6 +4,8 @@ import ( "testing" sq "github.com/Masterminds/squirrel" + + "github.com/stellar/go/keypair" "github.com/stellar/go/services/horizon/internal/test" ) @@ -13,12 +15,16 @@ func TestAddOperationParticipants(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - builder := q.NewOperationParticipantBatchInsertBuilder(1) - err := builder.Add(tt.Ctx, 240518172673, 1) + accountLoader := NewAccountLoader() + address := keypair.MustRandom().Address() + tt.Assert.NoError(q.Begin(tt.Ctx)) + builder := q.NewOperationParticipantBatchInsertBuilder() + err := builder.Add(240518172673, accountLoader.GetFuture(address)) tt.Assert.NoError(err) - err = builder.Exec(tt.Ctx) - tt.Assert.NoError(err) + tt.Assert.NoError(accountLoader.Exec(tt.Ctx, q)) + tt.Assert.NoError(builder.Exec(tt.Ctx, q)) + tt.Assert.NoError(q.Commit()) type hop struct { OperationID int64 `db:"history_operation_id"` @@ -37,6 +43,8 @@ func TestAddOperationParticipants(t *testing.T) { op := ops[0] tt.Assert.Equal(int64(240518172673), op.OperationID) - tt.Assert.Equal(int64(1), op.AccountID) + val, err := accountLoader.GetNow(address) + tt.Assert.NoError(err) + tt.Assert.Equal(val, op.AccountID) } } diff --git a/services/horizon/internal/db2/history/operation_test.go b/services/horizon/internal/db2/history/operation_test.go index 480ec9c837..f7533ee5f3 100644 --- a/services/horizon/internal/db2/history/operation_test.go +++ b/services/horizon/internal/db2/history/operation_test.go @@ -5,6 +5,7 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/guregu/null" + "github.com/stellar/go/services/horizon/internal/db2" "github.com/stellar/go/services/horizon/internal/test" "github.com/stellar/go/toid" @@ -77,8 +78,10 @@ func TestOperationByLiquidityPool(t *testing.T) { opID1 := toid.New(sequence, txIndex, 1).ToInt64() opID2 := toid.New(sequence, txIndex, 2).ToInt64() + tt.Assert.NoError(q.Begin(tt.Ctx)) + // Insert a phony transaction - transactionBuilder := q.NewTransactionBatchInsertBuilder(2) + transactionBuilder := q.NewTransactionBatchInsertBuilder() firstTransaction := buildLedgerTransaction(tt.T, testTransaction{ index: uint32(txIndex), envelopeXDR: "AAAAACiSTRmpH6bHC6Ekna5e82oiGY5vKDEEUgkq9CB//t+rAAAAyAEXUhsAADDRAAAAAAAAAAAAAAABAAAAAAAAAAsBF1IbAABX4QAAAAAAAAAA", @@ -87,15 +90,14 @@ func TestOperationByLiquidityPool(t *testing.T) { metaXDR: "AAAAAQAAAAAAAAAA", hash: "19aaa18db88605aedec04659fb45e06f240b022eb2d429e05133e4d53cd945ba", }) - err := transactionBuilder.Add(tt.Ctx, firstTransaction, uint32(sequence)) + err := transactionBuilder.Add(firstTransaction, uint32(sequence)) tt.Assert.NoError(err) - err = transactionBuilder.Exec(tt.Ctx) + err = transactionBuilder.Exec(tt.Ctx, q) tt.Assert.NoError(err) // Insert a two phony operations - operationBuilder := q.NewOperationBatchInsertBuilder(2) + operationBuilder := q.NewOperationBatchInsertBuilder() err = operationBuilder.Add( - tt.Ctx, opID1, txID, 1, @@ -106,11 +108,8 @@ func TestOperationByLiquidityPool(t *testing.T) { false, ) tt.Assert.NoError(err) - err = operationBuilder.Exec(tt.Ctx) - tt.Assert.NoError(err) err = operationBuilder.Add( - tt.Ctx, opID2, txID, 1, @@ -121,23 +120,20 @@ func TestOperationByLiquidityPool(t *testing.T) { false, ) tt.Assert.NoError(err) - err = operationBuilder.Exec(tt.Ctx) + err = operationBuilder.Exec(tt.Ctx, q) tt.Assert.NoError(err) // Insert Liquidity Pool history liquidityPoolID := "a2f38836a839de008cf1d782c81f45e1253cc5d3dad9110b872965484fec0a49" - toInternalID, err := q.CreateHistoryLiquidityPools(tt.Ctx, []string{liquidityPoolID}, 2) - tt.Assert.NoError(err) - lpOperationBuilder := q.NewOperationLiquidityPoolBatchInsertBuilder(3) - tt.Assert.NoError(err) - internalID, ok := toInternalID[liquidityPoolID] - tt.Assert.True(ok) - err = lpOperationBuilder.Add(tt.Ctx, opID1, internalID) - tt.Assert.NoError(err) - err = lpOperationBuilder.Add(tt.Ctx, opID2, internalID) - tt.Assert.NoError(err) - err = lpOperationBuilder.Exec(tt.Ctx) - tt.Assert.NoError(err) + lpLoader := NewLiquidityPoolLoader() + + lpOperationBuilder := q.NewOperationLiquidityPoolBatchInsertBuilder() + tt.Assert.NoError(lpOperationBuilder.Add(opID1, lpLoader.GetFuture(liquidityPoolID))) + tt.Assert.NoError(lpOperationBuilder.Add(opID2, lpLoader.GetFuture(liquidityPoolID))) + tt.Assert.NoError(lpLoader.Exec(tt.Ctx, q)) + tt.Assert.NoError(lpOperationBuilder.Exec(tt.Ctx, q)) + + tt.Assert.NoError(q.Commit()) // Check ascending order pq := db2.PageQuery{ diff --git a/services/horizon/internal/db2/history/participants.go b/services/horizon/internal/db2/history/participants.go index 658e877f78..f73b5ab577 100644 --- a/services/horizon/internal/db2/history/participants.go +++ b/services/horizon/internal/db2/history/participants.go @@ -9,40 +9,39 @@ import ( // QParticipants defines ingestion participant related queries. type QParticipants interface { QCreateAccountsHistory - NewTransactionParticipantsBatchInsertBuilder(maxBatchSize int) TransactionParticipantsBatchInsertBuilder - NewOperationParticipantBatchInsertBuilder(maxBatchSize int) OperationParticipantBatchInsertBuilder + NewTransactionParticipantsBatchInsertBuilder() TransactionParticipantsBatchInsertBuilder + NewOperationParticipantBatchInsertBuilder() OperationParticipantBatchInsertBuilder } // TransactionParticipantsBatchInsertBuilder is used to insert transaction participants into the // history_transaction_participants table type TransactionParticipantsBatchInsertBuilder interface { - Add(ctx context.Context, transactionID, accountID int64) error - Exec(ctx context.Context) error + Add(transactionID int64, accountID FutureAccountID) error + Exec(ctx context.Context, session db.SessionInterface) error } type transactionParticipantsBatchInsertBuilder struct { - builder db.BatchInsertBuilder + tableName string + builder db.FastBatchInsertBuilder } // NewTransactionParticipantsBatchInsertBuilder constructs a new TransactionParticipantsBatchInsertBuilder instance -func (q *Q) NewTransactionParticipantsBatchInsertBuilder(maxBatchSize int) TransactionParticipantsBatchInsertBuilder { +func (q *Q) NewTransactionParticipantsBatchInsertBuilder() TransactionParticipantsBatchInsertBuilder { return &transactionParticipantsBatchInsertBuilder{ - builder: db.BatchInsertBuilder{ - Table: q.GetTable("history_transaction_participants"), - MaxBatchSize: maxBatchSize, - }, + tableName: "history_transaction_participants", + builder: db.FastBatchInsertBuilder{}, } } // Add adds a new transaction participant to the batch -func (i *transactionParticipantsBatchInsertBuilder) Add(ctx context.Context, transactionID, accountID int64) error { - return i.builder.Row(ctx, map[string]interface{}{ +func (i *transactionParticipantsBatchInsertBuilder) Add(transactionID int64, accountID FutureAccountID) error { + return i.builder.Row(map[string]interface{}{ "history_transaction_id": transactionID, "history_account_id": accountID, }) } // Exec flushes all pending transaction participants to the db -func (i *transactionParticipantsBatchInsertBuilder) Exec(ctx context.Context) error { - return i.builder.Exec(ctx) +func (i *transactionParticipantsBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + return i.builder.Exec(ctx, session, i.tableName) } diff --git a/services/horizon/internal/db2/history/participants_test.go b/services/horizon/internal/db2/history/participants_test.go index 4aaf70b151..37f7654abb 100644 --- a/services/horizon/internal/db2/history/participants_test.go +++ b/services/horizon/internal/db2/history/participants_test.go @@ -4,6 +4,8 @@ import ( "testing" sq "github.com/Masterminds/squirrel" + + "github.com/stellar/go/keypair" "github.com/stellar/go/services/horizon/internal/test" ) @@ -32,27 +34,38 @@ func TestTransactionParticipantsBatch(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - batch := q.NewTransactionParticipantsBatchInsertBuilder(0) + batch := q.NewTransactionParticipantsBatchInsertBuilder() + accountLoader := NewAccountLoader() transactionID := int64(1) otherTransactionID := int64(2) - accountID := int64(100) - + var addresses []string for i := int64(0); i < 3; i++ { - tt.Assert.NoError(batch.Add(tt.Ctx, transactionID, accountID+i)) + address := keypair.MustRandom().Address() + addresses = append(addresses, address) + tt.Assert.NoError(batch.Add(transactionID, accountLoader.GetFuture(address))) } - tt.Assert.NoError(batch.Add(tt.Ctx, otherTransactionID, accountID)) - tt.Assert.NoError(batch.Exec(tt.Ctx)) + address := keypair.MustRandom().Address() + addresses = append(addresses, address) + tt.Assert.NoError(batch.Add(otherTransactionID, accountLoader.GetFuture(address))) + + tt.Assert.NoError(q.Begin(tt.Ctx)) + tt.Assert.NoError(accountLoader.Exec(tt.Ctx, q)) + tt.Assert.NoError(batch.Exec(tt.Ctx, q)) + tt.Assert.NoError(q.Commit()) participants := getTransactionParticipants(tt, q) - tt.Assert.Equal( - []transactionParticipant{ - {TransactionID: 1, AccountID: 100}, - {TransactionID: 1, AccountID: 101}, - {TransactionID: 1, AccountID: 102}, - {TransactionID: 2, AccountID: 100}, - }, - participants, - ) + expected := []transactionParticipant{ + {TransactionID: 1}, + {TransactionID: 1}, + {TransactionID: 1}, + {TransactionID: 2}, + } + for i := range expected { + val, err := accountLoader.GetNow(addresses[i]) + tt.Assert.NoError(err) + expected[i].AccountID = val + } + tt.Assert.ElementsMatch(expected, participants) } diff --git a/services/horizon/internal/db2/history/trade.go b/services/horizon/internal/db2/history/trade.go index 65b6a1ce98..6d8a7fca56 100644 --- a/services/horizon/internal/db2/history/trade.go +++ b/services/horizon/internal/db2/history/trade.go @@ -348,7 +348,7 @@ func getCanonicalAssetOrder( type QTrades interface { QCreateAccountsHistory - NewTradeBatchInsertBuilder(maxBatchSize int) TradeBatchInsertBuilder + NewTradeBatchInsertBuilder() TradeBatchInsertBuilder RebuildTradeAggregationBuckets(ctx context.Context, fromledger, toLedger uint32, roundingSlippageFilter int) error CreateAssets(ctx context.Context, assets []xdr.Asset, maxBatchSize int) (map[string]Asset, error) CreateHistoryLiquidityPools(ctx context.Context, poolIDs []string, batchSize int) (map[string]int64, error) diff --git a/services/horizon/internal/db2/history/trade_batch_insert_builder.go b/services/horizon/internal/db2/history/trade_batch_insert_builder.go index 1f2d614424..8420fabd36 100644 --- a/services/horizon/internal/db2/history/trade_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/trade_batch_insert_builder.go @@ -24,7 +24,7 @@ const ( // rows into the history_trades table type InsertTrade struct { HistoryOperationID int64 `db:"history_operation_id"` - Order int32 `db:"\"order\""` + Order int32 `db:"order"` LedgerCloseTime time.Time `db:"ledger_closed_at"` CounterAssetID int64 `db:"counter_asset_id"` @@ -55,36 +55,33 @@ type InsertTrade struct { // TradeBatchInsertBuilder is used to insert trades into the // history_trades table type TradeBatchInsertBuilder interface { - Add(ctx context.Context, entries ...InsertTrade) error - Exec(ctx context.Context) error + Add(entries ...InsertTrade) error + Exec(ctx context.Context, session db.SessionInterface) error } // tradeBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder type tradeBatchInsertBuilder struct { - builder db.BatchInsertBuilder - q *Q + builder db.FastBatchInsertBuilder + table string } // NewTradeBatchInsertBuilder constructs a new TradeBatchInsertBuilder instance -func (q *Q) NewTradeBatchInsertBuilder(maxBatchSize int) TradeBatchInsertBuilder { +func (q *Q) NewTradeBatchInsertBuilder() TradeBatchInsertBuilder { return &tradeBatchInsertBuilder{ - builder: db.BatchInsertBuilder{ - Table: q.GetTable("history_trades"), - MaxBatchSize: maxBatchSize, - }, - q: q, + table: "history_trades", + builder: db.FastBatchInsertBuilder{}, } } // Exec flushes all outstanding trades to the database -func (i *tradeBatchInsertBuilder) Exec(ctx context.Context) error { - return i.builder.Exec(ctx) +func (i *tradeBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + return i.builder.Exec(ctx, session, i.table) } // Add adds a new trade to the batch -func (i *tradeBatchInsertBuilder) Add(ctx context.Context, entries ...InsertTrade) error { +func (i *tradeBatchInsertBuilder) Add(entries ...InsertTrade) error { for _, entry := range entries { - err := i.builder.RowStruct(ctx, entry) + err := i.builder.RowStruct(entry) if err != nil { return errors.Wrap(err, "failed to add trade") } diff --git a/services/horizon/internal/db2/history/trade_scenario.go b/services/horizon/internal/db2/history/trade_scenario.go index 22e1830277..7296238a35 100644 --- a/services/horizon/internal/db2/history/trade_scenario.go +++ b/services/horizon/internal/db2/history/trade_scenario.go @@ -199,7 +199,7 @@ func FilterTradesByType(trades []Trade, tradeType string) []Trade { // TradeScenario inserts trade rows into the Horizon DB func TradeScenario(tt *test.T, q *Q) TradeFixtures { - builder := q.NewTradeBatchInsertBuilder(0) + builder := q.NewTradeBatchInsertBuilder() addresses := []string{ "GB2QIYT2IAUFMRXKLSLLPRECC6OCOGJMADSPTRK7TGNT2SFR2YGWDARD", @@ -229,10 +229,12 @@ func TradeScenario(tt *test.T, q *Q) TradeFixtures { inserts := createInsertTrades(accountIDs, assetIDs, poolIDs, 3) + tt.Assert.NoError(q.Begin(tt.Ctx)) tt.Assert.NoError( - builder.Add(tt.Ctx, inserts...), + builder.Add(inserts...), ) - tt.Assert.NoError(builder.Exec(tt.Ctx)) + tt.Assert.NoError(builder.Exec(tt.Ctx, q)) + tt.Assert.NoError(q.Commit()) idToAccount := buildIDtoAccountMapping(addresses, accountIDs) idToAsset := buildIDtoAssetMapping(assets, assetIDs) diff --git a/services/horizon/internal/db2/history/transaction.go b/services/horizon/internal/db2/history/transaction.go index 0771a626b8..a308ab9ddc 100644 --- a/services/horizon/internal/db2/history/transaction.go +++ b/services/horizon/internal/db2/history/transaction.go @@ -232,8 +232,8 @@ func (q *TransactionsQ) Select(ctx context.Context, dest interface{}) error { // QTransactions defines transaction related queries. type QTransactions interface { - NewTransactionBatchInsertBuilder(maxBatchSize int) TransactionBatchInsertBuilder - NewTransactionFilteredTmpBatchInsertBuilder(maxBatchSize int) TransactionBatchInsertBuilder + NewTransactionBatchInsertBuilder() TransactionBatchInsertBuilder + NewTransactionFilteredTmpBatchInsertBuilder() TransactionBatchInsertBuilder } func selectTransaction(table string) sq.SelectBuilder { diff --git a/services/horizon/internal/db2/history/transaction_batch_insert_builder.go b/services/horizon/internal/db2/history/transaction_batch_insert_builder.go index 2ecb25dbe7..742621cec5 100644 --- a/services/horizon/internal/db2/history/transaction_batch_insert_builder.go +++ b/services/horizon/internal/db2/history/transaction_batch_insert_builder.go @@ -21,50 +21,47 @@ import ( // TransactionBatchInsertBuilder is used to insert transactions into the // history_transactions table type TransactionBatchInsertBuilder interface { - Add(ctx context.Context, transaction ingest.LedgerTransaction, sequence uint32) error - Exec(ctx context.Context) error + Add(transaction ingest.LedgerTransaction, sequence uint32) error + Exec(ctx context.Context, session db.SessionInterface) error } // transactionBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder type transactionBatchInsertBuilder struct { encodingBuffer *xdr.EncodingBuffer - builder db.BatchInsertBuilder + table string + builder db.FastBatchInsertBuilder } // NewTransactionBatchInsertBuilder constructs a new TransactionBatchInsertBuilder instance -func (q *Q) NewTransactionBatchInsertBuilder(maxBatchSize int) TransactionBatchInsertBuilder { +func (q *Q) NewTransactionBatchInsertBuilder() TransactionBatchInsertBuilder { return &transactionBatchInsertBuilder{ encodingBuffer: xdr.NewEncodingBuffer(), - builder: db.BatchInsertBuilder{ - Table: q.GetTable("history_transactions"), - MaxBatchSize: maxBatchSize, - }, + table: "history_transactions", + builder: db.FastBatchInsertBuilder{}, } } -// NewTransactionBatchInsertBuilder constructs a new TransactionBatchInsertBuilder instance -func (q *Q) NewTransactionFilteredTmpBatchInsertBuilder(maxBatchSize int) TransactionBatchInsertBuilder { +// NewTransactionFilteredTmpBatchInsertBuilder constructs a new TransactionBatchInsertBuilder instance +func (q *Q) NewTransactionFilteredTmpBatchInsertBuilder() TransactionBatchInsertBuilder { return &transactionBatchInsertBuilder{ encodingBuffer: xdr.NewEncodingBuffer(), - builder: db.BatchInsertBuilder{ - Table: q.GetTable("history_transactions_filtered_tmp"), - MaxBatchSize: maxBatchSize, - }, + table: "history_transactions_filtered_tmp", + builder: db.FastBatchInsertBuilder{}, } } // Add adds a new transaction to the batch -func (i *transactionBatchInsertBuilder) Add(ctx context.Context, transaction ingest.LedgerTransaction, sequence uint32) error { +func (i *transactionBatchInsertBuilder) Add(transaction ingest.LedgerTransaction, sequence uint32) error { row, err := transactionToRow(transaction, sequence, i.encodingBuffer) if err != nil { return err } - return i.builder.RowStruct(ctx, row) + return i.builder.RowStruct(row) } -func (i *transactionBatchInsertBuilder) Exec(ctx context.Context) error { - return i.builder.Exec(ctx) +func (i *transactionBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error { + return i.builder.Exec(ctx, session, i.table) } func signatures(xdrSignatures []xdr.DecoratedSignature) pq.StringArray { diff --git a/services/horizon/internal/db2/history/transaction_test.go b/services/horizon/internal/db2/history/transaction_test.go index 0f3592c439..30c22c7660 100644 --- a/services/horizon/internal/db2/history/transaction_test.go +++ b/services/horizon/internal/db2/history/transaction_test.go @@ -8,6 +8,7 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/guregu/null" + "github.com/stellar/go/xdr" "github.com/stellar/go/ingest" @@ -45,7 +46,8 @@ func TestTransactionByLiquidityPool(t *testing.T) { // Insert a phony ledger ledgerCloseTime := time.Now().Unix() - _, err := q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{ + ledgerBatch := q.NewLedgerBatchInsertBuilder() + err := ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ LedgerSeq: xdr.Uint32(sequence), ScpValue: xdr.StellarValue{ @@ -55,8 +57,12 @@ func TestTransactionByLiquidityPool(t *testing.T) { }, 0, 0, 0, 0, 0) tt.Assert.NoError(err) + tt.Assert.NoError(q.Begin(tt.Ctx)) + + tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q.SessionInterface)) + // Insert a phony transaction - transactionBuilder := q.NewTransactionBatchInsertBuilder(2) + transactionBuilder := q.NewTransactionBatchInsertBuilder() firstTransaction := buildLedgerTransaction(tt.T, testTransaction{ index: uint32(txIndex), envelopeXDR: "AAAAACiSTRmpH6bHC6Ekna5e82oiGY5vKDEEUgkq9CB//t+rAAAAyAEXUhsAADDRAAAAAAAAAAAAAAABAAAAAAAAAAsBF1IbAABX4QAAAAAAAAAA", @@ -65,27 +71,24 @@ func TestTransactionByLiquidityPool(t *testing.T) { metaXDR: "AAAAAQAAAAAAAAAA", hash: "19aaa18db88605aedec04659fb45e06f240b022eb2d429e05133e4d53cd945ba", }) - err = transactionBuilder.Add(tt.Ctx, firstTransaction, uint32(sequence)) + err = transactionBuilder.Add(firstTransaction, uint32(sequence)) tt.Assert.NoError(err) - err = transactionBuilder.Exec(tt.Ctx) + err = transactionBuilder.Exec(tt.Ctx, q) tt.Assert.NoError(err) // Insert Liquidity Pool history liquidityPoolID := "a2f38836a839de008cf1d782c81f45e1253cc5d3dad9110b872965484fec0a49" - toInternalID, err := q.CreateHistoryLiquidityPools(tt.Ctx, []string{liquidityPoolID}, 2) - tt.Assert.NoError(err) - lpTransactionBuilder := q.NewTransactionLiquidityPoolBatchInsertBuilder(2) - tt.Assert.NoError(err) - internalID, ok := toInternalID[liquidityPoolID] - tt.Assert.True(ok) - err = lpTransactionBuilder.Add(tt.Ctx, txID, internalID) - tt.Assert.NoError(err) - err = lpTransactionBuilder.Exec(tt.Ctx) - tt.Assert.NoError(err) + lpLoader := NewLiquidityPoolLoader() + lpTransactionBuilder := q.NewTransactionLiquidityPoolBatchInsertBuilder() + tt.Assert.NoError(lpTransactionBuilder.Add(txID, lpLoader.GetFuture(liquidityPoolID))) + tt.Assert.NoError(lpLoader.Exec(tt.Ctx, q)) + tt.Assert.NoError(lpTransactionBuilder.Exec(tt.Ctx, q)) + tt.Assert.NoError(q.Commit()) var records []Transaction - err = q.Transactions().ForLiquidityPool(tt.Ctx, liquidityPoolID).Select(tt.Ctx, &records) - tt.Assert.NoError(err) + tt.Assert.NoError( + q.Transactions().ForLiquidityPool(tt.Ctx, liquidityPoolID).Select(tt.Ctx, &records), + ) tt.Assert.Len(records, 1) } @@ -206,8 +209,10 @@ func TestInsertTransactionDoesNotAllowDuplicateIndex(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} + tt.Assert.NoError(q.Begin(tt.Ctx)) + sequence := uint32(123) - insertBuilder := q.NewTransactionBatchInsertBuilder(0) + insertBuilder := q.NewTransactionBatchInsertBuilder() firstTransaction := buildLedgerTransaction(tt.T, testTransaction{ index: 1, @@ -226,16 +231,18 @@ func TestInsertTransactionDoesNotAllowDuplicateIndex(t *testing.T) { hash: "7e2def20d5a21a56be2a457b648f702ee1af889d3df65790e92a05081e9fabf1", }) - tt.Assert.NoError(insertBuilder.Add(tt.Ctx, firstTransaction, sequence)) - tt.Assert.NoError(insertBuilder.Exec(tt.Ctx)) + tt.Assert.NoError(insertBuilder.Add(firstTransaction, sequence)) + tt.Assert.NoError(insertBuilder.Exec(tt.Ctx, q)) + tt.Assert.NoError(q.Commit()) - tt.Assert.NoError(insertBuilder.Add(tt.Ctx, secondTransaction, sequence)) + tt.Assert.NoError(q.Begin(tt.Ctx)) + insertBuilder = q.NewTransactionBatchInsertBuilder() + tt.Assert.NoError(insertBuilder.Add(secondTransaction, sequence)) tt.Assert.EqualError( - insertBuilder.Exec(tt.Ctx), - "error adding values while inserting to history_transactions: "+ - "exec failed: pq: duplicate key value violates unique constraint "+ - "\"hs_transaction_by_id\"", + insertBuilder.Exec(tt.Ctx, q), + "pq: duplicate key value violates unique constraint \"hs_transaction_by_id\"", ) + tt.Assert.NoError(q.Rollback()) ledger := Ledger{ Sequence: int32(sequence), @@ -301,8 +308,6 @@ func TestInsertTransaction(t *testing.T) { _, err := q.Exec(tt.Ctx, sq.Insert("history_ledgers").SetMap(ledgerToMap(ledger))) tt.Assert.NoError(err) - insertBuilder := q.NewTransactionBatchInsertBuilder(0) - success := true emptySignatures := []string{} @@ -822,8 +827,11 @@ func TestInsertTransaction(t *testing.T) { }, } { t.Run(testCase.name, func(t *testing.T) { - tt.Assert.NoError(insertBuilder.Add(tt.Ctx, testCase.toInsert, sequence)) - tt.Assert.NoError(insertBuilder.Exec(tt.Ctx)) + insertBuilder := q.NewTransactionBatchInsertBuilder() + tt.Assert.NoError(q.Begin(tt.Ctx)) + tt.Assert.NoError(insertBuilder.Add(testCase.toInsert, sequence)) + tt.Assert.NoError(insertBuilder.Exec(tt.Ctx, q)) + tt.Assert.NoError(q.Commit()) var transactions []Transaction tt.Assert.NoError(q.Transactions().IncludeFailed().Select(tt.Ctx, &transactions)) diff --git a/services/horizon/internal/ingest/fsm.go b/services/horizon/internal/ingest/fsm.go index 50f98879c2..2f9a40783c 100644 --- a/services/horizon/internal/ingest/fsm.go +++ b/services/horizon/internal/ingest/fsm.go @@ -457,6 +457,7 @@ func (r resumeState) run(s *system) (transition, error) { // Update cursor if there's more than one ingesting instance: either // Captive-Core or DB ingestion connected to another Stellar-Core. + // remove now? if err = s.updateCursor(lastIngestedLedger); err != nil { // Don't return updateCursor error. log.WithError(err).Warn("error updating stellar-core cursor") @@ -522,6 +523,7 @@ func (r resumeState) run(s *system) (transition, error) { return retryResume(r), err } + //TODO remove now? stellar-core-db-url is removed if err = s.updateCursor(ingestLedger); err != nil { // Don't return updateCursor error. log.WithError(err).Warn("error updating stellar-core cursor") diff --git a/services/horizon/internal/ingest/group_processors.go b/services/horizon/internal/ingest/group_processors.go index 86622810b5..af486a35cf 100644 --- a/services/horizon/internal/ingest/group_processors.go +++ b/services/horizon/internal/ingest/group_processors.go @@ -7,7 +7,9 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/ingest/processors" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" + "github.com/stellar/go/xdr" ) type processorsRunDurations map[string]time.Duration @@ -51,21 +53,23 @@ func (g groupChangeProcessors) Commit(ctx context.Context) error { } type groupTransactionProcessors struct { - processors []horizonTransactionProcessor + processors []horizonTransactionProcessor + lazyLoaders []horizonLazyLoader processorsRunDurations } -func newGroupTransactionProcessors(processors []horizonTransactionProcessor) *groupTransactionProcessors { +func newGroupTransactionProcessors(processors []horizonTransactionProcessor, lazyLoaders []horizonLazyLoader) *groupTransactionProcessors { return &groupTransactionProcessors{ processors: processors, processorsRunDurations: make(map[string]time.Duration), + lazyLoaders: lazyLoaders, } } -func (g groupTransactionProcessors) ProcessTransaction(ctx context.Context, tx ingest.LedgerTransaction) error { +func (g groupTransactionProcessors) ProcessTransaction(lcm xdr.LedgerCloseMeta, tx ingest.LedgerTransaction) error { for _, p := range g.processors { startTime := time.Now() - if err := p.ProcessTransaction(ctx, tx); err != nil { + if err := p.ProcessTransaction(lcm, tx); err != nil { return errors.Wrapf(err, "error in %T.ProcessTransaction", p) } g.AddRunDuration(fmt.Sprintf("%T", p), startTime) @@ -73,11 +77,21 @@ func (g groupTransactionProcessors) ProcessTransaction(ctx context.Context, tx i return nil } -func (g groupTransactionProcessors) Commit(ctx context.Context) error { +func (g groupTransactionProcessors) Flush(ctx context.Context, session db.SessionInterface) error { + // need to trigger all lazy loaders to now resolve their future placeholders + // with real db values first + for _, loader := range g.lazyLoaders { + if err := loader.Exec(ctx, session); err != nil { + return errors.Wrapf(err, "error during lazy loader resolution, %T.Exec", loader) + } + } + + // now flush each processor which may call loader.GetNow(), which + // required the prior loader.Exec() to have been called. for _, p := range g.processors { startTime := time.Now() - if err := p.Commit(ctx); err != nil { - return errors.Wrapf(err, "error in %T.Commit", p) + if err := p.Flush(ctx, session); err != nil { + return errors.Wrapf(err, "error in %T.Flush", p) } g.AddRunDuration(fmt.Sprintf("%T", p), startTime) } diff --git a/services/horizon/internal/ingest/group_processors_test.go b/services/horizon/internal/ingest/group_processors_test.go index 6848c24a66..73d4f56f3f 100644 --- a/services/horizon/internal/ingest/group_processors_test.go +++ b/services/horizon/internal/ingest/group_processors_test.go @@ -11,6 +11,8 @@ import ( "github.com/stretchr/testify/suite" "github.com/stellar/go/ingest" + "github.com/stellar/go/support/db" + "github.com/stellar/go/xdr" ) var _ horizonChangeProcessor = (*mockHorizonChangeProcessor)(nil) @@ -35,13 +37,13 @@ type mockHorizonTransactionProcessor struct { mock.Mock } -func (m *mockHorizonTransactionProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error { - args := m.Called(ctx, transaction) +func (m *mockHorizonTransactionProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error { + args := m.Called(lcm, transaction) return args.Error(0) } -func (m *mockHorizonTransactionProcessor) Commit(ctx context.Context) error { - args := m.Called(ctx) +func (m *mockHorizonTransactionProcessor) Flush(ctx context.Context, session db.SessionInterface) error { + args := m.Called(ctx, session) return args.Error(0) } @@ -124,6 +126,7 @@ type GroupTransactionProcessorsTestSuiteLedger struct { processors *groupTransactionProcessors processorA *mockHorizonTransactionProcessor processorB *mockHorizonTransactionProcessor + session db.SessionInterface } func TestGroupTransactionProcessorsTestSuiteLedger(t *testing.T) { @@ -137,7 +140,8 @@ func (s *GroupTransactionProcessorsTestSuiteLedger) SetupTest() { s.processors = newGroupTransactionProcessors([]horizonTransactionProcessor{ s.processorA, s.processorB, - }) + }, nil) + s.session = &db.MockSession{} } func (s *GroupTransactionProcessorsTestSuiteLedger) TearDownTest() { @@ -147,46 +151,48 @@ func (s *GroupTransactionProcessorsTestSuiteLedger) TearDownTest() { func (s *GroupTransactionProcessorsTestSuiteLedger) TestProcessTransactionFails() { transaction := ingest.LedgerTransaction{} + closeMeta := xdr.LedgerCloseMeta{} s.processorA. - On("ProcessTransaction", s.ctx, transaction). + On("ProcessTransaction", closeMeta, transaction). Return(errors.New("transient error")).Once() - err := s.processors.ProcessTransaction(s.ctx, transaction) + err := s.processors.ProcessTransaction(closeMeta, transaction) s.Assert().Error(err) s.Assert().EqualError(err, "error in *ingest.mockHorizonTransactionProcessor.ProcessTransaction: transient error") } func (s *GroupTransactionProcessorsTestSuiteLedger) TestProcessTransactionSucceeds() { transaction := ingest.LedgerTransaction{} + closeMeta := xdr.LedgerCloseMeta{} s.processorA. - On("ProcessTransaction", s.ctx, transaction). + On("ProcessTransaction", closeMeta, transaction). Return(nil).Once() s.processorB. - On("ProcessTransaction", s.ctx, transaction). + On("ProcessTransaction", closeMeta, transaction). Return(nil).Once() - err := s.processors.ProcessTransaction(s.ctx, transaction) + err := s.processors.ProcessTransaction(closeMeta, transaction) s.Assert().NoError(err) } -func (s *GroupTransactionProcessorsTestSuiteLedger) TestCommitFails() { +func (s *GroupTransactionProcessorsTestSuiteLedger) TestFlushFails() { s.processorA. - On("Commit", s.ctx). + On("Flush", s.ctx, s.session). Return(errors.New("transient error")).Once() - err := s.processors.Commit(s.ctx) + err := s.processors.Flush(s.ctx, s.session) s.Assert().Error(err) - s.Assert().EqualError(err, "error in *ingest.mockHorizonTransactionProcessor.Commit: transient error") + s.Assert().EqualError(err, "error in *ingest.mockHorizonTransactionProcessor.Flush: transient error") } -func (s *GroupTransactionProcessorsTestSuiteLedger) TestCommitSucceeds() { +func (s *GroupTransactionProcessorsTestSuiteLedger) TestFlushSucceeds() { s.processorA. - On("Commit", s.ctx). + On("Flush", s.ctx, s.session). Return(nil).Once() s.processorB. - On("Commit", s.ctx). + On("Flush", s.ctx, s.session). Return(nil).Once() - err := s.processors.Commit(s.ctx) + err := s.processors.Flush(s.ctx, s.session) s.Assert().NoError(err) } diff --git a/services/horizon/internal/ingest/main.go b/services/horizon/internal/ingest/main.go index 6535a3e097..b9ade405de 100644 --- a/services/horizon/internal/ingest/main.go +++ b/services/horizon/internal/ingest/main.go @@ -299,6 +299,7 @@ func NewSystem(config Config) (System, error) { ctx: ctx, config: config, historyQ: historyQ, + session: historyQ, historyAdapter: historyAdapter, filters: filters, }, diff --git a/services/horizon/internal/ingest/main_test.go b/services/horizon/internal/ingest/main_test.go index fb5ad15948..be2687b8e7 100644 --- a/services/horizon/internal/ingest/main_test.go +++ b/services/horizon/internal/ingest/main_test.go @@ -413,18 +413,18 @@ func (m *mockDBQ) DeleteRangeAll(ctx context.Context, start, end int64) error { // Methods from interfaces duplicating methods: -func (m *mockDBQ) NewTransactionParticipantsBatchInsertBuilder(maxBatchSize int) history.TransactionParticipantsBatchInsertBuilder { - args := m.Called(maxBatchSize) +func (m *mockDBQ) NewTransactionParticipantsBatchInsertBuilder() history.TransactionParticipantsBatchInsertBuilder { + args := m.Called() return args.Get(0).(history.TransactionParticipantsBatchInsertBuilder) } -func (m *mockDBQ) NewOperationParticipantBatchInsertBuilder(maxBatchSize int) history.OperationParticipantBatchInsertBuilder { - args := m.Called(maxBatchSize) - return args.Get(0).(history.TransactionParticipantsBatchInsertBuilder) +func (m *mockDBQ) NewOperationParticipantBatchInsertBuilder() history.OperationParticipantBatchInsertBuilder { + args := m.Called() + return args.Get(0).(history.OperationParticipantBatchInsertBuilder) } -func (m *mockDBQ) NewTradeBatchInsertBuilder(maxBatchSize int) history.TradeBatchInsertBuilder { - args := m.Called(maxBatchSize) +func (m *mockDBQ) NewTradeBatchInsertBuilder() history.TradeBatchInsertBuilder { + args := m.Called() return args.Get(0).(history.TradeBatchInsertBuilder) } diff --git a/services/horizon/internal/ingest/processor_runner.go b/services/horizon/internal/ingest/processor_runner.go index 41264ad262..cff8960c1d 100644 --- a/services/horizon/internal/ingest/processor_runner.go +++ b/services/horizon/internal/ingest/processor_runner.go @@ -10,6 +10,7 @@ import ( "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/ingest/filters" "github.com/stellar/go/services/horizon/internal/ingest/processors" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) @@ -31,7 +32,10 @@ type horizonChangeProcessor interface { type horizonTransactionProcessor interface { processors.LedgerTransactionProcessor - Commit(context.Context) error +} + +type horizonLazyLoader interface { + Exec(ctx context.Context, session db.SessionInterface) error } type statsChangeProcessor struct { @@ -46,10 +50,6 @@ type statsLedgerTransactionProcessor struct { *processors.StatsLedgerTransactionProcessor } -func (statsLedgerTransactionProcessor) Commit(ctx context.Context) error { - return nil -} - type ledgerStats struct { changeStats ingest.StatsChangeProcessorResults changeDurations processorsRunDurations @@ -88,6 +88,7 @@ type ProcessorRunner struct { ctx context.Context historyQ history.IngestionQ + session db.SessionInterface historyAdapter historyArchiveAdapterInterface logMemoryStats bool filters filters.Filters @@ -134,24 +135,36 @@ func buildChangeProcessor( func (s *ProcessorRunner) buildTransactionProcessor( ledgerTransactionStats *processors.StatsLedgerTransactionProcessor, tradeProcessor *processors.TradeProcessor, - ledger xdr.LedgerHeaderHistoryEntry, + ledgersProcessor *processors.LedgersProcessor, ) *groupTransactionProcessors { + accountLoader := history.NewAccountLoader() + assetLoader := history.NewAssetLoader() + lpLoader := history.NewLiquidityPoolLoader() + cbLoader := history.NewClaimableBalanceLoader() + + lazyLoaders := []horizonLazyLoader{accountLoader, assetLoader, lpLoader, cbLoader} + statsLedgerTransactionProcessor := &statsLedgerTransactionProcessor{ StatsLedgerTransactionProcessor: ledgerTransactionStats, } - *tradeProcessor = *processors.NewTradeProcessor(s.historyQ, ledger) - sequence := uint32(ledger.Header.LedgerSeq) - return newGroupTransactionProcessors([]horizonTransactionProcessor{ + *tradeProcessor = *processors.NewTradeProcessor(accountLoader, + lpLoader, assetLoader, s.historyQ.NewTradeBatchInsertBuilder()) + + processors := []horizonTransactionProcessor{ statsLedgerTransactionProcessor, - processors.NewEffectProcessor(s.historyQ, sequence, s.config.NetworkPassphrase), - processors.NewLedgerProcessor(s.historyQ, ledger, CurrentVersion), - processors.NewOperationProcessor(s.historyQ, sequence, s.config.NetworkPassphrase), + processors.NewEffectProcessor(accountLoader, s.historyQ.NewEffectBatchInsertBuilder(), s.config.NetworkPassphrase), + ledgersProcessor, + processors.NewOperationProcessor(s.historyQ.NewOperationBatchInsertBuilder(), s.config.NetworkPassphrase), tradeProcessor, - processors.NewParticipantsProcessor(s.historyQ, sequence), - processors.NewTransactionProcessor(s.historyQ, sequence), - processors.NewClaimableBalancesTransactionProcessor(s.historyQ, sequence), - processors.NewLiquidityPoolsTransactionProcessor(s.historyQ, sequence), - }) + processors.NewParticipantsProcessor(accountLoader, + s.historyQ.NewTransactionParticipantsBatchInsertBuilder(), s.historyQ.NewOperationParticipantBatchInsertBuilder()), + processors.NewTransactionProcessor(s.historyQ.NewTransactionBatchInsertBuilder()), + processors.NewClaimableBalancesTransactionProcessor(cbLoader, + s.historyQ.NewTransactionClaimableBalanceBatchInsertBuilder(), s.historyQ.NewOperationClaimableBalanceBatchInsertBuilder()), + processors.NewLiquidityPoolsTransactionProcessor(lpLoader, + s.historyQ.NewTransactionLiquidityPoolBatchInsertBuilder(), s.historyQ.NewOperationLiquidityPoolBatchInsertBuilder())} + + return newGroupTransactionProcessors(processors, lazyLoaders) } func (s *ProcessorRunner) buildTransactionFilterer() *groupTransactionFilterers { @@ -163,15 +176,15 @@ func (s *ProcessorRunner) buildTransactionFilterer() *groupTransactionFilterers return newGroupTransactionFilterers(f) } -func (s *ProcessorRunner) buildFilteredOutProcessor(ledger xdr.LedgerHeaderHistoryEntry) *groupTransactionProcessors { +func (s *ProcessorRunner) buildFilteredOutProcessor() *groupTransactionProcessors { // when in online mode, the submission result processor must always run (regardless of filtering) var p []horizonTransactionProcessor if s.config.EnableIngestionFiltering { - txSubProc := processors.NewTransactionFilteredTmpProcessor(s.historyQ, uint32(ledger.Header.LedgerSeq)) + txSubProc := processors.NewTransactionFilteredTmpProcessor(s.historyQ.NewTransactionFilteredTmpBatchInsertBuilder()) p = append(p, txSubProc) } - return newGroupTransactionProcessors(p) + return newGroupTransactionProcessors(p, nil) } // checkIfProtocolVersionSupported checks if this Horizon version supports the @@ -316,26 +329,31 @@ func (s *ProcessorRunner) RunTransactionProcessorsOnLedger(ledger xdr.LedgerClos transactionReader *ingest.LedgerTransactionReader ) + if err = s.checkIfProtocolVersionSupported(ledger.ProtocolVersion()); err != nil { + err = errors.Wrap(err, "Error while checking for supported protocol version") + return + } + + // ensure capture of the ledger to history regardless of whether it has transactions. + ledgersProcessor := processors.NewLedgerProcessor(s.historyQ.NewLedgerBatchInsertBuilder(), CurrentVersion) + ledgersProcessor.ProcessLedger(ledger) + transactionReader, err = ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(s.config.NetworkPassphrase, ledger) if err != nil { err = errors.Wrap(err, "Error creating ledger reader") return } - if err = s.checkIfProtocolVersionSupported(ledger.ProtocolVersion()); err != nil { - err = errors.Wrap(err, "Error while checking for supported protocol version") - return - } - header := transactionReader.GetHeader() groupTransactionFilterers := s.buildTransactionFilterer() - groupFilteredOutProcessors := s.buildFilteredOutProcessor(header) + groupFilteredOutProcessors := s.buildFilteredOutProcessor() groupTransactionProcessors := s.buildTransactionProcessor( - &ledgerTransactionStats, &tradeProcessor, header) + &ledgerTransactionStats, &tradeProcessor, ledgersProcessor) err = processors.StreamLedgerTransactions(s.ctx, groupTransactionFilterers, groupFilteredOutProcessors, groupTransactionProcessors, transactionReader, + ledger, ) if err != nil { err = errors.Wrap(err, "Error streaming changes from ledger") @@ -343,9 +361,9 @@ func (s *ProcessorRunner) RunTransactionProcessorsOnLedger(ledger xdr.LedgerClos } if s.config.EnableIngestionFiltering { - err = groupFilteredOutProcessors.Commit(s.ctx) + err = groupFilteredOutProcessors.Flush(s.ctx, s.session) if err != nil { - err = errors.Wrap(err, "Error committing filtered changes from processor") + err = errors.Wrap(err, "Error flushing temp filtered tx from processor") return } if time.Since(s.lastTransactionsTmpGC) > transactionsFilteredTmpGCPeriod { @@ -353,9 +371,9 @@ func (s *ProcessorRunner) RunTransactionProcessorsOnLedger(ledger xdr.LedgerClos } } - err = groupTransactionProcessors.Commit(s.ctx) + err = groupTransactionProcessors.Flush(s.ctx, s.session) if err != nil { - err = errors.Wrap(err, "Error committing changes from processor") + err = errors.Wrap(err, "Error flushing changes from processor") return } @@ -401,9 +419,6 @@ func (s *ProcessorRunner) RunAllProcessorsOnLedger(ledger xdr.LedgerCloseMeta) ( stats.transactionStats, stats.transactionDurations, stats.tradeStats, err = s.RunTransactionProcessorsOnLedger(ledger) - if err != nil { - return - } return } diff --git a/services/horizon/internal/ingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go index 507ba64da7..4af1a5be11 100644 --- a/services/horizon/internal/ingest/processor_runner_test.go +++ b/services/horizon/internal/ingest/processor_runner_test.go @@ -16,6 +16,7 @@ import ( "github.com/stellar/go/network" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/services/horizon/internal/ingest/processors" + "github.com/stellar/go/support/db" "github.com/stellar/go/xdr" ) @@ -233,17 +234,31 @@ func TestProcessorRunnerBuildChangeProcessor(t *testing.T) { func TestProcessorRunnerBuildTransactionProcessor(t *testing.T) { ctx := context.Background() - maxBatchSize := 100000 q := &mockDBQ{} defer mock.AssertExpectationsForObjects(t, q) - q.MockQOperations.On("NewOperationBatchInsertBuilder", maxBatchSize). - Return(&history.MockOperationsBatchInsertBuilder{}).Twice() // Twice = with/without failed - q.MockQTransactions.On("NewTransactionBatchInsertBuilder", maxBatchSize). - Return(&history.MockTransactionsBatchInsertBuilder{}).Twice() - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize). - Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Twice() + q.MockQTransactions.On("NewTransactionBatchInsertBuilder"). + Return(&history.MockTransactionsBatchInsertBuilder{}) + q.On("NewTradeBatchInsertBuilder").Return(&history.MockTradeBatchInsertBuilder{}) + q.MockQLedgers.On("NewLedgerBatchInsertBuilder"). + Return(&history.MockLedgersBatchInsertBuilder{}) + q.MockQEffects.On("NewEffectBatchInsertBuilder"). + Return(&history.MockEffectBatchInsertBuilder{}) + q.MockQOperations.On("NewOperationBatchInsertBuilder"). + Return(&history.MockOperationsBatchInsertBuilder{}) + q.On("NewTransactionParticipantsBatchInsertBuilder"). + Return(&history.MockTransactionParticipantsBatchInsertBuilder{}) + q.On("NewOperationParticipantBatchInsertBuilder"). + Return(&history.MockOperationParticipantBatchInsertBuilder{}) + q.MockQHistoryClaimableBalances.On("NewTransactionClaimableBalanceBatchInsertBuilder"). + Return(&history.MockTransactionClaimableBalanceBatchInsertBuilder{}) + q.MockQHistoryClaimableBalances.On("NewOperationClaimableBalanceBatchInsertBuilder"). + Return(&history.MockOperationClaimableBalanceBatchInsertBuilder{}) + q.MockQHistoryLiquidityPools.On("NewTransactionLiquidityPoolBatchInsertBuilder"). + Return(&history.MockTransactionLiquidityPoolBatchInsertBuilder{}) + q.MockQHistoryLiquidityPools.On("NewOperationLiquidityPoolBatchInsertBuilder"). + Return(&history.MockOperationLiquidityPoolBatchInsertBuilder{}) runner := ProcessorRunner{ ctx: ctx, @@ -253,17 +268,19 @@ func TestProcessorRunnerBuildTransactionProcessor(t *testing.T) { stats := &processors.StatsLedgerTransactionProcessor{} trades := &processors.TradeProcessor{} - ledger := xdr.LedgerHeaderHistoryEntry{} - processor := runner.buildTransactionProcessor(stats, trades, ledger) - assert.IsType(t, &groupTransactionProcessors{}, processor) + ledgersProcessor := &processors.LedgersProcessor{} + + processor := runner.buildTransactionProcessor(stats, trades, ledgersProcessor) + assert.IsType(t, &groupTransactionProcessors{}, processor) assert.IsType(t, &statsLedgerTransactionProcessor{}, processor.processors[0]) assert.IsType(t, &processors.EffectProcessor{}, processor.processors[1]) assert.IsType(t, &processors.LedgersProcessor{}, processor.processors[2]) assert.IsType(t, &processors.OperationProcessor{}, processor.processors[3]) assert.IsType(t, &processors.TradeProcessor{}, processor.processors[4]) assert.IsType(t, &processors.ParticipantsProcessor{}, processor.processors[5]) - assert.IsType(t, &processors.TransactionProcessor{}, processor.processors[6]) + assert.IsType(t, &processors.ClaimableBalancesTransactionProcessor{}, processor.processors[7]) + assert.IsType(t, &processors.LiquidityPoolsTransactionProcessor{}, processor.processors[8]) } func TestProcessorRunnerWithFilterEnabled(t *testing.T) { @@ -276,6 +293,7 @@ func TestProcessorRunnerWithFilterEnabled(t *testing.T) { } q := &mockDBQ{} + mockSession := &db.MockSession{} defer mock.AssertExpectationsForObjects(t, q) ledger := xdr.LedgerCloseMeta{ @@ -289,40 +307,33 @@ func TestProcessorRunnerWithFilterEnabled(t *testing.T) { } // Batches - mockAccountSignersBatchInsertBuilder := &history.MockAccountSignersBatchInsertBuilder{} - defer mock.AssertExpectationsForObjects(t, mockAccountSignersBatchInsertBuilder) - q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize). - Return(mockAccountSignersBatchInsertBuilder).Once() - - mockOperationsBatchInsertBuilder := &history.MockOperationsBatchInsertBuilder{} - defer mock.AssertExpectationsForObjects(t, mockOperationsBatchInsertBuilder) - mockOperationsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() - q.MockQOperations.On("NewOperationBatchInsertBuilder", maxBatchSize). - Return(mockOperationsBatchInsertBuilder).Twice() - - mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{} - defer mock.AssertExpectationsForObjects(t, mockTransactionsBatchInsertBuilder) - mockTransactionsBatchInsertBuilder.On("Exec", ctx).Return(nil).Twice() - - q.MockQTransactions.On("NewTransactionBatchInsertBuilder", maxBatchSize). - Return(mockTransactionsBatchInsertBuilder) - - q.MockQTransactions.On("NewTransactionFilteredTmpBatchInsertBuilder", maxBatchSize). - Return(mockTransactionsBatchInsertBuilder) - - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize). - Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() + mockTransactionsFilteredTmpBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{} + defer mock.AssertExpectationsForObjects(t, mockTransactionsFilteredTmpBatchInsertBuilder) + mockTransactionsFilteredTmpBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once() + q.MockQTransactions.On("NewTransactionFilteredTmpBatchInsertBuilder"). + Return(mockTransactionsFilteredTmpBatchInsertBuilder) q.On("DeleteTransactionsFilteredTmpOlderThan", ctx, mock.AnythingOfType("uint64")). Return(int64(0), nil) - q.MockQLedgers.On("InsertLedger", ctx, ledger.V0.LedgerHeader, 0, 0, 0, 0, CurrentVersion). - Return(int64(1), nil).Once() + defer mock.AssertExpectationsForObjects(t, mockBatchBuilders(q, mockSession, ctx, maxBatchSize)...) + + mockBatchInsertBuilder := &history.MockLedgersBatchInsertBuilder{} + q.MockQLedgers.On("NewLedgerBatchInsertBuilder").Return(mockBatchInsertBuilder) + mockBatchInsertBuilder.On( + "Add", + ledger.V0.LedgerHeader, 0, 0, 0, 0, CurrentVersion).Return(nil) + mockBatchInsertBuilder.On( + "Exec", + ctx, + mockSession, + ).Return(nil) runner := ProcessorRunner{ ctx: ctx, config: config, historyQ: q, + session: mockSession, filters: &MockFilters{}, } @@ -338,6 +349,7 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) { NetworkPassphrase: network.PublicNetworkPassphrase, } + mockSession := &db.MockSession{} q := &mockDBQ{} defer mock.AssertExpectationsForObjects(t, q) @@ -352,33 +364,24 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) { } // Batches - mockAccountSignersBatchInsertBuilder := &history.MockAccountSignersBatchInsertBuilder{} - defer mock.AssertExpectationsForObjects(t, mockAccountSignersBatchInsertBuilder) - q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize). - Return(mockAccountSignersBatchInsertBuilder).Once() - - mockOperationsBatchInsertBuilder := &history.MockOperationsBatchInsertBuilder{} - defer mock.AssertExpectationsForObjects(t, mockOperationsBatchInsertBuilder) - mockOperationsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() - q.MockQOperations.On("NewOperationBatchInsertBuilder", maxBatchSize). - Return(mockOperationsBatchInsertBuilder).Twice() - - mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{} - defer mock.AssertExpectationsForObjects(t, mockTransactionsBatchInsertBuilder) - mockTransactionsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() - q.MockQTransactions.On("NewTransactionBatchInsertBuilder", maxBatchSize). - Return(mockTransactionsBatchInsertBuilder).Twice() - - q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize). - Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() - - q.MockQLedgers.On("InsertLedger", ctx, ledger.V0.LedgerHeader, 0, 0, 0, 0, CurrentVersion). - Return(int64(1), nil).Once() + defer mock.AssertExpectationsForObjects(t, mockBatchBuilders(q, mockSession, ctx, maxBatchSize)...) + + mockBatchInsertBuilder := &history.MockLedgersBatchInsertBuilder{} + q.MockQLedgers.On("NewLedgerBatchInsertBuilder").Return(mockBatchInsertBuilder) + mockBatchInsertBuilder.On( + "Add", + ledger.V0.LedgerHeader, 0, 0, 0, 0, CurrentVersion).Return(nil) + mockBatchInsertBuilder.On( + "Exec", + ctx, + mockSession, + ).Return(nil) runner := ProcessorRunner{ ctx: ctx, config: config, historyQ: q, + session: mockSession, filters: &MockFilters{}, } @@ -408,21 +411,21 @@ func TestProcessorRunnerRunAllProcessorsOnLedgerProtocolVersionNotSupported(t *t } // Batches + mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{} + q.MockQTransactions.On("NewTransactionBatchInsertBuilder", maxBatchSize). + Return(mockTransactionsBatchInsertBuilder).Twice() mockAccountSignersBatchInsertBuilder := &history.MockAccountSignersBatchInsertBuilder{} - defer mock.AssertExpectationsForObjects(t, mockAccountSignersBatchInsertBuilder) q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize). Return(mockAccountSignersBatchInsertBuilder).Once() mockOperationsBatchInsertBuilder := &history.MockOperationsBatchInsertBuilder{} - defer mock.AssertExpectationsForObjects(t, mockOperationsBatchInsertBuilder) - q.MockQOperations.On("NewOperationBatchInsertBuilder", maxBatchSize). + q.MockQOperations.On("NewOperationBatchInsertBuilder"). Return(mockOperationsBatchInsertBuilder).Twice() - mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{} - defer mock.AssertExpectationsForObjects(t, mockTransactionsBatchInsertBuilder) - q.MockQTransactions.On("NewTransactionBatchInsertBuilder", maxBatchSize). - Return(mockTransactionsBatchInsertBuilder).Twice() + defer mock.AssertExpectationsForObjects(t, mockTransactionsBatchInsertBuilder, + mockAccountSignersBatchInsertBuilder, + mockOperationsBatchInsertBuilder) runner := ProcessorRunner{ ctx: ctx, @@ -439,3 +442,63 @@ func TestProcessorRunnerRunAllProcessorsOnLedgerProtocolVersionNotSupported(t *t ), ) } + +func mockBatchBuilders(q *mockDBQ, mockSession *db.MockSession, ctx context.Context, maxBatchSize int) []interface{} { + mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{} + mockTransactionsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once() + q.MockQTransactions.On("NewTransactionBatchInsertBuilder"). + Return(mockTransactionsBatchInsertBuilder) + + mockAccountSignersBatchInsertBuilder := &history.MockAccountSignersBatchInsertBuilder{} + q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize). + Return(mockAccountSignersBatchInsertBuilder).Once() + + mockOperationsBatchInsertBuilder := &history.MockOperationsBatchInsertBuilder{} + mockOperationsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once() + q.MockQOperations.On("NewOperationBatchInsertBuilder"). + Return(mockOperationsBatchInsertBuilder).Twice() + + mockEffectBatchInsertBuilder := &history.MockEffectBatchInsertBuilder{} + mockEffectBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once() + q.MockQEffects.On("NewEffectBatchInsertBuilder"). + Return(mockEffectBatchInsertBuilder) + + mockTransactionsParticipantsBatchInsertBuilder := &history.MockTransactionParticipantsBatchInsertBuilder{} + mockTransactionsParticipantsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil) + q.On("NewTransactionParticipantsBatchInsertBuilder"). + Return(mockTransactionsParticipantsBatchInsertBuilder) + + mockOperationParticipantBatchInsertBuilder := &history.MockOperationParticipantBatchInsertBuilder{} + mockOperationParticipantBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil) + q.On("NewOperationParticipantBatchInsertBuilder"). + Return(mockOperationParticipantBatchInsertBuilder) + + mockTransactionClaimableBalanceBatchInsertBuilder := &history.MockTransactionClaimableBalanceBatchInsertBuilder{} + mockTransactionClaimableBalanceBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil) + q.MockQHistoryClaimableBalances.On("NewTransactionClaimableBalanceBatchInsertBuilder"). + Return(mockTransactionClaimableBalanceBatchInsertBuilder) + + mockOperationClaimableBalanceBatchInsertBuilder := &history.MockOperationClaimableBalanceBatchInsertBuilder{} + mockOperationClaimableBalanceBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil) + q.MockQHistoryClaimableBalances.On("NewOperationClaimableBalanceBatchInsertBuilder"). + Return(mockOperationClaimableBalanceBatchInsertBuilder) + + mockTransactionLiquidityPoolBatchInsertBuilder := &history.MockTransactionLiquidityPoolBatchInsertBuilder{} + mockTransactionLiquidityPoolBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil) + q.MockQHistoryLiquidityPools.On("NewTransactionLiquidityPoolBatchInsertBuilder"). + Return(mockTransactionLiquidityPoolBatchInsertBuilder) + + mockOperationLiquidityPoolBatchInsertBuilder := &history.MockOperationLiquidityPoolBatchInsertBuilder{} + mockOperationLiquidityPoolBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil) + q.MockQHistoryLiquidityPools.On("NewOperationLiquidityPoolBatchInsertBuilder"). + Return(mockOperationLiquidityPoolBatchInsertBuilder) + + q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize). + Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once() + + q.On("NewTradeBatchInsertBuilder").Return(&history.MockTradeBatchInsertBuilder{}) + + return []interface{}{mockAccountSignersBatchInsertBuilder, + mockOperationsBatchInsertBuilder, + mockTransactionsBatchInsertBuilder} +} diff --git a/services/horizon/internal/ingest/processors/change_processors.go b/services/horizon/internal/ingest/processors/change_processors.go index 2e5b126d8f..ee9eb127f1 100644 --- a/services/horizon/internal/ingest/processors/change_processors.go +++ b/services/horizon/internal/ingest/processors/change_processors.go @@ -8,63 +8,6 @@ import ( "github.com/stellar/go/support/errors" ) -type ChangeProcessor interface { - ProcessChange(ctx context.Context, change ingest.Change) error -} - -type LedgerTransactionProcessor interface { - ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error -} - -type LedgerTransactionFilterer interface { - FilterTransaction(ctx context.Context, transaction ingest.LedgerTransaction) (bool, error) -} - -func StreamLedgerTransactions( - ctx context.Context, - txFilterer LedgerTransactionFilterer, - filteredTxProcessor LedgerTransactionProcessor, - txProcessor LedgerTransactionProcessor, - reader *ingest.LedgerTransactionReader, -) error { - for { - tx, err := reader.Read() - if err == io.EOF { - return nil - } - if err != nil { - return errors.Wrap(err, "could not read transaction") - } - include, err := txFilterer.FilterTransaction(ctx, tx) - if err != nil { - return errors.Wrapf( - err, - "could not filter transaction %v", - tx.Index, - ) - } - if !include { - if err = filteredTxProcessor.ProcessTransaction(ctx, tx); err != nil { - return errors.Wrapf( - err, - "could not process transaction %v", - tx.Index, - ) - } - log.Debugf("Filters did not find match on transaction, dropping this tx with hash %v", tx.Result.TransactionHash.HexString()) - continue - } - - if err = txProcessor.ProcessTransaction(ctx, tx); err != nil { - return errors.Wrapf( - err, - "could not process transaction %v", - tx.Index, - ) - } - } -} - func StreamChanges( ctx context.Context, changeProcessor ChangeProcessor, diff --git a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor.go b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor.go index 33c8bfd9c6..394d2e0f9b 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor.go @@ -5,53 +5,39 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" - set "github.com/stellar/go/support/collections/set" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/toid" "github.com/stellar/go/xdr" ) -type claimableBalance struct { - internalID int64 // Bigint auto-generated by postgres - transactionSet set.Set[int64] - operationSet set.Set[int64] -} - -func (b *claimableBalance) addTransactionID(id int64) { - if b.transactionSet == nil { - b.transactionSet = set.Set[int64]{} - } - b.transactionSet.Add(id) -} - -func (b *claimableBalance) addOperationID(id int64) { - if b.operationSet == nil { - b.operationSet = set.Set[int64]{} - } - b.operationSet.Add(id) -} - type ClaimableBalancesTransactionProcessor struct { - sequence uint32 - claimableBalanceSet map[string]claimableBalance - qClaimableBalances history.QHistoryClaimableBalances + cbLoader *history.ClaimableBalanceLoader + txBatch history.TransactionClaimableBalanceBatchInsertBuilder + opBatch history.OperationClaimableBalanceBatchInsertBuilder } -func NewClaimableBalancesTransactionProcessor(Q history.QHistoryClaimableBalances, sequence uint32) *ClaimableBalancesTransactionProcessor { +func NewClaimableBalancesTransactionProcessor( + cbLoader *history.ClaimableBalanceLoader, + txBatch history.TransactionClaimableBalanceBatchInsertBuilder, + opBatch history.OperationClaimableBalanceBatchInsertBuilder, +) *ClaimableBalancesTransactionProcessor { return &ClaimableBalancesTransactionProcessor{ - qClaimableBalances: Q, - sequence: sequence, - claimableBalanceSet: map[string]claimableBalance{}, + cbLoader: cbLoader, + txBatch: txBatch, + opBatch: opBatch, } } -func (p *ClaimableBalancesTransactionProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error { - err := p.addTransactionClaimableBalances(p.claimableBalanceSet, p.sequence, transaction) +func (p *ClaimableBalancesTransactionProcessor) ProcessTransaction( + lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction, +) error { + err := p.addTransactionClaimableBalances(lcm.LedgerSequence(), transaction) if err != nil { return err } - err = p.addOperationClaimableBalances(p.claimableBalanceSet, p.sequence, transaction) + err = p.addOperationClaimableBalances(lcm.LedgerSequence(), transaction) if err != nil { return err } @@ -59,27 +45,25 @@ func (p *ClaimableBalancesTransactionProcessor) ProcessTransaction(ctx context.C return nil } -func (p *ClaimableBalancesTransactionProcessor) addTransactionClaimableBalances(cbSet map[string]claimableBalance, sequence uint32, transaction ingest.LedgerTransaction) error { +func (p *ClaimableBalancesTransactionProcessor) addTransactionClaimableBalances( + sequence uint32, transaction ingest.LedgerTransaction, +) error { transactionID := toid.New(int32(sequence), int32(transaction.Index), 0).ToInt64() - transactionClaimableBalances, err := claimableBalancesForTransaction( - sequence, - transaction, - ) + transactionClaimableBalances, err := claimableBalancesForTransaction(transaction) if err != nil { return errors.Wrap(err, "Could not determine claimable balances for transaction") } - for _, cb := range transactionClaimableBalances { - entry := cbSet[cb] - entry.addTransactionID(transactionID) - cbSet[cb] = entry + for _, cb := range dedupeStrings(transactionClaimableBalances) { + if err = p.txBatch.Add(transactionID, p.cbLoader.GetFuture(cb)); err != nil { + return err + } } return nil } func claimableBalancesForTransaction( - sequence uint32, transaction ingest.LedgerTransaction, ) ([]string, error) { changes, err := transaction.GetChanges() @@ -90,19 +74,7 @@ func claimableBalancesForTransaction( if err != nil { return nil, errors.Wrapf(err, "reading transaction %v claimable balances", transaction.Index) } - return dedupeClaimableBalances(cbs) -} - -func dedupeClaimableBalances(in []string) (out []string, err error) { - set := set.Set[string]{} - for _, id := range in { - set.Add(id) - } - - for id := range set { - out = append(out, id) - } - return + return cbs, nil } func claimableBalancesForChanges( @@ -136,26 +108,9 @@ func claimableBalancesForChanges( return cbs, nil } -func (p *ClaimableBalancesTransactionProcessor) addOperationClaimableBalances(cbSet map[string]claimableBalance, sequence uint32, transaction ingest.LedgerTransaction) error { - claimableBalances, err := claimableBalancesForOperations(transaction, sequence) - if err != nil { - return errors.Wrap(err, "could not determine operation claimable balances") - } - - for operationID, cbs := range claimableBalances { - for _, cb := range cbs { - entry := cbSet[cb] - entry.addOperationID(operationID) - cbSet[cb] = entry - } - } - - return nil -} - -func claimableBalancesForOperations(transaction ingest.LedgerTransaction, sequence uint32) (map[int64][]string, error) { - cbs := map[int64][]string{} - +func (p *ClaimableBalancesTransactionProcessor) addOperationClaimableBalances( + sequence uint32, transaction ingest.LedgerTransaction, +) error { for opi, op := range transaction.Envelope.Operations() { operation := transactionOperationWrapper{ index: uint32(opi), @@ -166,92 +121,33 @@ func claimableBalancesForOperations(transaction ingest.LedgerTransaction, sequen changes, err := transaction.GetOperationChanges(uint32(opi)) if err != nil { - return cbs, err - } - c, err := claimableBalancesForChanges(changes) - if err != nil { - return cbs, errors.Wrapf(err, "reading operation %v claimable balances", operation.ID()) - } - cbs[operation.ID()] = c - } - - return cbs, nil -} - -func (p *ClaimableBalancesTransactionProcessor) Commit(ctx context.Context) error { - if len(p.claimableBalanceSet) > 0 { - if err := p.loadClaimableBalanceIDs(ctx, p.claimableBalanceSet); err != nil { return err } - - if err := p.insertDBTransactionClaimableBalances(ctx, p.claimableBalanceSet); err != nil { - return err + cbs, err := claimableBalancesForChanges(changes) + if err != nil { + return errors.Wrapf(err, "reading operation %v claimable balances", operation.ID()) } - if err := p.insertDBOperationsClaimableBalances(ctx, p.claimableBalanceSet); err != nil { - return err + for _, cb := range dedupeStrings(cbs) { + if err = p.opBatch.Add(operation.ID(), p.cbLoader.GetFuture(cb)); err != nil { + return err + } } } return nil } -func (p *ClaimableBalancesTransactionProcessor) loadClaimableBalanceIDs(ctx context.Context, claimableBalanceSet map[string]claimableBalance) error { - ids := make([]string, 0, len(claimableBalanceSet)) - for id := range claimableBalanceSet { - ids = append(ids, id) - } - - toInternalID, err := p.qClaimableBalances.CreateHistoryClaimableBalances(ctx, ids, maxBatchSize) +func (p *ClaimableBalancesTransactionProcessor) Flush(ctx context.Context, session db.SessionInterface) error { + err := p.txBatch.Exec(ctx, session) if err != nil { - return errors.Wrap(err, "Could not create claimable balance ids") - } - - for _, id := range ids { - internalID, ok := toInternalID[id] - if !ok { - // TODO: Figure out the right way to convert the id to a string here. %v will be nonsense. - return errors.Errorf("no internal id found for claimable balance %v", id) - } - - cb := claimableBalanceSet[id] - cb.internalID = internalID - claimableBalanceSet[id] = cb - } - - return nil -} - -func (p ClaimableBalancesTransactionProcessor) insertDBTransactionClaimableBalances(ctx context.Context, claimableBalanceSet map[string]claimableBalance) error { - batch := p.qClaimableBalances.NewTransactionClaimableBalanceBatchInsertBuilder(maxBatchSize) - - for _, entry := range claimableBalanceSet { - for transactionID := range entry.transactionSet { - if err := batch.Add(ctx, transactionID, entry.internalID); err != nil { - return errors.Wrap(err, "could not insert transaction claimable balance in db") - } - } - } - - if err := batch.Exec(ctx); err != nil { - return errors.Wrap(err, "could not flush transaction claimable balances to db") + return err } - return nil -} -func (p ClaimableBalancesTransactionProcessor) insertDBOperationsClaimableBalances(ctx context.Context, claimableBalanceSet map[string]claimableBalance) error { - batch := p.qClaimableBalances.NewOperationClaimableBalanceBatchInsertBuilder(maxBatchSize) - - for _, entry := range claimableBalanceSet { - for operationID := range entry.operationSet { - if err := batch.Add(ctx, operationID, entry.internalID); err != nil { - return errors.Wrap(err, "could not insert operation claimable balance in db") - } - } + err = p.opBatch.Exec(ctx, session) + if err != nil { + return err } - if err := batch.Exec(ctx); err != nil { - return errors.Wrap(err, "could not flush operation claimable balances to db") - } return nil } diff --git a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go index abdaf5b070..11ce54505a 100644 --- a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go +++ b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go @@ -6,10 +6,10 @@ import ( "context" "testing" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "github.com/stellar/go/services/horizon/internal/db2/history" + "github.com/stellar/go/support/db" "github.com/stellar/go/toid" "github.com/stellar/go/xdr" ) @@ -18,11 +18,12 @@ type ClaimableBalancesTransactionProcessorTestSuiteLedger struct { suite.Suite ctx context.Context processor *ClaimableBalancesTransactionProcessor - mockQ *history.MockQHistoryClaimableBalances + mockSession *db.MockSession mockTransactionBatchInsertBuilder *history.MockTransactionClaimableBalanceBatchInsertBuilder mockOperationBatchInsertBuilder *history.MockOperationClaimableBalanceBatchInsertBuilder + cbLoader *history.ClaimableBalanceLoader - sequence uint32 + lcm xdr.LedgerCloseMeta } func TestClaimableBalancesTransactionProcessorTestSuiteLedger(t *testing.T) { @@ -31,40 +32,41 @@ func TestClaimableBalancesTransactionProcessorTestSuiteLedger(t *testing.T) { func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) SetupTest() { s.ctx = context.Background() - s.mockQ = &history.MockQHistoryClaimableBalances{} s.mockTransactionBatchInsertBuilder = &history.MockTransactionClaimableBalanceBatchInsertBuilder{} s.mockOperationBatchInsertBuilder = &history.MockOperationClaimableBalanceBatchInsertBuilder{} - s.sequence = 20 + sequence := uint32(20) + s.lcm = xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(sequence), + }, + }, + }, + } + s.cbLoader = history.NewClaimableBalanceLoader() s.processor = NewClaimableBalancesTransactionProcessor( - s.mockQ, - s.sequence, + s.cbLoader, + s.mockTransactionBatchInsertBuilder, + s.mockOperationBatchInsertBuilder, ) } func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) TearDownTest() { - s.mockQ.AssertExpectations(s.T()) s.mockTransactionBatchInsertBuilder.AssertExpectations(s.T()) s.mockOperationBatchInsertBuilder.AssertExpectations(s.T()) } -func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) mockTransactionBatchAdd(transactionID, internalID int64, err error) { - s.mockTransactionBatchInsertBuilder.On("Add", s.ctx, transactionID, internalID).Return(err).Once() -} - -func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) mockOperationBatchAdd(operationID, internalID int64, err error) { - s.mockOperationBatchInsertBuilder.On("Add", s.ctx, operationID, internalID).Return(err).Once() -} - func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) TestEmptyClaimableBalances() { - // What is this expecting? Doesn't seem to assert anything meaningful... - err := s.processor.Commit(context.Background()) - s.Assert().NoError(err) + s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() + s.mockOperationBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() + + s.Assert().NoError(s.processor.Flush(s.ctx, s.mockSession)) } func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) testOperationInserts(balanceID xdr.ClaimableBalanceId, body xdr.OperationBody, change xdr.LedgerEntryChange) { // Setup the transaction - internalID := int64(1234) txn := createTransaction(true, 1) txn.Envelope.Operations()[0].Body = body txn.UnsafeMeta.V = 2 @@ -82,6 +84,20 @@ func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) testOperationInse }, }, change, + // add a duplicate change to test that the processor + // does not insert duplicate rows + { + Type: xdr.LedgerEntryChangeTypeLedgerEntryState, + State: &xdr.LedgerEntry{ + Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeClaimableBalance, + ClaimableBalance: &xdr.ClaimableBalanceEntry{ + BalanceId: balanceID, + }, + }, + }, + }, + change, }}, } @@ -101,46 +117,28 @@ func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) testOperationInse }, } } - txnID := toid.New(int32(s.sequence), int32(txn.Index), 0).ToInt64() + txnID := toid.New(int32(s.lcm.LedgerSequence()), int32(txn.Index), 0).ToInt64() opID := (&transactionOperationWrapper{ index: uint32(0), transaction: txn, operation: txn.Envelope.Operations()[0], - ledgerSequence: s.sequence, + ledgerSequence: s.lcm.LedgerSequence(), }).ID() - hexID, _ := xdr.MarshalHex(balanceID) + hexID, err := xdr.MarshalHex(balanceID) + s.Assert().NoError(err) - // Setup a q - s.mockQ.On("CreateHistoryClaimableBalances", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]string) - s.Assert().ElementsMatch( - []string{ - hexID, - }, - arg, - ) - }).Return(map[string]int64{ - hexID: internalID, - }, nil).Once() - - // Prepare to process transactions successfully - s.mockQ.On("NewTransactionClaimableBalanceBatchInsertBuilder", maxBatchSize). - Return(s.mockTransactionBatchInsertBuilder).Once() - s.mockTransactionBatchAdd(txnID, internalID, nil) - s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() + s.mockTransactionBatchInsertBuilder.On("Add", txnID, s.cbLoader.GetFuture(hexID)).Return(nil).Once() + s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() // Prepare to process operations successfully - s.mockQ.On("NewOperationClaimableBalanceBatchInsertBuilder", maxBatchSize). - Return(s.mockOperationBatchInsertBuilder).Once() - s.mockOperationBatchAdd(opID, internalID, nil) - s.mockOperationBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() + s.mockOperationBatchInsertBuilder.On("Add", opID, s.cbLoader.GetFuture(hexID)).Return(nil).Once() + s.mockOperationBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() // Process the transaction - err := s.processor.ProcessTransaction(s.ctx, txn) + err = s.processor.ProcessTransaction(s.lcm, txn) s.Assert().NoError(err) - err = s.processor.Commit(s.ctx) + err = s.processor.Flush(s.ctx, s.mockSession) s.Assert().NoError(err) } diff --git a/services/horizon/internal/ingest/processors/effects_processor.go b/services/horizon/internal/ingest/processors/effects_processor.go index 4cfd703bd2..496c4bc9b5 100644 --- a/services/horizon/internal/ingest/processors/effects_processor.go +++ b/services/horizon/internal/ingest/processors/effects_processor.go @@ -10,6 +10,7 @@ import ( "strconv" "github.com/guregu/null" + "github.com/stellar/go/amount" "github.com/stellar/go/ingest" "github.com/stellar/go/keypair" @@ -17,158 +18,61 @@ import ( "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/strkey" "github.com/stellar/go/support/contractevents" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) // EffectProcessor process effects type EffectProcessor struct { - effects []effect - effectsQ history.QEffects - sequence uint32 - network string + accountLoader *history.AccountLoader + batch history.EffectBatchInsertBuilder + network string } -func NewEffectProcessor(effectsQ history.QEffects, sequence uint32, networkPassphrase string) *EffectProcessor { +func NewEffectProcessor( + accountLoader *history.AccountLoader, + batch history.EffectBatchInsertBuilder, + network string, +) *EffectProcessor { return &EffectProcessor{ - effectsQ: effectsQ, - sequence: sequence, - network: networkPassphrase, + accountLoader: accountLoader, + batch: batch, + network: network, } } -func (p *EffectProcessor) loadAccountIDs(ctx context.Context, accountSet map[string]int64) error { - addresses := make([]string, 0, len(accountSet)) - for address := range accountSet { - addresses = append(addresses, address) - } - - addressToID, err := p.effectsQ.CreateAccounts(ctx, addresses, maxBatchSize) - if err != nil { - return errors.Wrap(err, "Could not create account ids") - } - - for _, address := range addresses { - id, ok := addressToID[address] - if !ok { - return errors.Errorf("no id found for account address %s", address) - } - - accountSet[address] = id +func (p *EffectProcessor) ProcessTransaction( + lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction, +) error { + // Failed transactions don't have operation effects + if !transaction.Result.Successful() { + return nil } - return nil -} - -func operationsEffects( - transaction ingest.LedgerTransaction, - sequence uint32, - networkPassphrase string) ([]effect, error) { - effects := []effect{} - for opi, op := range transaction.Envelope.Operations() { operation := transactionOperationWrapper{ index: uint32(opi), transaction: transaction, operation: op, - ledgerSequence: sequence, - network: networkPassphrase, + ledgerSequence: uint32(lcm.LedgerSequence()), + network: p.network, } - - p, err := operation.effects() - if err != nil { - return effects, errors.Wrapf(err, "reading operation %v effects", operation.ID()) - } - effects = append(effects, p...) - } - - return effects, nil -} - -func (p *EffectProcessor) insertDBOperationsEffects(ctx context.Context, effects []effect, accountSet map[string]int64) error { - batch := p.effectsQ.NewEffectBatchInsertBuilder(maxBatchSize) - - for _, effect := range effects { - accountID, found := accountSet[effect.address] - - if !found { - return errors.Errorf("Error finding history_account_id for address %v", effect.address) - } - - var detailsJSON []byte - detailsJSON, err := json.Marshal(effect.details) - - if err != nil { - return errors.Wrapf(err, "Error marshaling details for operation effect %v", effect.operationID) - } - - if err := batch.Add(ctx, - accountID, - effect.addressMuxed, - effect.operationID, - effect.order, - effect.effectType, - detailsJSON, - ); err != nil { - return errors.Wrap(err, "could not insert operation effect in db") + if err := operation.ingestEffects(p.accountLoader, p.batch); err != nil { + return errors.Wrapf(err, "reading operation %v effects", operation.ID()) } } - - if err := batch.Exec(ctx); err != nil { - return errors.Wrap(err, "could not flush operation effects to db") - } - return nil -} - -func (p *EffectProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) (err error) { - // Failed transactions don't have operation effects - if !transaction.Result.Successful() { - return nil - } - - var effectsForTx []effect - effectsForTx, err = operationsEffects(transaction, p.sequence, p.network) - if err != nil { - return err - } - p.effects = append(p.effects, effectsForTx...) - return nil } -func (p *EffectProcessor) Commit(ctx context.Context) (err error) { - if len(p.effects) > 0 { - accountSet := map[string]int64{} - - for _, effect := range p.effects { - accountSet[effect.address] = 0 - } - - if err = p.loadAccountIDs(ctx, accountSet); err != nil { - return err - } - - if err = p.insertDBOperationsEffects(ctx, p.effects, accountSet); err != nil { - return err - } - } - - return err -} - -type effect struct { - address string - addressMuxed null.String - operationID int64 - details map[string]interface{} - effectType history.EffectType - order uint32 +func (p *EffectProcessor) Flush(ctx context.Context, session db.SessionInterface) (err error) { + return p.batch.Exec(ctx, session) } -// Effects returns the operation effects -func (operation *transactionOperationWrapper) effects() ([]effect, error) { +// ingestEffects adds effects from the operation to the given EffectBatchInsertBuilder +func (operation *transactionOperationWrapper) ingestEffects(accountLoader *history.AccountLoader, batch history.EffectBatchInsertBuilder) error { if !operation.transaction.Result.Successful() { - return []effect{}, nil + return nil } var ( op = operation.operation @@ -177,19 +81,21 @@ func (operation *transactionOperationWrapper) effects() ([]effect, error) { changes, err := operation.transaction.GetOperationChanges(operation.index) if err != nil { - return nil, err + return err } wrapper := &effectsWrapper{ - effects: []effect{}, - operation: operation, + accountLoader: accountLoader, + batch: batch, + order: 1, + operation: operation, } switch operation.OperationType() { case xdr.OperationTypeCreateAccount: - wrapper.addAccountCreatedEffects() + err = wrapper.addAccountCreatedEffects() case xdr.OperationTypePayment: - wrapper.addPaymentEffects() + err = wrapper.addPaymentEffects() case xdr.OperationTypePathPaymentStrictReceive: err = wrapper.pathPaymentStrictReceiveEffects() case xdr.OperationTypePathPaymentStrictSend: @@ -201,15 +107,15 @@ func (operation *transactionOperationWrapper) effects() ([]effect, error) { case xdr.OperationTypeCreatePassiveSellOffer: err = wrapper.addCreatePassiveSellOfferEffect() case xdr.OperationTypeSetOptions: - wrapper.addSetOptionsEffects() + err = wrapper.addSetOptionsEffects() case xdr.OperationTypeChangeTrust: err = wrapper.addChangeTrustEffects() case xdr.OperationTypeAllowTrust: err = wrapper.addAllowTrustEffects() case xdr.OperationTypeAccountMerge: - wrapper.addAccountMergeEffects() + err = wrapper.addAccountMergeEffects() case xdr.OperationTypeInflation: - wrapper.addInflationEffects() + err = wrapper.addInflationEffects() case xdr.OperationTypeManageData: err = wrapper.addManageDataEffects() case xdr.OperationTypeBumpSequence: @@ -238,7 +144,7 @@ func (operation *transactionOperationWrapper) effects() ([]effect, error) { // meta in the transaction, which means this error is real. diagnosticEvents, innerErr := operation.transaction.GetDiagnosticEvents() if innerErr != nil { - return nil, innerErr + return innerErr } // For now, the only effects are related to the events themselves. @@ -248,10 +154,10 @@ func (operation *transactionOperationWrapper) effects() ([]effect, error) { // do not produce effects for these operations as horizon only provides // limited visibility into soroban operations default: - return nil, fmt.Errorf("unknown operation type: %s", op.Body.Type) + err = fmt.Errorf("Unknown operation type: %s", op.Body.Type) } if err != nil { - return nil, err + return err } // Effects generated for multiple operations. Keep the effect categories @@ -260,20 +166,24 @@ func (operation *transactionOperationWrapper) effects() ([]effect, error) { // Sponsorships for _, change := range changes { - if err = wrapper.addLedgerEntrySponsorshipEffects(change); err != nil { - return nil, err + if err := wrapper.addLedgerEntrySponsorshipEffects(change); err != nil { + return err + } + if err := wrapper.addSignerSponsorshipEffects(change); err != nil { + return err } - wrapper.addSignerSponsorshipEffects(change) } // Liquidity pools for _, change := range changes { - // Effects caused by ChangeTrust (creation), AllowTrust and - // SetTrustlineFlags (removal through revocation) - wrapper.addLedgerEntryLiquidityPoolEffects(change) + + // Effects caused by ChangeTrust (creation), AllowTrust and SetTrustlineFlags (removal through revocation) + if err := wrapper.addLedgerEntryLiquidityPoolEffects(change); err != nil { + return err + } } - return wrapper.effects, nil + return nil } func filterEvents(diagnosticEvents []xdr.DiagnosticEvent) []xdr.ContractEvent { @@ -288,32 +198,43 @@ func filterEvents(diagnosticEvents []xdr.DiagnosticEvent) []xdr.ContractEvent { } type effectsWrapper struct { - effects []effect - operation *transactionOperationWrapper + accountLoader *history.AccountLoader + batch history.EffectBatchInsertBuilder + order uint32 + operation *transactionOperationWrapper } -func (e *effectsWrapper) add(address string, addressMuxed null.String, effectType history.EffectType, details map[string]interface{}) { - e.effects = append(e.effects, effect{ - address: address, - addressMuxed: addressMuxed, - operationID: e.operation.ID(), - effectType: effectType, - order: uint32(len(e.effects) + 1), - details: details, - }) +func (e *effectsWrapper) add(address string, addressMuxed null.String, effectType history.EffectType, details map[string]interface{}) error { + detailsJSON, err := json.Marshal(details) + if err != nil { + return errors.Wrapf(err, "Error marshaling details for operation effect %v", e.operation.ID()) + } + + if err := e.batch.Add( + e.accountLoader.GetFuture(address), + addressMuxed, + e.operation.ID(), + e.order, + effectType, + detailsJSON, + ); err != nil { + return errors.Wrap(err, "could not insert operation effect in db") + } + e.order++ + return nil } -func (e *effectsWrapper) addUnmuxed(address *xdr.AccountId, effectType history.EffectType, details map[string]interface{}) { - e.add(address.Address(), null.String{}, effectType, details) +func (e *effectsWrapper) addUnmuxed(address *xdr.AccountId, effectType history.EffectType, details map[string]interface{}) error { + return e.add(address.Address(), null.String{}, effectType, details) } -func (e *effectsWrapper) addMuxed(address *xdr.MuxedAccount, effectType history.EffectType, details map[string]interface{}) { +func (e *effectsWrapper) addMuxed(address *xdr.MuxedAccount, effectType history.EffectType, details map[string]interface{}) error { var addressMuxed null.String if address.Type == xdr.CryptoKeyTypeKeyTypeMuxedEd25519 { addressMuxed = null.StringFrom(address.Address()) } accID := address.ToAccountId() - e.add(accID.Address(), addressMuxed, effectType, details) + return e.add(accID.Address(), addressMuxed, effectType, details) } var sponsoringEffectsTable = map[xdr.LedgerEntryType]struct { @@ -344,9 +265,9 @@ var sponsoringEffectsTable = map[xdr.LedgerEntryType]struct { // entries because we don't generate creation effects for them. } -func (e *effectsWrapper) addSignerSponsorshipEffects(change ingest.Change) { +func (e *effectsWrapper) addSignerSponsorshipEffects(change ingest.Change) error { if change.Type != xdr.LedgerEntryTypeAccount { - return + return nil } preSigners := map[string]xdr.AccountId{} @@ -384,12 +305,16 @@ func (e *effectsWrapper) addSignerSponsorshipEffects(change ingest.Change) { details["sponsor"] = post.Address() details["signer"] = signer srcAccount := change.Post.Data.MustAccount().AccountId - e.addUnmuxed(&srcAccount, history.EffectSignerSponsorshipCreated, details) + if err := e.addUnmuxed(&srcAccount, history.EffectSignerSponsorshipCreated, details); err != nil { + return err + } case !foundPost && foundPre: details["former_sponsor"] = pre.Address() details["signer"] = signer srcAccount := change.Pre.Data.MustAccount().AccountId - e.addUnmuxed(&srcAccount, history.EffectSignerSponsorshipRemoved, details) + if err := e.addUnmuxed(&srcAccount, history.EffectSignerSponsorshipRemoved, details); err != nil { + return err + } case foundPre && foundPost: formerSponsor := pre.Address() newSponsor := post.Address() @@ -401,9 +326,12 @@ func (e *effectsWrapper) addSignerSponsorshipEffects(change ingest.Change) { details["new_sponsor"] = newSponsor details["signer"] = signer srcAccount := change.Post.Data.MustAccount().AccountId - e.addUnmuxed(&srcAccount, history.EffectSignerSponsorshipUpdated, details) + if err := e.addUnmuxed(&srcAccount, history.EffectSignerSponsorshipUpdated, details); err != nil { + return err + } } } + return nil } func (e *effectsWrapper) addLedgerEntrySponsorshipEffects(change ingest.Change) error { @@ -481,9 +409,13 @@ func (e *effectsWrapper) addLedgerEntrySponsorshipEffects(change ingest.Change) } if accountID != nil { - e.addUnmuxed(accountID, effectType, details) + if err := e.addUnmuxed(accountID, effectType, details); err != nil { + return err + } } else { - e.addMuxed(muxedAccount, effectType, details) + if err := e.addMuxed(muxedAccount, effectType, details); err != nil { + return err + } } return nil @@ -511,55 +443,64 @@ func (e *effectsWrapper) addLedgerEntryLiquidityPoolEffects(change ingest.Change default: return nil } - e.addMuxed( + return e.addMuxed( e.operation.SourceAccount(), effectType, details, ) - - return nil } -func (e *effectsWrapper) addAccountCreatedEffects() { +func (e *effectsWrapper) addAccountCreatedEffects() error { op := e.operation.operation.Body.MustCreateAccountOp() - e.addUnmuxed( + if err := e.addUnmuxed( &op.Destination, history.EffectAccountCreated, map[string]interface{}{ "starting_balance": amount.String(op.StartingBalance), }, - ) - e.addMuxed( + ); err != nil { + return err + } + if err := e.addMuxed( e.operation.SourceAccount(), history.EffectAccountDebited, map[string]interface{}{ "asset_type": "native", "amount": amount.String(op.StartingBalance), }, - ) - e.addUnmuxed( + ); err != nil { + return err + } + if err := e.addUnmuxed( &op.Destination, history.EffectSignerCreated, map[string]interface{}{ "public_key": op.Destination.Address(), "weight": keypair.DefaultSignerWeight, }, - ) + ); err != nil { + return err + } + return nil } -func (e *effectsWrapper) addPaymentEffects() { +func (e *effectsWrapper) addPaymentEffects() error { op := e.operation.operation.Body.MustPaymentOp() details := map[string]interface{}{"amount": amount.String(op.Amount)} - addAssetDetails(details, op.Asset, "") + if err := addAssetDetails(details, op.Asset, ""); err != nil { + return err + } - e.addMuxed( + if err := e.addMuxed( &op.Destination, history.EffectAccountCredited, details, - ) - e.addMuxed( + ); err != nil { + return err + } + return e.addMuxed( e.operation.SourceAccount(), history.EffectAccountDebited, details, @@ -572,23 +513,31 @@ func (e *effectsWrapper) pathPaymentStrictReceiveEffects() error { source := e.operation.SourceAccount() details := map[string]interface{}{"amount": amount.String(op.DestAmount)} - addAssetDetails(details, op.DestAsset, "") + if err := addAssetDetails(details, op.DestAsset, ""); err != nil { + return err + } - e.addMuxed( + if err := e.addMuxed( &op.Destination, history.EffectAccountCredited, details, - ) + ); err != nil { + return err + } result := e.operation.OperationResult().MustPathPaymentStrictReceiveResult() details = map[string]interface{}{"amount": amount.String(result.SendAmount())} - addAssetDetails(details, op.SendAsset, "") + if err := addAssetDetails(details, op.SendAsset, ""); err != nil { + return err + } - e.addMuxed( + if err := e.addMuxed( source, history.EffectAccountDebited, details, - ) + ); err != nil { + return err + } return e.addIngestTradeEffects(*source, resultSuccess.Offers) } @@ -600,12 +549,20 @@ func (e *effectsWrapper) addPathPaymentStrictSendEffects() error { result := e.operation.OperationResult().MustPathPaymentStrictSendResult() details := map[string]interface{}{"amount": amount.String(result.DestAmount())} - addAssetDetails(details, op.DestAsset, "") - e.addMuxed(&op.Destination, history.EffectAccountCredited, details) + if err := addAssetDetails(details, op.DestAsset, ""); err != nil { + return err + } + if err := e.addMuxed(&op.Destination, history.EffectAccountCredited, details); err != nil { + return err + } details = map[string]interface{}{"amount": amount.String(op.SendAmount)} - addAssetDetails(details, op.SendAsset, "") - e.addMuxed(source, history.EffectAccountDebited, details) + if err := addAssetDetails(details, op.SendAsset, ""); err != nil { + return err + } + if err := e.addMuxed(source, history.EffectAccountDebited, details); err != nil { + return err + } return e.addIngestTradeEffects(*source, resultSuccess.Offers) } @@ -644,11 +601,13 @@ func (e *effectsWrapper) addSetOptionsEffects() error { op := e.operation.operation.Body.MustSetOptionsOp() if op.HomeDomain != nil { - e.addMuxed(source, history.EffectAccountHomeDomainUpdated, + if err := e.addMuxed(source, history.EffectAccountHomeDomainUpdated, map[string]interface{}{ "home_domain": string(*op.HomeDomain), }, - ) + ); err != nil { + return err + } } thresholdDetails := map[string]interface{}{} @@ -666,7 +625,9 @@ func (e *effectsWrapper) addSetOptionsEffects() error { } if len(thresholdDetails) > 0 { - e.addMuxed(source, history.EffectAccountThresholdsUpdated, thresholdDetails) + if err := e.addMuxed(source, history.EffectAccountThresholdsUpdated, thresholdDetails); err != nil { + return err + } } flagDetails := map[string]interface{}{} @@ -678,15 +639,19 @@ func (e *effectsWrapper) addSetOptionsEffects() error { } if len(flagDetails) > 0 { - e.addMuxed(source, history.EffectAccountFlagsUpdated, flagDetails) + if err := e.addMuxed(source, history.EffectAccountFlagsUpdated, flagDetails); err != nil { + return err + } } if op.InflationDest != nil { - e.addMuxed(source, history.EffectAccountInflationDestinationUpdated, + if err := e.addMuxed(source, history.EffectAccountInflationDestinationUpdated, map[string]interface{}{ "inflation_destination": op.InflationDest.Address(), }, - ) + ); err != nil { + return err + } } changes, err := e.operation.transaction.GetOperationChanges(e.operation.index) if err != nil { @@ -709,7 +674,7 @@ func (e *effectsWrapper) addSetOptionsEffects() error { continue } - beforeSortedSigners := []string{} + var beforeSortedSigners []string for signer := range before { beforeSortedSigners = append(beforeSortedSigners, signer) } @@ -718,21 +683,25 @@ func (e *effectsWrapper) addSetOptionsEffects() error { for _, addy := range beforeSortedSigners { weight, ok := after[addy] if !ok { - e.addMuxed(source, history.EffectSignerRemoved, map[string]interface{}{ + if err := e.addMuxed(source, history.EffectSignerRemoved, map[string]interface{}{ "public_key": addy, - }) + }); err != nil { + return err + } continue } if weight != before[addy] { - e.addMuxed(source, history.EffectSignerUpdated, map[string]interface{}{ + if err := e.addMuxed(source, history.EffectSignerUpdated, map[string]interface{}{ "public_key": addy, "weight": weight, - }) + }); err != nil { + return err + } } } - afterSortedSigners := []string{} + var afterSortedSigners []string for signer := range after { afterSortedSigners = append(afterSortedSigners, signer) } @@ -747,10 +716,12 @@ func (e *effectsWrapper) addSetOptionsEffects() error { continue } - e.addMuxed(source, history.EffectSignerCreated, map[string]interface{}{ + if err := e.addMuxed(source, history.EffectSignerCreated, map[string]interface{}{ "public_key": addy, "weight": weight, - }) + }); err != nil { + return err + } } } return nil @@ -806,10 +777,14 @@ func (e *effectsWrapper) addChangeTrustEffects() error { return err } } else { - addAssetDetails(details, op.Line.ToAsset(), "") + if err := addAssetDetails(details, op.Line.ToAsset(), ""); err != nil { + return err + } } - e.addMuxed(source, effect, details) + if err := e.addMuxed(source, effect, details); err != nil { + return err + } break } @@ -823,33 +798,47 @@ func (e *effectsWrapper) addAllowTrustEffects() error { details := map[string]interface{}{ "trustor": op.Trustor.Address(), } - addAssetDetails(details, asset, "") + if err := addAssetDetails(details, asset, ""); err != nil { + return err + } switch { case xdr.TrustLineFlags(op.Authorize).IsAuthorized(): - e.addMuxed(source, history.EffectTrustlineAuthorized, details) + if err := e.addMuxed(source, history.EffectTrustlineAuthorized, details); err != nil { + return err + } // Forward compatibility setFlags := xdr.Uint32(xdr.TrustLineFlagsAuthorizedFlag) - e.addTrustLineFlagsEffect(source, &op.Trustor, asset, &setFlags, nil) + if err := e.addTrustLineFlagsEffect(source, &op.Trustor, asset, &setFlags, nil); err != nil { + return err + } case xdr.TrustLineFlags(op.Authorize).IsAuthorizedToMaintainLiabilitiesFlag(): - e.addMuxed( + if err := e.addMuxed( source, history.EffectTrustlineAuthorizedToMaintainLiabilities, details, - ) + ); err != nil { + return err + } // Forward compatibility setFlags := xdr.Uint32(xdr.TrustLineFlagsAuthorizedToMaintainLiabilitiesFlag) - e.addTrustLineFlagsEffect(source, &op.Trustor, asset, &setFlags, nil) + if err := e.addTrustLineFlagsEffect(source, &op.Trustor, asset, &setFlags, nil); err != nil { + return err + } default: - e.addMuxed(source, history.EffectTrustlineDeauthorized, details) + if err := e.addMuxed(source, history.EffectTrustlineDeauthorized, details); err != nil { + return err + } // Forward compatibility, show both as cleared clearFlags := xdr.Uint32(xdr.TrustLineFlagsAuthorizedFlag | xdr.TrustLineFlagsAuthorizedToMaintainLiabilitiesFlag) - e.addTrustLineFlagsEffect(source, &op.Trustor, asset, nil, &clearFlags) + if err := e.addTrustLineFlagsEffect(source, &op.Trustor, asset, nil, &clearFlags); err != nil { + return err + } } return e.addLiquidityPoolRevokedEffect() } -func (e *effectsWrapper) addAccountMergeEffects() { +func (e *effectsWrapper) addAccountMergeEffects() error { source := e.operation.SourceAccount() dest := e.operation.operation.Body.MustDestination() @@ -859,21 +848,31 @@ func (e *effectsWrapper) addAccountMergeEffects() { "asset_type": "native", } - e.addMuxed(source, history.EffectAccountDebited, details) - e.addMuxed(&dest, history.EffectAccountCredited, details) - e.addMuxed(source, history.EffectAccountRemoved, map[string]interface{}{}) + if err := e.addMuxed(source, history.EffectAccountDebited, details); err != nil { + return err + } + if err := e.addMuxed(&dest, history.EffectAccountCredited, details); err != nil { + return err + } + if err := e.addMuxed(source, history.EffectAccountRemoved, map[string]interface{}{}); err != nil { + return err + } + return nil } -func (e *effectsWrapper) addInflationEffects() { +func (e *effectsWrapper) addInflationEffects() error { payouts := e.operation.OperationResult().MustInflationResult().MustPayouts() for _, payout := range payouts { - e.addUnmuxed(&payout.Destination, history.EffectAccountCredited, + if err := e.addUnmuxed(&payout.Destination, history.EffectAccountCredited, map[string]interface{}{ "amount": amount.String(payout.Amount), "asset_type": "native", }, - ) + ); err != nil { + return err + } } + return nil } func (e *effectsWrapper) addManageDataEffects() error { @@ -913,8 +912,7 @@ func (e *effectsWrapper) addManageDataEffects() error { break } - e.addMuxed(source, effect, details) - return nil + return e.addMuxed(source, effect, details) } func (e *effectsWrapper) addBumpSequenceEffects() error { @@ -937,7 +935,9 @@ func (e *effectsWrapper) addBumpSequenceEffects() error { if beforeAccount.SeqNum != afterAccount.SeqNum { details := map[string]interface{}{"new_seq": afterAccount.SeqNum} - e.addMuxed(source, history.EffectSequenceBumped, details) + if err := e.addMuxed(source, history.EffectSequenceBumped, details); err != nil { + return err + } } break } @@ -960,7 +960,9 @@ func (e *effectsWrapper) addCreateClaimableBalanceEffects(changes []ingest.Chang continue } cb = change.Post.Data.ClaimableBalance - e.addClaimableBalanceEntryCreatedEffects(source, cb) + if err := e.addClaimableBalanceEntryCreatedEffects(source, cb); err != nil { + return err + } break } if cb == nil { @@ -970,14 +972,14 @@ func (e *effectsWrapper) addCreateClaimableBalanceEffects(changes []ingest.Chang details := map[string]interface{}{ "amount": amount.String(cb.Amount), } - addAssetDetails(details, cb.Asset, "") - e.addMuxed( + if err := addAssetDetails(details, cb.Asset, ""); err != nil { + return err + } + return e.addMuxed( source, history.EffectAccountDebited, details, ) - - return nil } func (e *effectsWrapper) addClaimableBalanceEntryCreatedEffects(source *xdr.MuxedAccount, cb *xdr.ClaimableBalanceEntry) error { @@ -991,11 +993,13 @@ func (e *effectsWrapper) addClaimableBalanceEntryCreatedEffects(source *xdr.Muxe "asset": cb.Asset.StringCanonical(), } setClaimableBalanceFlagDetails(details, cb.Flags()) - e.addMuxed( + if err := e.addMuxed( source, history.EffectClaimableBalanceCreated, details, - ) + ); err != nil { + return err + } // EffectClaimableBalanceClaimantCreated can be generated by // `create_claimable_balance` operation but also by `liquidity_pool_withdraw` // operation causing a revocation. @@ -1011,7 +1015,7 @@ func (e *effectsWrapper) addClaimableBalanceEntryCreatedEffects(source *xdr.Muxe } for _, c := range claimants { cv0 := c.MustV0() - e.addUnmuxed( + if err := e.addUnmuxed( &cv0.Destination, history.EffectClaimableBalanceClaimantCreated, map[string]interface{}{ @@ -1020,9 +1024,11 @@ func (e *effectsWrapper) addClaimableBalanceEntryCreatedEffects(source *xdr.Muxe "predicate": cv0.Predicate, "asset": cb.Asset.StringCanonical(), }, - ) + ); err != nil { + return err + } } - return err + return nil } func (e *effectsWrapper) addClaimClaimableBalanceEffects(changes []ingest.Change) error { @@ -1065,23 +1071,25 @@ func (e *effectsWrapper) addClaimClaimableBalanceEffects(changes []ingest.Change } setClaimableBalanceFlagDetails(details, cBalance.Flags()) source := e.operation.SourceAccount() - e.addMuxed( + if err := e.addMuxed( source, history.EffectClaimableBalanceClaimed, details, - ) + ); err != nil { + return err + } details = map[string]interface{}{ "amount": amount.String(cBalance.Amount), } - addAssetDetails(details, cBalance.Asset, "") - e.addMuxed( + if err := addAssetDetails(details, cBalance.Asset, ""); err != nil { + return err + } + return e.addMuxed( source, history.EffectAccountCredited, details, ) - - return nil } func (e *effectsWrapper) addIngestTradeEffects(buyer xdr.MuxedAccount, claims []xdr.ClaimAtom) error { @@ -1095,23 +1103,30 @@ func (e *effectsWrapper) addIngestTradeEffects(buyer xdr.MuxedAccount, claims [] return err } default: - e.addClaimTradeEffects(buyer, claim) + if err := e.addClaimTradeEffects(buyer, claim); err != nil { + return err + } } } return nil } -func (e *effectsWrapper) addClaimTradeEffects(buyer xdr.MuxedAccount, claim xdr.ClaimAtom) { +func (e *effectsWrapper) addClaimTradeEffects(buyer xdr.MuxedAccount, claim xdr.ClaimAtom) error { seller := claim.SellerId() - bd, sd := tradeDetails(buyer, seller, claim) + bd, sd, err := tradeDetails(buyer, seller, claim) + if err != nil { + return err + } - e.addMuxed( + if err := e.addMuxed( &buyer, history.EffectTrade, bd, - ) + ); err != nil { + return err + } - e.addUnmuxed( + return e.addUnmuxed( &seller, history.EffectTrade, sd, @@ -1134,8 +1149,7 @@ func (e *effectsWrapper) addClaimLiquidityPoolTradeEffect(claim xdr.ClaimAtom) e "amount": amount.String(claim.LiquidityPool.AmountBought), }, } - e.addMuxed(e.operation.SourceAccount(), history.EffectLiquidityPoolTrade, details) - return nil + return e.addMuxed(e.operation.SourceAccount(), history.EffectLiquidityPoolTrade, details) } func (e *effectsWrapper) addClawbackEffects() error { @@ -1144,20 +1158,26 @@ func (e *effectsWrapper) addClawbackEffects() error { "amount": amount.String(op.Amount), } source := e.operation.SourceAccount() - addAssetDetails(details, op.Asset, "") + if err := addAssetDetails(details, op.Asset, ""); err != nil { + return err + } // The funds will be burned, but even with that, we generated an account credited effect - e.addMuxed( + if err := e.addMuxed( source, history.EffectAccountCredited, details, - ) + ); err != nil { + return err + } - e.addMuxed( + if err := e.addMuxed( &op.From, history.EffectAccountDebited, details, - ) + ); err != nil { + return err + } return nil } @@ -1172,23 +1192,29 @@ func (e *effectsWrapper) addClawbackClaimableBalanceEffects(changes []ingest.Cha "balance_id": balanceId, } source := e.operation.SourceAccount() - e.addMuxed( + if err := e.addMuxed( source, history.EffectClaimableBalanceClawedBack, details, - ) + ); err != nil { + return err + } // Generate the account credited effect (although the funds will be burned) for the asset issuer for _, c := range changes { if c.Type == xdr.LedgerEntryTypeClaimableBalance && c.Post == nil && c.Pre != nil { cb := c.Pre.Data.ClaimableBalance details = map[string]interface{}{"amount": amount.String(cb.Amount)} - addAssetDetails(details, cb.Asset, "") - e.addMuxed( + if err := addAssetDetails(details, cb.Asset, ""); err != nil { + return err + } + if err := e.addMuxed( source, history.EffectAccountCredited, details, - ) + ); err != nil { + return err + } break } } @@ -1199,7 +1225,9 @@ func (e *effectsWrapper) addClawbackClaimableBalanceEffects(changes []ingest.Cha func (e *effectsWrapper) addSetTrustLineFlagsEffects() error { source := e.operation.SourceAccount() op := e.operation.operation.Body.MustSetTrustLineFlagsOp() - e.addTrustLineFlagsEffect(source, &op.Trustor, op.Asset, &op.SetFlags, &op.ClearFlags) + if err := e.addTrustLineFlagsEffect(source, &op.Trustor, op.Asset, &op.SetFlags, &op.ClearFlags); err != nil { + return err + } return e.addLiquidityPoolRevokedEffect() } @@ -1208,11 +1236,13 @@ func (e *effectsWrapper) addTrustLineFlagsEffect( trustor *xdr.AccountId, asset xdr.Asset, setFlags *xdr.Uint32, - clearFlags *xdr.Uint32) { + clearFlags *xdr.Uint32) error { details := map[string]interface{}{ "trustor": trustor.Address(), } - addAssetDetails(details, asset, "") + if err := addAssetDetails(details, asset, ""); err != nil { + return err + } var flagDetailsAdded bool if setFlags != nil { @@ -1225,8 +1255,11 @@ func (e *effectsWrapper) addTrustLineFlagsEffect( } if flagDetailsAdded { - e.addMuxed(account, history.EffectTrustlineFlagsUpdated, details) + if err := e.addMuxed(account, history.EffectTrustlineFlagsUpdated, details); err != nil { + return err + } } + return nil } func setTrustLineFlagDetails(flagDetails map[string]interface{}, flags xdr.TrustLineFlags, setValue bool) { @@ -1312,8 +1345,8 @@ func (e *effectsWrapper) addLiquidityPoolRevokedEffect() error { "reserves_revoked": reservesRevoked, "shares_revoked": amount.String(-delta.TotalPoolShares), } - e.addMuxed(source, history.EffectLiquidityPoolRevoked, details) - return nil + + return e.addMuxed(source, history.EffectLiquidityPoolRevoked, details) } func setAuthFlagDetails(flagDetails map[string]interface{}, flags xdr.AccountFlags, setValue bool) { @@ -1331,15 +1364,19 @@ func setAuthFlagDetails(flagDetails map[string]interface{}, flags xdr.AccountFla } } -func tradeDetails(buyer xdr.MuxedAccount, seller xdr.AccountId, claim xdr.ClaimAtom) (bd map[string]interface{}, sd map[string]interface{}) { +func tradeDetails(buyer xdr.MuxedAccount, seller xdr.AccountId, claim xdr.ClaimAtom) (bd map[string]interface{}, sd map[string]interface{}, err error) { bd = map[string]interface{}{ "offer_id": claim.OfferId(), "seller": seller.Address(), "bought_amount": amount.String(claim.AmountSold()), "sold_amount": amount.String(claim.AmountBought()), } - addAssetDetails(bd, claim.AssetSold(), "bought_") - addAssetDetails(bd, claim.AssetBought(), "sold_") + if err = addAssetDetails(bd, claim.AssetSold(), "bought_"); err != nil { + return + } + if err = addAssetDetails(bd, claim.AssetBought(), "sold_"); err != nil { + return + } sd = map[string]interface{}{ "offer_id": claim.OfferId(), @@ -1347,9 +1384,12 @@ func tradeDetails(buyer xdr.MuxedAccount, seller xdr.AccountId, claim xdr.ClaimA "sold_amount": amount.String(claim.AmountSold()), } addAccountAndMuxedAccountDetails(sd, buyer, "seller") - addAssetDetails(sd, claim.AssetBought(), "bought_") - addAssetDetails(sd, claim.AssetSold(), "sold_") - + if err = addAssetDetails(sd, claim.AssetBought(), "bought_"); err != nil { + return + } + if err = addAssetDetails(sd, claim.AssetSold(), "sold_"); err != nil { + return + } return } @@ -1393,8 +1433,8 @@ func (e *effectsWrapper) addLiquidityPoolDepositEffect() error { }, "shares_received": amount.String(delta.TotalPoolShares), } - e.addMuxed(e.operation.SourceAccount(), history.EffectLiquidityPoolDeposited, details) - return nil + + return e.addMuxed(e.operation.SourceAccount(), history.EffectLiquidityPoolDeposited, details) } func (e *effectsWrapper) addLiquidityPoolWithdrawEffect() error { @@ -1417,8 +1457,8 @@ func (e *effectsWrapper) addLiquidityPoolWithdrawEffect() error { }, "shares_redeemed": amount.String(-delta.TotalPoolShares), } - e.addMuxed(e.operation.SourceAccount(), history.EffectLiquidityPoolWithdrew, details) - return nil + + return e.addMuxed(e.operation.SourceAccount(), history.EffectLiquidityPoolWithdrew, details) } // addInvokeHostFunctionEffects iterates through the events and generates @@ -1437,7 +1477,9 @@ func (e *effectsWrapper) addInvokeHostFunctionEffects(events []contractevents.Ev } details := make(map[string]interface{}, 4) - addAssetDetails(details, evt.GetAsset(), "") + if err := addAssetDetails(details, evt.GetAsset(), ""); err != nil { + return errors.Wrapf(err, "invokeHostFunction asset details had an error") + } // // Note: We ignore effects that involve contracts (until the day we have @@ -1456,24 +1498,28 @@ func (e *effectsWrapper) addInvokeHostFunctionEffects(events []contractevents.Ev } if strkey.IsValidEd25519PublicKey(transferEvent.From) { - e.add( + if err := e.add( transferEvent.From, null.String{}, history.EffectAccountDebited, details, - ) + ); err != nil { + return errors.Wrapf(err, "invokeHostFunction asset details from contract xfr-from had an error") + } } else { details["contract"] = transferEvent.From e.addMuxed(source, history.EffectContractDebited, details) } if strkey.IsValidEd25519PublicKey(transferEvent.To) { - e.add( + if err := e.add( transferEvent.To, null.String{}, history.EffectAccountCredited, toDetails, - ) + ); err != nil { + return errors.Wrapf(err, "invokeHostFunction asset details from contract xfr-to had an error") + } } else { toDetails["contract"] = transferEvent.To e.addMuxed(source, history.EffectContractCredited, toDetails) @@ -1485,12 +1531,14 @@ func (e *effectsWrapper) addInvokeHostFunctionEffects(events []contractevents.Ev mintEvent := evt.(*contractevents.MintEvent) details["amount"] = amount.String128(mintEvent.Amount) if strkey.IsValidEd25519PublicKey(mintEvent.To) { - e.add( + if err := e.add( mintEvent.To, null.String{}, history.EffectAccountCredited, details, - ) + ); err != nil { + return errors.Wrapf(err, "invokeHostFunction asset details from contract mint had an error") + } } else { details["contract"] = mintEvent.To e.addMuxed(source, history.EffectContractCredited, details) @@ -1502,12 +1550,14 @@ func (e *effectsWrapper) addInvokeHostFunctionEffects(events []contractevents.Ev cbEvent := evt.(*contractevents.ClawbackEvent) details["amount"] = amount.String128(cbEvent.Amount) if strkey.IsValidEd25519PublicKey(cbEvent.From) { - e.add( + if err := e.add( cbEvent.From, null.String{}, history.EffectAccountDebited, details, - ) + ); err != nil { + return errors.Wrapf(err, "invokeHostFunction asset details from contract clawback had an error") + } } else { details["contract"] = cbEvent.From e.addMuxed(source, history.EffectContractDebited, details) @@ -1517,12 +1567,14 @@ func (e *effectsWrapper) addInvokeHostFunctionEffects(events []contractevents.Ev burnEvent := evt.(*contractevents.BurnEvent) details["amount"] = amount.String128(burnEvent.Amount) if strkey.IsValidEd25519PublicKey(burnEvent.From) { - e.add( + if err := e.add( burnEvent.From, null.String{}, history.EffectAccountDebited, details, - ) + ); err != nil { + return errors.Wrapf(err, "invokeHostFunction asset details from contract burn had an error") + } } else { details["contract"] = burnEvent.From e.addMuxed(source, history.EffectContractDebited, details) diff --git a/services/horizon/internal/ingest/processors/effects_processor_test.go b/services/horizon/internal/ingest/processors/effects_processor_test.go index 9199625cc2..0243768fde 100644 --- a/services/horizon/internal/ingest/processors/effects_processor_test.go +++ b/services/horizon/internal/ingest/processors/effects_processor_test.go @@ -6,22 +6,25 @@ import ( "context" "crypto/rand" "encoding/hex" - "math/big" - "strings" + "encoding/json" "testing" "github.com/guregu/null" + "math/big" + "strings" + "github.com/stellar/go/keypair" "github.com/stellar/go/protocols/horizon/base" "github.com/stellar/go/strkey" + "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" . "github.com/stellar/go/services/horizon/internal/test/transactions" "github.com/stellar/go/support/contractevents" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/toid" "github.com/stellar/go/xdr" @@ -35,9 +38,11 @@ type EffectsProcessorTestSuiteLedger struct { suite.Suite ctx context.Context processor *EffectProcessor - mockQ *history.MockQEffects + mockSession *db.MockSession + accountLoader *history.AccountLoader mockBatchInsertBuilder *history.MockEffectBatchInsertBuilder + lcm xdr.LedgerCloseMeta firstTx ingest.LedgerTransaction secondTx ingest.LedgerTransaction thirdTx ingest.LedgerTransaction @@ -46,7 +51,6 @@ type EffectsProcessorTestSuiteLedger struct { secondTxID int64 thirdTxID int64 failedTxID int64 - sequence uint32 addresses []string addressToID map[string]int64 txs []ingest.LedgerTransaction @@ -58,11 +62,18 @@ func TestEffectsProcessorTestSuiteLedger(t *testing.T) { func (s *EffectsProcessorTestSuiteLedger) SetupTest() { s.ctx = context.Background() - s.mockQ = &history.MockQEffects{} + s.accountLoader = history.NewAccountLoader() s.mockBatchInsertBuilder = &history.MockEffectBatchInsertBuilder{} - s.sequence = uint32(20) - + s.lcm = xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(20), + }, + }, + }, + } s.addresses = []string{ "GANFZDRBCNTUXIODCJEYMACPMCSZEVE4WZGZ3CZDZ3P2SXK4KH75IK6Y", "GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H", @@ -80,7 +91,7 @@ func (s *EffectsProcessorTestSuiteLedger) SetupTest() { Hash: "829d53f2dceebe10af8007564b0aefde819b95734ad431df84270651e7ed8a90", }, ) - s.firstTxID = toid.New(int32(s.sequence), 1, 0).ToInt64() + s.firstTxID = toid.New(int32(s.lcm.LedgerSequence()), 1, 0).ToInt64() s.secondTx = BuildLedgerTransaction( s.Suite.T(), @@ -94,7 +105,7 @@ func (s *EffectsProcessorTestSuiteLedger) SetupTest() { }, ) - s.secondTxID = toid.New(int32(s.sequence), 2, 0).ToInt64() + s.secondTxID = toid.New(int32(s.lcm.LedgerSequence()), 2, 0).ToInt64() s.thirdTx = BuildLedgerTransaction( s.Suite.T(), @@ -107,7 +118,7 @@ func (s *EffectsProcessorTestSuiteLedger) SetupTest() { Hash: "2a805712c6d10f9e74bb0ccf54ae92a2b4b1e586451fe8133a2433816f6b567c", }, ) - s.thirdTxID = toid.New(int32(s.sequence), 3, 0).ToInt64() + s.thirdTxID = toid.New(int32(s.lcm.LedgerSequence()), 3, 0).ToInt64() s.failedTx = BuildLedgerTransaction( s.Suite.T(), @@ -120,7 +131,7 @@ func (s *EffectsProcessorTestSuiteLedger) SetupTest() { Hash: "24206737a02f7f855c46e367418e38c223f897792c76bbfb948e1b0dbd695f8b", }, ) - s.failedTxID = toid.New(int32(s.sequence), 4, 0).ToInt64() + s.failedTxID = toid.New(int32(s.lcm.LedgerSequence()), 4, 0).ToInt64() s.addressToID = map[string]int64{ s.addresses[0]: 2, @@ -129,8 +140,8 @@ func (s *EffectsProcessorTestSuiteLedger) SetupTest() { } s.processor = NewEffectProcessor( - s.mockQ, - 20, + s.accountLoader, + s.mockBatchInsertBuilder, networkPassphrase, ) @@ -142,46 +153,42 @@ func (s *EffectsProcessorTestSuiteLedger) SetupTest() { } func (s *EffectsProcessorTestSuiteLedger) TearDownTest() { - s.mockQ.AssertExpectations(s.T()) + s.mockBatchInsertBuilder.AssertExpectations(s.T()) } func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulEffectBatchAdds() { s.mockBatchInsertBuilder.On( "Add", - s.ctx, - s.addressToID[s.addresses[2]], + s.accountLoader.GetFuture(s.addresses[2]), null.String{}, - toid.New(int32(s.sequence), 1, 1).ToInt64(), + toid.New(int32(s.lcm.LedgerSequence()), 1, 1).ToInt64(), uint32(1), history.EffectSequenceBumped, []byte("{\"new_seq\":300000000000}"), ).Return(nil).Once() s.mockBatchInsertBuilder.On( "Add", - s.ctx, - s.addressToID[s.addresses[2]], + s.accountLoader.GetFuture(s.addresses[2]), null.String{}, - toid.New(int32(s.sequence), 2, 1).ToInt64(), + toid.New(int32(s.lcm.LedgerSequence()), 2, 1).ToInt64(), uint32(1), history.EffectAccountCreated, []byte("{\"starting_balance\":\"1000.0000000\"}"), ).Return(nil).Once() s.mockBatchInsertBuilder.On( "Add", - s.ctx, - s.addressToID[s.addresses[1]], + s.accountLoader.GetFuture(s.addresses[1]), null.String{}, - toid.New(int32(s.sequence), 2, 1).ToInt64(), + toid.New(int32(s.lcm.LedgerSequence()), 2, 1).ToInt64(), uint32(2), history.EffectAccountDebited, []byte("{\"amount\":\"1000.0000000\",\"asset_type\":\"native\"}"), ).Return(nil).Once() s.mockBatchInsertBuilder.On( "Add", - s.ctx, - s.addressToID[s.addresses[2]], + s.accountLoader.GetFuture(s.addresses[2]), null.String{}, - toid.New(int32(s.sequence), 2, 1).ToInt64(), + toid.New(int32(s.lcm.LedgerSequence()), 2, 1).ToInt64(), uint32(3), history.EffectSignerCreated, []byte("{\"public_key\":\"GCQZP3IU7XU6EJ63JZXKCQOYT2RNXN3HB5CNHENNUEUHSMA4VUJJJSEN\",\"weight\":1}"), @@ -189,10 +196,9 @@ func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulEffectBatchAdds() { s.mockBatchInsertBuilder.On( "Add", - s.ctx, - s.addressToID[s.addresses[0]], + s.accountLoader.GetFuture(s.addresses[0]), null.String{}, - toid.New(int32(s.sequence), 3, 1).ToInt64(), + toid.New(int32(s.lcm.LedgerSequence()), 3, 1).ToInt64(), uint32(1), history.EffectAccountCredited, []byte("{\"amount\":\"10.0000000\",\"asset_type\":\"native\"}"), @@ -200,82 +206,45 @@ func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulEffectBatchAdds() { s.mockBatchInsertBuilder.On( "Add", - s.ctx, - s.addressToID[s.addresses[0]], + s.accountLoader.GetFuture(s.addresses[0]), null.String{}, - toid.New(int32(s.sequence), 3, 1).ToInt64(), + toid.New(int32(s.lcm.LedgerSequence()), 3, 1).ToInt64(), uint32(2), history.EffectAccountDebited, []byte("{\"amount\":\"10.0000000\",\"asset_type\":\"native\"}"), ).Return(nil).Once() } -func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulCreateAccounts() { - s.mockQ.On( - "CreateAccounts", - s.ctx, - mock.AnythingOfType("[]string"), - maxBatchSize, - ).Run(func(args mock.Arguments) { - arg := args.Get(1).([]string) - s.Assert().ElementsMatch(s.addresses, arg) - }).Return(s.addressToID, nil).Once() -} - func (s *EffectsProcessorTestSuiteLedger) TestEmptyEffects() { - err := s.processor.Commit(context.Background()) - s.Assert().NoError(err) + s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() + s.Assert().NoError(s.processor.Flush(s.ctx, s.mockSession)) } func (s *EffectsProcessorTestSuiteLedger) TestIngestEffectsSucceeds() { - s.mockSuccessfulCreateAccounts() - s.mockQ.On("NewEffectBatchInsertBuilder", maxBatchSize). - Return(s.mockBatchInsertBuilder).Once() - s.mockSuccessfulEffectBatchAdds() - - s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() - for _, tx := range s.txs { - err := s.processor.ProcessTransaction(s.ctx, tx) - s.Assert().NoError(err) + s.Assert().NoError(s.processor.ProcessTransaction(s.lcm, tx)) } - err := s.processor.Commit(s.ctx) - s.Assert().NoError(err) -} - -func (s *EffectsProcessorTestSuiteLedger) TestCreateAccountsFails() { - s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Return(s.addressToID, errors.New("transient error")).Once() - for _, tx := range s.txs { - err := s.processor.ProcessTransaction(s.ctx, tx) - s.Assert().NoError(err) - } - err := s.processor.Commit(s.ctx) - s.Assert().EqualError(err, "Could not create account ids: transient error") + s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() + s.Assert().NoError(s.processor.Flush(s.ctx, s.mockSession)) } func (s *EffectsProcessorTestSuiteLedger) TestBatchAddFails() { - s.mockSuccessfulCreateAccounts() - s.mockQ.On("NewEffectBatchInsertBuilder", maxBatchSize). - Return(s.mockBatchInsertBuilder).Once() - s.mockBatchInsertBuilder.On( - "Add", s.ctx, - s.addressToID[s.addresses[2]], + "Add", + s.accountLoader.GetFuture(s.addresses[2]), null.String{}, - toid.New(int32(s.sequence), 1, 1).ToInt64(), + toid.New(int32(s.lcm.LedgerSequence()), 1, 1).ToInt64(), uint32(1), history.EffectSequenceBumped, []byte("{\"new_seq\":300000000000}"), ).Return(errors.New("transient error")).Once() - for _, tx := range s.txs { - err := s.processor.ProcessTransaction(s.ctx, tx) - s.Assert().NoError(err) - } - err := s.processor.Commit(s.ctx) - s.Assert().EqualError(err, "could not insert operation effect in db: transient error") + + s.Assert().EqualError( + s.processor.ProcessTransaction(s.lcm, s.txs[0]), + "reading operation 85899350017 effects: could not insert operation effect in db: transient error", + ) } func getRevokeSponsorshipMeta(t *testing.T) (string, []effect) { @@ -477,7 +446,7 @@ func TestEffectsCoversAllOperationTypes(t *testing.T) { } assert.True(t, err2 != nil || err == nil, s) }() - _, err = operation.effects() + err = operation.ingestEffects(history.NewAccountLoader(), &history.MockEffectBatchInsertBuilder{}) }() } @@ -499,8 +468,8 @@ func TestEffectsCoversAllOperationTypes(t *testing.T) { ledgerSequence: 1, } // calling effects should error due to the unknown operation - _, err := operation.effects() - assert.Contains(t, err.Error(), "unknown operation type") + err := operation.ingestEffects(history.NewAccountLoader(), &history.MockEffectBatchInsertBuilder{}) + assert.Contains(t, err.Error(), "Unknown operation type") } func TestOperationEffects(t *testing.T) { @@ -1561,7 +1530,6 @@ func TestOperationEffects(t *testing.T) { } for _, tc := range testCases { t.Run(tc.desc, func(t *testing.T) { - tt := assert.New(t) transaction := BuildLedgerTransaction( t, TestTransaction{ @@ -1581,15 +1549,12 @@ func TestOperationEffects(t *testing.T) { ledgerSequence: tc.sequence, } - effects, err := operation.effects() - tt.NoError(err) - tt.Equal(tc.expected, effects) + assertIngestEffects(t, operation, tc.expected) }) } } func TestOperationEffectsSetOptionsSignersOrder(t *testing.T) { - tt := assert.New(t) transaction := ingest.LedgerTransaction{ UnsafeMeta: createTransactionMeta([]xdr.OperationMeta{ { @@ -1671,8 +1636,6 @@ func TestOperationEffectsSetOptionsSignersOrder(t *testing.T) { ledgerSequence: 46, } - effects, err := operation.effects() - tt.NoError(err) expected := []effect{ { address: "GCBBDQLCTNASZJ3MTKAOYEOWRGSHDFAJVI7VPZUOP7KXNHYR3HP2BUKV", @@ -1715,12 +1678,11 @@ func TestOperationEffectsSetOptionsSignersOrder(t *testing.T) { order: uint32(4), }, } - tt.Equal(expected, effects) + assertIngestEffects(t, operation, expected) } // Regression for https://github.com/stellar/go/issues/2136 func TestOperationEffectsSetOptionsSignersNoUpdated(t *testing.T) { - tt := assert.New(t) transaction := ingest.LedgerTransaction{ UnsafeMeta: createTransactionMeta([]xdr.OperationMeta{ { @@ -1802,8 +1764,6 @@ func TestOperationEffectsSetOptionsSignersNoUpdated(t *testing.T) { ledgerSequence: 46, } - effects, err := operation.effects() - tt.NoError(err) expected := []effect{ { address: "GCBBDQLCTNASZJ3MTKAOYEOWRGSHDFAJVI7VPZUOP7KXNHYR3HP2BUKV", @@ -1835,11 +1795,10 @@ func TestOperationEffectsSetOptionsSignersNoUpdated(t *testing.T) { order: uint32(3), }, } - tt.Equal(expected, effects) + assertIngestEffects(t, operation, expected) } func TestOperationRegressionAccountTrustItself(t *testing.T) { - tt := assert.New(t) // NOTE: when an account trusts itself, the transaction is successful but // no ledger entries are actually modified. transaction := ingest.LedgerTransaction{ @@ -1868,9 +1827,7 @@ func TestOperationRegressionAccountTrustItself(t *testing.T) { ledgerSequence: 46, } - effects, err := operation.effects() - tt.NoError(err) - tt.Equal([]effect{}, effects) + assertIngestEffects(t, operation, []effect{}) } func TestOperationEffectsAllowTrustAuthorizedToMaintainLiabilities(t *testing.T) { @@ -1904,9 +1861,6 @@ func TestOperationEffectsAllowTrustAuthorizedToMaintainLiabilities(t *testing.T) ledgerSequence: 1, } - effects, err := operation.effects() - tt.NoError(err) - expected := []effect{ { address: "GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD", @@ -1934,11 +1888,10 @@ func TestOperationEffectsAllowTrustAuthorizedToMaintainLiabilities(t *testing.T) order: uint32(2), }, } - tt.Equal(expected, effects) + assertIngestEffects(t, operation, expected) } func TestOperationEffectsClawback(t *testing.T) { - tt := assert.New(t) aid := xdr.MustAddress("GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD") source := aid.ToMuxedAccount() op := xdr.Operation{ @@ -1965,9 +1918,6 @@ func TestOperationEffectsClawback(t *testing.T) { ledgerSequence: 1, } - effects, err := operation.effects() - tt.NoError(err) - expected := []effect{ { address: "GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD", @@ -1994,11 +1944,10 @@ func TestOperationEffectsClawback(t *testing.T) { order: uint32(2), }, } - tt.Equal(expected, effects) + assertIngestEffects(t, operation, expected) } func TestOperationEffectsClawbackClaimableBalance(t *testing.T) { - tt := assert.New(t) aid := xdr.MustAddress("GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD") source := aid.ToMuxedAccount() var balanceID xdr.ClaimableBalanceId @@ -2025,9 +1974,6 @@ func TestOperationEffectsClawbackClaimableBalance(t *testing.T) { ledgerSequence: 1, } - effects, err := operation.effects() - tt.NoError(err) - expected := []effect{ { address: "GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD", @@ -2039,11 +1985,10 @@ func TestOperationEffectsClawbackClaimableBalance(t *testing.T) { order: uint32(1), }, } - tt.Equal(expected, effects) + assertIngestEffects(t, operation, expected) } func TestOperationEffectsSetTrustLineFlags(t *testing.T) { - tt := assert.New(t) aid := xdr.MustAddress("GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD") source := aid.ToMuxedAccount() trustor := xdr.MustAddress("GAUJETIZVEP2NRYLUESJ3LS66NVCEGMON4UDCBCSBEVPIID773P2W6AY") @@ -2074,9 +2019,6 @@ func TestOperationEffectsSetTrustLineFlags(t *testing.T) { ledgerSequence: 1, } - effects, err := operation.effects() - tt.NoError(err) - expected := []effect{ { address: "GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD", @@ -2094,7 +2036,7 @@ func TestOperationEffectsSetTrustLineFlags(t *testing.T) { order: uint32(1), }, } - tt.Equal(expected, effects) + assertIngestEffects(t, operation, expected) } type CreateClaimableBalanceEffectsTestSuite struct { @@ -2343,9 +2285,7 @@ func (s *CreateClaimableBalanceEffectsTestSuite) TestEffects() { ledgerSequence: 1, } - effects, err := operation.effects() - s.Assert().NoError(err) - s.Assert().Equal(tc.expected, effects) + assertIngestEffects(t, operation, tc.expected) }) } } @@ -2603,13 +2543,42 @@ func (s *ClaimClaimableBalanceEffectsTestSuite) TestEffects() { ledgerSequence: 1, } - effects, err := operation.effects() - s.Assert().NoError(err) - s.Assert().Equal(tc.expected, effects) + assertIngestEffects(t, operation, tc.expected) }) } } +type effect struct { + address string + addressMuxed null.String + operationID int64 + details map[string]interface{} + effectType history.EffectType + order uint32 +} + +func assertIngestEffects(t *testing.T, operation transactionOperationWrapper, expected []effect) { + accountLoader := history.NewAccountLoader() + mockBatchInsertBuilder := &history.MockEffectBatchInsertBuilder{} + + for _, expectedEffect := range expected { + detailsJSON, err := json.Marshal(expectedEffect.details) + assert.NoError(t, err) + mockBatchInsertBuilder.On( + "Add", + accountLoader.GetFuture(expectedEffect.address), + expectedEffect.addressMuxed, + expectedEffect.operationID, + expectedEffect.order, + expectedEffect.effectType, + detailsJSON, + ).Return(nil).Once() + } + + assert.NoError(t, operation.ingestEffects(accountLoader, mockBatchInsertBuilder)) + mockBatchInsertBuilder.AssertExpectations(t) +} + func TestClaimClaimableBalanceEffectsTestSuite(t *testing.T) { suite.Run(t, new(ClaimClaimableBalanceEffectsTestSuite)) } @@ -2826,10 +2795,7 @@ func TestTrustlineSponsorshipEffects(t *testing.T) { ledgerSequence: 1, } - effects, err := operation.effects() - assert.NoError(t, err) - assert.Equal(t, expected, effects) - + assertIngestEffects(t, operation, expected) } func TestLiquidityPoolEffects(t *testing.T) { @@ -3460,9 +3426,7 @@ func TestLiquidityPoolEffects(t *testing.T) { ledgerSequence: 1, } - effects, err := operation.effects() - assert.NoError(t, err) - assert.Equal(t, tc.expected, effects) + assertIngestEffects(t, operation, tc.expected) }) } } @@ -3768,10 +3732,7 @@ func TestInvokeHostFunctionEffects(t *testing.T) { network: networkPassphrase, } - effects, err := operation.effects() - assert.NoErrorf(t, err, "event type %v", testCase.eventType) - assert.Lenf(t, effects, len(testCase.expected), "event type %v", testCase.eventType) - assert.Equalf(t, testCase.expected, effects, "event type %v", testCase.eventType) + assertIngestEffects(t, operation, testCase.expected) }) } } diff --git a/services/horizon/internal/ingest/processors/ledgers_processor.go b/services/horizon/internal/ingest/processors/ledgers_processor.go index 01c29b43d9..942a5f8522 100644 --- a/services/horizon/internal/ingest/processors/ledgers_processor.go +++ b/services/horizon/internal/ingest/processors/ledgers_processor.go @@ -5,69 +5,84 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) -type LedgersProcessor struct { - ledgersQ history.QLedgers - ledger xdr.LedgerHeaderHistoryEntry - ingestVersion int +type ledgerInfo struct { + header xdr.LedgerHeaderHistoryEntry successTxCount int failedTxCount int opCount int txSetOpCount int } -func NewLedgerProcessor( - ledgerQ history.QLedgers, - ledger xdr.LedgerHeaderHistoryEntry, - ingestVersion int, -) *LedgersProcessor { +type LedgersProcessor struct { + batch history.LedgerBatchInsertBuilder + ledgers map[uint32]*ledgerInfo + ingestVersion int +} + +func NewLedgerProcessor(batch history.LedgerBatchInsertBuilder, ingestVersion int) *LedgersProcessor { return &LedgersProcessor{ - ledger: ledger, - ledgersQ: ledgerQ, + batch: batch, + ledgers: map[uint32]*ledgerInfo{}, ingestVersion: ingestVersion, } } -func (p *LedgersProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) (err error) { +func (p *LedgersProcessor) ProcessLedger(lcm xdr.LedgerCloseMeta) *ledgerInfo { + sequence := lcm.LedgerSequence() + entry, ok := p.ledgers[sequence] + if !ok { + entry = &ledgerInfo{header: lcm.LedgerHeaderHistoryEntry()} + p.ledgers[sequence] = entry + } + return entry +} + +func (p *LedgersProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error { + entry := p.ProcessLedger(lcm) opCount := len(transaction.Envelope.Operations()) - p.txSetOpCount += opCount + entry.txSetOpCount += opCount if transaction.Result.Successful() { - p.successTxCount++ - p.opCount += opCount + entry.successTxCount++ + entry.opCount += opCount } else { - p.failedTxCount++ + entry.failedTxCount++ } return nil } -func (p *LedgersProcessor) Commit(ctx context.Context) error { - rowsAffected, err := p.ledgersQ.InsertLedger(ctx, - p.ledger, - p.successTxCount, - p.failedTxCount, - p.opCount, - p.txSetOpCount, - p.ingestVersion, - ) - - if err != nil { - return errors.Wrap(err, "Could not insert ledger") +func (p *LedgersProcessor) Flush(ctx context.Context, session db.SessionInterface) error { + if len(p.ledgers) == 0 { + return nil } - - sequence := uint32(p.ledger.Header.LedgerSeq) - - if rowsAffected != 1 { - log.WithField("rowsAffected", rowsAffected). - WithField("sequence", sequence). - Error("Invalid number of rows affected when ingesting new ledger") - return errors.Errorf( - "0 rows affected when ingesting new ledger: %v", - sequence, + var min, max uint32 + for ledger, entry := range p.ledgers { + err := p.batch.Add( + entry.header, + entry.successTxCount, + entry.failedTxCount, + entry.opCount, + entry.txSetOpCount, + p.ingestVersion, ) + if err != nil { + return errors.Wrapf(err, "error adding ledger %d to batch", ledger) + } + if min == 0 || ledger < min { + min = ledger + } + if max == 0 || ledger > max { + max = ledger + } + } + + if err := p.batch.Exec(ctx, session); err != nil { + return errors.Wrapf(err, "error committing ledgers %d - %d", min, max) } return nil diff --git a/services/horizon/internal/ingest/processors/ledgers_processor_test.go b/services/horizon/internal/ingest/processors/ledgers_processor_test.go index 05bd2c3c3b..308bb6995d 100644 --- a/services/horizon/internal/ingest/processors/ledgers_processor_test.go +++ b/services/horizon/internal/ingest/processors/ledgers_processor_test.go @@ -8,6 +8,7 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" "github.com/stretchr/testify/mock" @@ -16,15 +17,16 @@ import ( type LedgersProcessorTestSuiteLedger struct { suite.Suite - processor *LedgersProcessor - mockQ *history.MockQLedgers - header xdr.LedgerHeaderHistoryEntry - successCount int - failedCount int - opCount int - ingestVersion int - txs []ingest.LedgerTransaction - txSetOpCount int + processor *LedgersProcessor + mockSession *db.MockSession + mockBatchInsertBuilder *history.MockLedgersBatchInsertBuilder + header xdr.LedgerHeaderHistoryEntry + successCount int + failedCount int + opCount int + ingestVersion int + txs []ingest.LedgerTransaction + txSetOpCount int } func TestLedgersProcessorTestSuiteLedger(t *testing.T) { @@ -78,16 +80,16 @@ func createTransaction(successful bool, numOps int) ingest.LedgerTransaction { } func (s *LedgersProcessorTestSuiteLedger) SetupTest() { - s.mockQ = &history.MockQLedgers{} + s.mockBatchInsertBuilder = &history.MockLedgersBatchInsertBuilder{} s.ingestVersion = 100 s.header = xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ LedgerSeq: xdr.Uint32(20), }, } + s.processor = NewLedgerProcessor( - s.mockQ, - s.header, + s.mockBatchInsertBuilder, s.ingestVersion, ) @@ -104,63 +106,120 @@ func (s *LedgersProcessorTestSuiteLedger) SetupTest() { } func (s *LedgersProcessorTestSuiteLedger) TearDownTest() { - s.mockQ.AssertExpectations(s.T()) + s.mockBatchInsertBuilder.AssertExpectations(s.T()) } func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerSucceeds() { ctx := context.Background() - s.mockQ.On( - "InsertLedger", - ctx, + + for _, tx := range s.txs { + err := s.processor.ProcessTransaction(xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: s.header, + }, + }, tx) + s.Assert().NoError(err) + } + + nextHeader := xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(21), + }, + } + nextTransactions := []ingest.LedgerTransaction{ + createTransaction(true, 1), + createTransaction(false, 2), + } + for _, tx := range nextTransactions { + err := s.processor.ProcessTransaction(xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: nextHeader, + }, + }, tx) + s.Assert().NoError(err) + } + + s.mockBatchInsertBuilder.On( + "Add", s.header, s.successCount, s.failedCount, s.opCount, s.txSetOpCount, s.ingestVersion, - ).Return(int64(1), nil) + ).Return(nil) + + s.mockBatchInsertBuilder.On( + "Add", + nextHeader, + 1, + 1, + 1, + 3, + s.ingestVersion, + ).Return(nil) - for _, tx := range s.txs { - err := s.processor.ProcessTransaction(ctx, tx) - s.Assert().NoError(err) - } + s.mockBatchInsertBuilder.On( + "Exec", + ctx, + s.mockSession, + ).Return(nil) - err := s.processor.Commit(ctx) + err := s.processor.Flush(ctx, s.mockSession) s.Assert().NoError(err) } func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerReturnsError() { - ctx := context.Background() - s.mockQ.On( - "InsertLedger", - ctx, + s.mockBatchInsertBuilder.On( + "Add", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, - ).Return(int64(0), errors.New("transient error")) + ).Return(errors.New("transient error")) + + err := s.processor.ProcessTransaction(xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: s.header, + }, + }, s.txs[0]) + s.Assert().NoError(err) - err := s.processor.Commit(ctx) - s.Assert().Error(err) - s.Assert().EqualError(err, "Could not insert ledger: transient error") + s.Assert().EqualError(s.processor.Flush( + context.Background(), s.mockSession), + "error adding ledger 20 to batch: transient error", + ) } -func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerNoRowsAffected() { +func (s *LedgersProcessorTestSuiteLedger) TestExecFails() { ctx := context.Background() - s.mockQ.On( - "InsertLedger", - ctx, + s.mockBatchInsertBuilder.On( + "Add", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything, - ).Return(int64(0), nil) + ).Return(nil) + + s.mockBatchInsertBuilder.On( + "Exec", + ctx, + s.mockSession, + ).Return(errors.New("transient exec error")) - err := s.processor.Commit(ctx) - s.Assert().Error(err) - s.Assert().EqualError(err, "0 rows affected when ingesting new ledger: 20") + err := s.processor.ProcessTransaction(xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: s.header, + }, + }, s.txs[0]) + s.Assert().NoError(err) + + s.Assert().EqualError(s.processor.Flush( + context.Background(), s.mockSession), + "error committing ledgers 20 - 20: transient exec error", + ) } diff --git a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor.go b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor.go index 38010ddb51..0a38215f08 100644 --- a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor.go +++ b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor.go @@ -5,53 +5,38 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" - set "github.com/stellar/go/support/collections/set" + "github.com/stellar/go/support/collections/set" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/toid" "github.com/stellar/go/xdr" ) -type liquidityPool struct { - internalID int64 // Bigint auto-generated by postgres - transactionSet set.Set[int64] - operationSet set.Set[int64] -} - -func (b *liquidityPool) addTransactionID(id int64) { - if b.transactionSet == nil { - b.transactionSet = set.Set[int64]{} - } - b.transactionSet.Add(id) -} - -func (b *liquidityPool) addOperationID(id int64) { - if b.operationSet == nil { - b.operationSet = set.Set[int64]{} - } - b.operationSet.Add(id) -} - type LiquidityPoolsTransactionProcessor struct { - sequence uint32 - liquidityPoolSet map[string]liquidityPool - qLiquidityPools history.QHistoryLiquidityPools + lpLoader *history.LiquidityPoolLoader + txBatch history.TransactionLiquidityPoolBatchInsertBuilder + opBatch history.OperationLiquidityPoolBatchInsertBuilder } -func NewLiquidityPoolsTransactionProcessor(Q history.QHistoryLiquidityPools, sequence uint32) *LiquidityPoolsTransactionProcessor { +func NewLiquidityPoolsTransactionProcessor( + lpLoader *history.LiquidityPoolLoader, + txBatch history.TransactionLiquidityPoolBatchInsertBuilder, + opBatch history.OperationLiquidityPoolBatchInsertBuilder, +) *LiquidityPoolsTransactionProcessor { return &LiquidityPoolsTransactionProcessor{ - qLiquidityPools: Q, - sequence: sequence, - liquidityPoolSet: map[string]liquidityPool{}, + lpLoader: lpLoader, + txBatch: txBatch, + opBatch: opBatch, } } -func (p *LiquidityPoolsTransactionProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error { - err := p.addTransactionLiquidityPools(p.liquidityPoolSet, p.sequence, transaction) +func (p *LiquidityPoolsTransactionProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error { + err := p.addTransactionLiquidityPools(lcm.LedgerSequence(), transaction) if err != nil { return err } - err = p.addOperationLiquidityPools(p.liquidityPoolSet, p.sequence, transaction) + err = p.addOperationLiquidityPools(lcm.LedgerSequence(), transaction) if err != nil { return err } @@ -59,29 +44,23 @@ func (p *LiquidityPoolsTransactionProcessor) ProcessTransaction(ctx context.Cont return nil } -func (p *LiquidityPoolsTransactionProcessor) addTransactionLiquidityPools(lpSet map[string]liquidityPool, sequence uint32, transaction ingest.LedgerTransaction) error { +func (p *LiquidityPoolsTransactionProcessor) addTransactionLiquidityPools(sequence uint32, transaction ingest.LedgerTransaction) error { transactionID := toid.New(int32(sequence), int32(transaction.Index), 0).ToInt64() - transactionLiquidityPools, err := liquidityPoolsForTransaction( - sequence, - transaction, - ) + lps, err := liquidityPoolsForTransaction(transaction) if err != nil { return errors.Wrap(err, "Could not determine liquidity pools for transaction") } - for _, lp := range transactionLiquidityPools { - entry := lpSet[lp] - entry.addTransactionID(transactionID) - lpSet[lp] = entry + for _, lp := range dedupeStrings(lps) { + if err = p.txBatch.Add(transactionID, p.lpLoader.GetFuture(lp)); err != nil { + return err + } } return nil } -func liquidityPoolsForTransaction( - sequence uint32, - transaction ingest.LedgerTransaction, -) ([]string, error) { +func liquidityPoolsForTransaction(transaction ingest.LedgerTransaction) ([]string, error) { changes, err := transaction.GetChanges() if err != nil { return nil, err @@ -90,19 +69,20 @@ func liquidityPoolsForTransaction( if err != nil { return nil, errors.Wrapf(err, "reading transaction %v liquidity pools", transaction.Index) } - return dedupeLiquidityPools(lps) + return lps, nil } -func dedupeLiquidityPools(in []string) (out []string, err error) { +func dedupeStrings(in []string) []string { set := set.Set[string]{} for _, id := range in { set.Add(id) } + out := make([]string, 0, len(in)) for id := range set { out = append(out, id) } - return + return out } func liquidityPoolsForChanges( @@ -132,26 +112,7 @@ func liquidityPoolsForChanges( return lps, nil } -func (p *LiquidityPoolsTransactionProcessor) addOperationLiquidityPools(lpSet map[string]liquidityPool, sequence uint32, transaction ingest.LedgerTransaction) error { - liquidityPools, err := liquidityPoolsForOperations(transaction, sequence) - if err != nil { - return errors.Wrap(err, "could not determine operation liquidity pools") - } - - for operationID, lps := range liquidityPools { - for _, lp := range lps { - entry := lpSet[lp] - entry.addOperationID(operationID) - lpSet[lp] = entry - } - } - - return nil -} - -func liquidityPoolsForOperations(transaction ingest.LedgerTransaction, sequence uint32) (map[int64][]string, error) { - lps := map[int64][]string{} - +func (p *LiquidityPoolsTransactionProcessor) addOperationLiquidityPools(sequence uint32, transaction ingest.LedgerTransaction) error { for opi, op := range transaction.Envelope.Operations() { operation := transactionOperationWrapper{ index: uint32(opi), @@ -162,91 +123,29 @@ func liquidityPoolsForOperations(transaction ingest.LedgerTransaction, sequence changes, err := transaction.GetOperationChanges(uint32(opi)) if err != nil { - return lps, err - } - c, err := liquidityPoolsForChanges(changes) - if err != nil { - return lps, errors.Wrapf(err, "reading operation %v liquidity pools", operation.ID()) - } - lps[operation.ID()] = c - } - - return lps, nil -} - -func (p *LiquidityPoolsTransactionProcessor) Commit(ctx context.Context) error { - if len(p.liquidityPoolSet) > 0 { - if err := p.loadLiquidityPoolIDs(ctx, p.liquidityPoolSet); err != nil { return err } - - if err := p.insertDBTransactionLiquidityPools(ctx, p.liquidityPoolSet); err != nil { - return err - } - - if err := p.insertDBOperationsLiquidityPools(ctx, p.liquidityPoolSet); err != nil { - return err - } - } - - return nil -} - -func (p *LiquidityPoolsTransactionProcessor) loadLiquidityPoolIDs(ctx context.Context, liquidityPoolSet map[string]liquidityPool) error { - ids := make([]string, 0, len(liquidityPoolSet)) - for id := range liquidityPoolSet { - ids = append(ids, id) - } - - toInternalID, err := p.qLiquidityPools.CreateHistoryLiquidityPools(ctx, ids, maxBatchSize) - if err != nil { - return errors.Wrap(err, "Could not create liquidity pool ids") - } - - for _, id := range ids { - internalID, ok := toInternalID[id] - if !ok { - return errors.Errorf("no internal id found for liquidity pool %s", id) + lps, err := liquidityPoolsForChanges(changes) + if err != nil { + return errors.Wrapf(err, "reading operation %v liquidity pools", operation.ID()) } - - lp := liquidityPoolSet[id] - lp.internalID = internalID - liquidityPoolSet[id] = lp - } - - return nil -} - -func (p LiquidityPoolsTransactionProcessor) insertDBTransactionLiquidityPools(ctx context.Context, liquidityPoolSet map[string]liquidityPool) error { - batch := p.qLiquidityPools.NewTransactionLiquidityPoolBatchInsertBuilder(maxBatchSize) - - for _, entry := range liquidityPoolSet { - for transactionID := range entry.transactionSet { - if err := batch.Add(ctx, transactionID, entry.internalID); err != nil { - return errors.Wrap(err, "could not insert transaction liquidity pool in db") + for _, lp := range dedupeStrings(lps) { + if err := p.opBatch.Add(operation.ID(), p.lpLoader.GetFuture(lp)); err != nil { + return err } } } - if err := batch.Exec(ctx); err != nil { - return errors.Wrap(err, "could not flush transaction liquidity pools to db") - } return nil } -func (p LiquidityPoolsTransactionProcessor) insertDBOperationsLiquidityPools(ctx context.Context, liquidityPoolSet map[string]liquidityPool) error { - batch := p.qLiquidityPools.NewOperationLiquidityPoolBatchInsertBuilder(maxBatchSize) - - for _, entry := range liquidityPoolSet { - for operationID := range entry.operationSet { - if err := batch.Add(ctx, operationID, entry.internalID); err != nil { - return errors.Wrap(err, "could not insert operation liquidity pool in db") - } - } +func (p *LiquidityPoolsTransactionProcessor) Flush(ctx context.Context, session db.SessionInterface) error { + if err := p.txBatch.Exec(ctx, session); err != nil { + return errors.Wrap(err, "Could not flush transaction liquidity pools to db") } - - if err := batch.Exec(ctx); err != nil { - return errors.Wrap(err, "could not flush operation liquidity pools to db") + if err := p.opBatch.Exec(ctx, session); err != nil { + return errors.Wrap(err, "Could not flush operation liquidity pools to db") } + return nil } diff --git a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go index bd80ca09cb..485d890dca 100644 --- a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go +++ b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go @@ -6,10 +6,10 @@ import ( "context" "testing" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "github.com/stellar/go/services/horizon/internal/db2/history" + "github.com/stellar/go/support/db" "github.com/stellar/go/toid" "github.com/stellar/go/xdr" ) @@ -18,11 +18,12 @@ type LiquidityPoolsTransactionProcessorTestSuiteLedger struct { suite.Suite ctx context.Context processor *LiquidityPoolsTransactionProcessor - mockQ *history.MockQHistoryLiquidityPools + mockSession *db.MockSession + lpLoader *history.LiquidityPoolLoader mockTransactionBatchInsertBuilder *history.MockTransactionLiquidityPoolBatchInsertBuilder mockOperationBatchInsertBuilder *history.MockOperationLiquidityPoolBatchInsertBuilder - sequence uint32 + lcm xdr.LedgerCloseMeta } func TestLiquidityPoolsTransactionProcessorTestSuiteLedger(t *testing.T) { @@ -31,40 +32,42 @@ func TestLiquidityPoolsTransactionProcessorTestSuiteLedger(t *testing.T) { func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) SetupTest() { s.ctx = context.Background() - s.mockQ = &history.MockQHistoryLiquidityPools{} s.mockTransactionBatchInsertBuilder = &history.MockTransactionLiquidityPoolBatchInsertBuilder{} s.mockOperationBatchInsertBuilder = &history.MockOperationLiquidityPoolBatchInsertBuilder{} - s.sequence = 20 + sequence := uint32(20) + s.lcm = xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(sequence), + }, + }, + }, + } + s.lpLoader = history.NewLiquidityPoolLoader() s.processor = NewLiquidityPoolsTransactionProcessor( - s.mockQ, - s.sequence, + s.lpLoader, + s.mockTransactionBatchInsertBuilder, + s.mockOperationBatchInsertBuilder, ) } func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) TearDownTest() { - s.mockQ.AssertExpectations(s.T()) s.mockTransactionBatchInsertBuilder.AssertExpectations(s.T()) s.mockOperationBatchInsertBuilder.AssertExpectations(s.T()) } -func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) mockTransactionBatchAdd(transactionID, internalID int64, err error) { - s.mockTransactionBatchInsertBuilder.On("Add", s.ctx, transactionID, internalID).Return(err).Once() -} - -func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) mockOperationBatchAdd(operationID, internalID int64, err error) { - s.mockOperationBatchInsertBuilder.On("Add", s.ctx, operationID, internalID).Return(err).Once() -} - func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) TestEmptyLiquidityPools() { - // What is this expecting? Doesn't seem to assert anything meaningful... - err := s.processor.Commit(context.Background()) + s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() + s.mockOperationBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() + + err := s.processor.Flush(context.Background(), s.mockSession) s.Assert().NoError(err) } func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) testOperationInserts(poolID xdr.PoolId, body xdr.OperationBody, change xdr.LedgerEntryChange) { // Setup the transaction - internalID := int64(1234) txn := createTransaction(true, 1) txn.Envelope.Operations()[0].Body = body txn.UnsafeMeta.V = 2 @@ -82,6 +85,20 @@ func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) testOperationInserts }, }, change, + // add a duplicate change to test that the processor + // does not insert duplicate rows + { + Type: xdr.LedgerEntryChangeTypeLedgerEntryState, + State: &xdr.LedgerEntry{ + Data: xdr.LedgerEntryData{ + Type: xdr.LedgerEntryTypeLiquidityPool, + LiquidityPool: &xdr.LiquidityPoolEntry{ + LiquidityPoolId: poolID, + }, + }, + }, + }, + change, }}, } @@ -100,44 +117,28 @@ func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) testOperationInserts }, } } - txnID := toid.New(int32(s.sequence), int32(txn.Index), 0).ToInt64() + txnID := toid.New(int32(s.lcm.LedgerSequence()), int32(txn.Index), 0).ToInt64() opID := (&transactionOperationWrapper{ index: uint32(0), transaction: txn, operation: txn.Envelope.Operations()[0], - ledgerSequence: s.sequence, + ledgerSequence: s.lcm.LedgerSequence(), }).ID() hexID := PoolIDToString(poolID) - // Setup a q - s.mockQ.On("CreateHistoryLiquidityPools", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]string) - s.Assert().ElementsMatch( - []string{hexID}, - arg, - ) - }).Return(map[string]int64{ - hexID: internalID, - }, nil).Once() - // Prepare to process transactions successfully - s.mockQ.On("NewTransactionLiquidityPoolBatchInsertBuilder", maxBatchSize). - Return(s.mockTransactionBatchInsertBuilder).Once() - s.mockTransactionBatchAdd(txnID, internalID, nil) - s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() + s.mockTransactionBatchInsertBuilder.On("Add", txnID, s.lpLoader.GetFuture(hexID)).Return(nil).Once() + s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() // Prepare to process operations successfully - s.mockQ.On("NewOperationLiquidityPoolBatchInsertBuilder", maxBatchSize). - Return(s.mockOperationBatchInsertBuilder).Once() - s.mockOperationBatchAdd(opID, internalID, nil) - s.mockOperationBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() + s.mockOperationBatchInsertBuilder.On("Add", opID, s.lpLoader.GetFuture(hexID)).Return(nil).Once() + s.mockOperationBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() // Process the transaction - err := s.processor.ProcessTransaction(s.ctx, txn) + err := s.processor.ProcessTransaction(s.lcm, txn) s.Assert().NoError(err) - err = s.processor.Commit(s.ctx) + err = s.processor.Flush(s.ctx, s.mockSession) s.Assert().NoError(err) } diff --git a/services/horizon/internal/ingest/processors/main.go b/services/horizon/internal/ingest/processors/main.go index 5088dd97aa..94f83f3fa9 100644 --- a/services/horizon/internal/ingest/processors/main.go +++ b/services/horizon/internal/ingest/processors/main.go @@ -1,7 +1,13 @@ package processors import ( + "context" + "io" + "github.com/guregu/null" + "github.com/stellar/go/ingest" + "github.com/stellar/go/support/db" + "github.com/stellar/go/support/errors" logpkg "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" ) @@ -10,6 +16,65 @@ var log = logpkg.DefaultLogger.WithField("service", "ingest") const maxBatchSize = 100000 +type ChangeProcessor interface { + ProcessChange(ctx context.Context, change ingest.Change) error +} + +type LedgerTransactionProcessor interface { + ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error + Flush(ctx context.Context, session db.SessionInterface) error +} + +type LedgerTransactionFilterer interface { + FilterTransaction(ctx context.Context, transaction ingest.LedgerTransaction) (bool, error) +} + +func StreamLedgerTransactions( + ctx context.Context, + txFilterer LedgerTransactionFilterer, + filteredTxProcessor LedgerTransactionProcessor, + txProcessor LedgerTransactionProcessor, + reader *ingest.LedgerTransactionReader, + ledger xdr.LedgerCloseMeta, +) error { + for { + tx, err := reader.Read() + if err == io.EOF { + return nil + } + if err != nil { + return errors.Wrap(err, "could not read transaction") + } + include, err := txFilterer.FilterTransaction(ctx, tx) + if err != nil { + return errors.Wrapf( + err, + "could not filter transaction %v", + tx.Index, + ) + } + if !include { + if err = filteredTxProcessor.ProcessTransaction(ledger, tx); err != nil { + return errors.Wrapf( + err, + "could not process transaction %v", + tx.Index, + ) + } + log.Debugf("Filters did not find match on transaction, dropping this tx with hash %v", tx.Result.TransactionHash.HexString()) + continue + } + + if err = txProcessor.ProcessTransaction(ledger, tx); err != nil { + return errors.Wrapf( + err, + "could not process transaction %v", + tx.Index, + ) + } + } +} + func ledgerEntrySponsorToNullString(entry xdr.LedgerEntry) null.String { sponsoringID := entry.SponsoringID() diff --git a/services/horizon/internal/ingest/processors/operations_processor.go b/services/horizon/internal/ingest/processors/operations_processor.go index c9c0e4f72f..c8ae1a9585 100644 --- a/services/horizon/internal/ingest/processors/operations_processor.go +++ b/services/horizon/internal/ingest/processors/operations_processor.go @@ -12,6 +12,7 @@ import ( "github.com/stellar/go/protocols/horizon/base" "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/support/contractevents" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/toid" "github.com/stellar/go/xdr" @@ -19,30 +20,25 @@ import ( // OperationProcessor operations processor type OperationProcessor struct { - operationsQ history.QOperations - - sequence uint32 - batch history.OperationBatchInsertBuilder - network string + batch history.OperationBatchInsertBuilder + network string } -func NewOperationProcessor(operationsQ history.QOperations, sequence uint32, network string) *OperationProcessor { +func NewOperationProcessor(batch history.OperationBatchInsertBuilder, network string) *OperationProcessor { return &OperationProcessor{ - operationsQ: operationsQ, - sequence: sequence, - batch: operationsQ.NewOperationBatchInsertBuilder(maxBatchSize), - network: network, + batch: batch, + network: network, } } // ProcessTransaction process the given transaction -func (p *OperationProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error { +func (p *OperationProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error { for i, op := range transaction.Envelope.Operations() { operation := transactionOperationWrapper{ index: uint32(i), transaction: transaction, operation: op, - ledgerSequence: p.sequence, + ledgerSequence: lcm.LedgerSequence(), network: p.network, } details, err := operation.Details() @@ -61,7 +57,7 @@ func (p *OperationProcessor) ProcessTransaction(ctx context.Context, transaction if source.Type == xdr.CryptoKeyTypeKeyTypeMuxedEd25519 { sourceAccountMuxed = null.StringFrom(source.Address()) } - if err := p.batch.Add(ctx, + if err := p.batch.Add( operation.ID(), operation.TransactionID(), operation.Order(), @@ -78,8 +74,8 @@ func (p *OperationProcessor) ProcessTransaction(ctx context.Context, transaction return nil } -func (p *OperationProcessor) Commit(ctx context.Context) error { - return p.batch.Exec(ctx) +func (p *OperationProcessor) Flush(ctx context.Context, session db.SessionInterface) error { + return p.batch.Exec(ctx, session) } // transactionOperationWrapper represents the data for a single operation within a transaction @@ -342,7 +338,9 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, addAccountAndMuxedAccountDetails(details, *source, "from") addAccountAndMuxedAccountDetails(details, op.Destination, "to") details["amount"] = amount.String(op.Amount) - addAssetDetails(details, op.Asset, "") + if err := addAssetDetails(details, op.Asset, ""); err != nil { + return nil, err + } case xdr.OperationTypePathPaymentStrictReceive: op := operation.operation.Body.MustPathPaymentStrictReceiveOp() addAccountAndMuxedAccountDetails(details, *source, "from") @@ -351,8 +349,12 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, details["amount"] = amount.String(op.DestAmount) details["source_amount"] = amount.String(0) details["source_max"] = amount.String(op.SendMax) - addAssetDetails(details, op.DestAsset, "") - addAssetDetails(details, op.SendAsset, "source_") + if err := addAssetDetails(details, op.DestAsset, ""); err != nil { + return nil, err + } + if err := addAssetDetails(details, op.SendAsset, "source_"); err != nil { + return nil, err + } if operation.transaction.Result.Successful() { result := operation.OperationResult().MustPathPaymentStrictReceiveResult() @@ -362,7 +364,9 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, var path = make([]map[string]interface{}, len(op.Path)) for i := range op.Path { path[i] = make(map[string]interface{}) - addAssetDetails(path[i], op.Path[i], "") + if err := addAssetDetails(path[i], op.Path[i], ""); err != nil { + return nil, err + } } details["path"] = path @@ -374,8 +378,12 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, details["amount"] = amount.String(0) details["source_amount"] = amount.String(op.SendAmount) details["destination_min"] = amount.String(op.DestMin) - addAssetDetails(details, op.DestAsset, "") - addAssetDetails(details, op.SendAsset, "source_") + if err := addAssetDetails(details, op.DestAsset, ""); err != nil { + return nil, err + } + if err := addAssetDetails(details, op.SendAsset, "source_"); err != nil { + return nil, err + } if operation.transaction.Result.Successful() { result := operation.OperationResult().MustPathPaymentStrictSendResult() @@ -385,7 +393,9 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, var path = make([]map[string]interface{}, len(op.Path)) for i := range op.Path { path[i] = make(map[string]interface{}) - addAssetDetails(path[i], op.Path[i], "") + if err := addAssetDetails(path[i], op.Path[i], ""); err != nil { + return nil, err + } } details["path"] = path case xdr.OperationTypeManageBuyOffer: @@ -397,8 +407,12 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, "n": op.Price.N, "d": op.Price.D, } - addAssetDetails(details, op.Buying, "buying_") - addAssetDetails(details, op.Selling, "selling_") + if err := addAssetDetails(details, op.Buying, "buying_"); err != nil { + return nil, err + } + if err := addAssetDetails(details, op.Selling, "selling_"); err != nil { + return nil, err + } case xdr.OperationTypeManageSellOffer: op := operation.operation.Body.MustManageSellOfferOp() details["offer_id"] = op.OfferId @@ -408,8 +422,12 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, "n": op.Price.N, "d": op.Price.D, } - addAssetDetails(details, op.Buying, "buying_") - addAssetDetails(details, op.Selling, "selling_") + if err := addAssetDetails(details, op.Buying, "buying_"); err != nil { + return nil, err + } + if err := addAssetDetails(details, op.Selling, "selling_"); err != nil { + return nil, err + } case xdr.OperationTypeCreatePassiveSellOffer: op := operation.operation.Body.MustCreatePassiveSellOfferOp() details["amount"] = amount.String(op.Amount) @@ -418,8 +436,12 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, "n": op.Price.N, "d": op.Price.D, } - addAssetDetails(details, op.Buying, "buying_") - addAssetDetails(details, op.Selling, "selling_") + if err := addAssetDetails(details, op.Buying, "buying_"); err != nil { + return nil, err + } + if err := addAssetDetails(details, op.Selling, "selling_"); err != nil { + return nil, err + } case xdr.OperationTypeSetOptions: op := operation.operation.Body.MustSetOptionsOp() @@ -466,14 +488,18 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, return nil, err } } else { - addAssetDetails(details, op.Line.ToAsset(), "") + if err := addAssetDetails(details, op.Line.ToAsset(), ""); err != nil { + return nil, err + } details["trustee"] = details["asset_issuer"] } addAccountAndMuxedAccountDetails(details, *source, "trustor") details["limit"] = amount.String(op.Limit) case xdr.OperationTypeAllowTrust: op := operation.operation.Body.MustAllowTrustOp() - addAssetDetails(details, op.Asset.ToAsset(source.ToAccountId()), "") + if err := addAssetDetails(details, op.Asset.ToAsset(source.ToAccountId()), ""); err != nil { + return nil, err + } addAccountAndMuxedAccountDetails(details, *source, "trustee") details["trustor"] = op.Trustor.Address() details["authorize"] = xdr.TrustLineFlags(op.Authorize).IsAuthorized() @@ -544,7 +570,9 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, } case xdr.OperationTypeClawback: op := operation.operation.Body.MustClawbackOp() - addAssetDetails(details, op.Asset, "") + if err := addAssetDetails(details, op.Asset, ""); err != nil { + return nil, err + } addAccountAndMuxedAccountDetails(details, op.From, "from") details["amount"] = amount.String(op.Amount) case xdr.OperationTypeClawbackClaimableBalance: @@ -557,7 +585,9 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{}, case xdr.OperationTypeSetTrustLineFlags: op := operation.operation.Body.MustSetTrustLineFlagsOp() details["trustor"] = op.Trustor.Address() - addAssetDetails(details, op.Asset, "") + if err := addAssetDetails(details, op.Asset, ""); err != nil { + return nil, err + } if op.SetFlags > 0 { addTrustLineFlagDetails(details, xdr.TrustLineFlags(op.SetFlags), "set") } diff --git a/services/horizon/internal/ingest/processors/operations_processor_test.go b/services/horizon/internal/ingest/processors/operations_processor_test.go index ff7b8c3d94..83ef1636a8 100644 --- a/services/horizon/internal/ingest/processors/operations_processor_test.go +++ b/services/horizon/internal/ingest/processors/operations_processor_test.go @@ -18,6 +18,7 @@ import ( "github.com/stellar/go/services/horizon/internal/db2/history" "github.com/stellar/go/strkey" "github.com/stellar/go/support/contractevents" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) @@ -26,7 +27,7 @@ type OperationsProcessorTestSuiteLedger struct { suite.Suite ctx context.Context processor *OperationProcessor - mockQ *history.MockQOperations + mockSession *db.MockSession mockBatchInsertBuilder *history.MockOperationsBatchInsertBuilder } @@ -36,21 +37,15 @@ func TestOperationProcessorTestSuiteLedger(t *testing.T) { func (s *OperationsProcessorTestSuiteLedger) SetupTest() { s.ctx = context.Background() - s.mockQ = &history.MockQOperations{} s.mockBatchInsertBuilder = &history.MockOperationsBatchInsertBuilder{} - s.mockQ. - On("NewOperationBatchInsertBuilder", maxBatchSize). - Return(s.mockBatchInsertBuilder).Once() s.processor = NewOperationProcessor( - s.mockQ, - 56, + s.mockBatchInsertBuilder, "test network", ) } func (s *OperationsProcessorTestSuiteLedger) TearDownTest() { - s.mockQ.AssertExpectations(s.T()) s.mockBatchInsertBuilder.AssertExpectations(s.T()) } @@ -80,7 +75,6 @@ func (s *OperationsProcessorTestSuiteLedger) mockBatchInsertAdds(txs []ingest.Le } s.mockBatchInsertBuilder.On( "Add", - s.ctx, expected.ID(), expected.TransactionID(), expected.Order(), @@ -394,6 +388,17 @@ func (s *OperationsProcessorTestSuiteLedger) assertInvokeHostFunctionParameter(p } func (s *OperationsProcessorTestSuiteLedger) TestAddOperationSucceeds() { + sequence := uint32(56) + lcm := xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(sequence), + }, + }, + }, + } + unmuxed := xdr.MustAddress("GA5WBPYA5Y4WAEHXWR2UKO2UO4BUGHUQ74EUPKON2QHV4WRHOIRNKKH2") muxed := xdr.MuxedAccount{ Type: xdr.CryptoKeyTypeKeyTypeMuxedEd25519, @@ -424,23 +429,33 @@ func (s *OperationsProcessorTestSuiteLedger) TestAddOperationSucceeds() { var err error - err = s.mockBatchInsertAdds(txs, uint32(56)) + err = s.mockBatchInsertAdds(txs, sequence) s.Assert().NoError(err) - s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() - s.Assert().NoError(s.processor.Commit(s.ctx)) + s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() for _, tx := range txs { - err = s.processor.ProcessTransaction(s.ctx, tx) + err = s.processor.ProcessTransaction(lcm, tx) s.Assert().NoError(err) } + s.Assert().NoError(s.processor.Flush(s.ctx, s.mockSession)) } func (s *OperationsProcessorTestSuiteLedger) TestAddOperationFails() { + sequence := uint32(56) + lcm := xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(sequence), + }, + }, + }, + } tx := createTransaction(true, 1) s.mockBatchInsertBuilder. On( - "Add", s.ctx, + "Add", mock.Anything, mock.Anything, mock.Anything, @@ -451,14 +466,40 @@ func (s *OperationsProcessorTestSuiteLedger) TestAddOperationFails() { mock.Anything, ).Return(errors.New("transient error")).Once() - err := s.processor.ProcessTransaction(s.ctx, tx) + err := s.processor.ProcessTransaction(lcm, tx) s.Assert().Error(err) s.Assert().EqualError(err, "Error batch inserting operation rows: transient error") } func (s *OperationsProcessorTestSuiteLedger) TestExecFails() { - s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(errors.New("transient error")).Once() - err := s.processor.Commit(s.ctx) + sequence := uint32(56) + lcm := xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(sequence), + }, + }, + }, + } + tx := createTransaction(true, 1) + + s.mockBatchInsertBuilder. + On( + "Add", + mock.Anything, + mock.Anything, + mock.Anything, + mock.Anything, + mock.Anything, + mock.Anything, + mock.Anything, + mock.Anything, + ).Return(nil).Once() + s.Assert().NoError(s.processor.ProcessTransaction(lcm, tx)) + + s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(errors.New("transient error")).Once() + err := s.processor.Flush(s.ctx, s.mockSession) s.Assert().Error(err) s.Assert().EqualError(err, "transient error") } diff --git a/services/horizon/internal/ingest/processors/participants_processor.go b/services/horizon/internal/ingest/processors/participants_processor.go index d908f9ac69..7d4ae7fe39 100644 --- a/services/horizon/internal/ingest/processors/participants_processor.go +++ b/services/horizon/internal/ingest/processors/participants_processor.go @@ -7,7 +7,7 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" - set "github.com/stellar/go/support/collections/set" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/toid" "github.com/stellar/go/xdr" @@ -16,64 +16,23 @@ import ( // ParticipantsProcessor is a processor which ingests various participants // from different sources (transactions, operations, etc) type ParticipantsProcessor struct { - participantsQ history.QParticipants - sequence uint32 - participantSet map[string]participant + accountLoader *history.AccountLoader + txBatch history.TransactionParticipantsBatchInsertBuilder + opBatch history.OperationParticipantBatchInsertBuilder } -func NewParticipantsProcessor(participantsQ history.QParticipants, sequence uint32) *ParticipantsProcessor { +func NewParticipantsProcessor( + accountLoader *history.AccountLoader, + txBatch history.TransactionParticipantsBatchInsertBuilder, + opBatch history.OperationParticipantBatchInsertBuilder, +) *ParticipantsProcessor { return &ParticipantsProcessor{ - participantsQ: participantsQ, - sequence: sequence, - participantSet: map[string]participant{}, + accountLoader: accountLoader, + txBatch: txBatch, + opBatch: opBatch, } } -type participant struct { - accountID int64 - transactionSet set.Set[int64] - operationSet set.Set[int64] -} - -func (p *participant) addTransactionID(id int64) { - if p.transactionSet == nil { - p.transactionSet = set.Set[int64]{} - } - p.transactionSet.Add(id) -} - -func (p *participant) addOperationID(id int64) { - if p.operationSet == nil { - p.operationSet = set.Set[int64]{} - } - p.operationSet.Add(id) -} - -func (p *ParticipantsProcessor) loadAccountIDs(ctx context.Context, participantSet map[string]participant) error { - addresses := make([]string, 0, len(participantSet)) - for address := range participantSet { - addresses = append(addresses, address) - } - - addressToID, err := p.participantsQ.CreateAccounts(ctx, addresses, maxBatchSize) - if err != nil { - return errors.Wrap(err, "Could not create account ids") - } - - for _, address := range addresses { - id, ok := addressToID[address] - if !ok { - return errors.Errorf("no id found for account address %s", address) - } - - participantForAddress := participantSet[address] - participantForAddress.accountID = id - participantSet[address] = participantForAddress - } - - return nil -} - func participantsForChanges( changes xdr.LedgerEntryChanges, ) ([]xdr.AccountId, error) { @@ -141,7 +100,6 @@ func participantsForMeta( } func (p *ParticipantsProcessor) addTransactionParticipants( - participantSet map[string]participant, sequence uint32, transaction ingest.LedgerTransaction, ) error { @@ -155,17 +113,15 @@ func (p *ParticipantsProcessor) addTransactionParticipants( } for _, participant := range transactionParticipants { - address := participant.Address() - entry := participantSet[address] - entry.addTransactionID(transactionID) - participantSet[address] = entry + if err := p.txBatch.Add(transactionID, p.accountLoader.GetFuture(participant.Address())); err != nil { + return err + } } return nil } func (p *ParticipantsProcessor) addOperationsParticipants( - participantSet map[string]participant, sequence uint32, transaction ingest.LedgerTransaction, ) error { @@ -174,82 +130,39 @@ func (p *ParticipantsProcessor) addOperationsParticipants( return errors.Wrap(err, "could not determine operation participants") } - for operationID, p := range participants { - for _, participant := range p { + for operationID, addresses := range participants { + for _, participant := range addresses { address := participant.Address() - entry := participantSet[address] - entry.addOperationID(operationID) - participantSet[address] = entry - } - } - - return nil -} - -func (p *ParticipantsProcessor) insertDBTransactionParticipants(ctx context.Context, participantSet map[string]participant) error { - batch := p.participantsQ.NewTransactionParticipantsBatchInsertBuilder(maxBatchSize) - - for _, entry := range participantSet { - for transactionID := range entry.transactionSet { - if err := batch.Add(ctx, transactionID, entry.accountID); err != nil { - return errors.Wrap(err, "Could not insert transaction participant in db") + if err := p.opBatch.Add(operationID, p.accountLoader.GetFuture(address)); err != nil { + return err } } } - if err := batch.Exec(ctx); err != nil { - return errors.Wrap(err, "Could not flush transaction participants to db") - } return nil } -func (p *ParticipantsProcessor) insertDBOperationsParticipants(ctx context.Context, participantSet map[string]participant) error { - batch := p.participantsQ.NewOperationParticipantBatchInsertBuilder(maxBatchSize) +func (p *ParticipantsProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error { - for _, entry := range participantSet { - for operationID := range entry.operationSet { - if err := batch.Add(ctx, operationID, entry.accountID); err != nil { - return errors.Wrap(err, "could not insert operation participant in db") - } - } - } - - if err := batch.Exec(ctx); err != nil { - return errors.Wrap(err, "could not flush operation participants to db") - } - return nil -} - -func (p *ParticipantsProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) (err error) { - err = p.addTransactionParticipants(p.participantSet, p.sequence, transaction) - if err != nil { + if err := p.addTransactionParticipants(lcm.LedgerSequence(), transaction); err != nil { return err } - err = p.addOperationsParticipants(p.participantSet, p.sequence, transaction) - if err != nil { + if err := p.addOperationsParticipants(lcm.LedgerSequence(), transaction); err != nil { return err } return nil } -func (p *ParticipantsProcessor) Commit(ctx context.Context) (err error) { - if len(p.participantSet) > 0 { - if err = p.loadAccountIDs(ctx, p.participantSet); err != nil { - return err - } - - if err = p.insertDBTransactionParticipants(ctx, p.participantSet); err != nil { - return err - } - - if err = p.insertDBOperationsParticipants(ctx, p.participantSet); err != nil { - return err - } +func (p *ParticipantsProcessor) Flush(ctx context.Context, session db.SessionInterface) error { + if err := p.txBatch.Exec(ctx, session); err != nil { + return errors.Wrap(err, "Could not flush transaction participants to db") } - - return err + if err := p.opBatch.Exec(ctx, session); err != nil { + return errors.Wrap(err, "Could not flush operation participants to db") + } + return nil } func ParticipantsForTransaction( diff --git a/services/horizon/internal/ingest/processors/participants_processor_test.go b/services/horizon/internal/ingest/processors/participants_processor_test.go index 4780c2709c..2348b79eaf 100644 --- a/services/horizon/internal/ingest/processors/participants_processor_test.go +++ b/services/horizon/internal/ingest/processors/participants_processor_test.go @@ -6,11 +6,11 @@ import ( "context" "testing" - "github.com/stretchr/testify/mock" "github.com/stretchr/testify/suite" "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/toid" "github.com/stellar/go/xdr" @@ -20,19 +20,21 @@ type ParticipantsProcessorTestSuiteLedger struct { suite.Suite ctx context.Context processor *ParticipantsProcessor - mockQ *history.MockQParticipants + mockSession *db.MockSession mockBatchInsertBuilder *history.MockTransactionParticipantsBatchInsertBuilder mockOperationsBatchInsertBuilder *history.MockOperationParticipantBatchInsertBuilder - - firstTx ingest.LedgerTransaction - secondTx ingest.LedgerTransaction - thirdTx ingest.LedgerTransaction - firstTxID int64 - secondTxID int64 - thirdTxID int64 - addresses []string - addressToID map[string]int64 - txs []ingest.LedgerTransaction + accountLoader *history.AccountLoader + + lcm xdr.LedgerCloseMeta + firstTx ingest.LedgerTransaction + secondTx ingest.LedgerTransaction + thirdTx ingest.LedgerTransaction + firstTxID int64 + secondTxID int64 + thirdTxID int64 + addresses []string + addressToFuture map[string]history.FutureAccountID + txs []ingest.LedgerTransaction } func TestParticipantsProcessorTestSuiteLedger(t *testing.T) { @@ -41,10 +43,18 @@ func TestParticipantsProcessorTestSuiteLedger(t *testing.T) { func (s *ParticipantsProcessorTestSuiteLedger) SetupTest() { s.ctx = context.Background() - s.mockQ = &history.MockQParticipants{} s.mockBatchInsertBuilder = &history.MockTransactionParticipantsBatchInsertBuilder{} s.mockOperationsBatchInsertBuilder = &history.MockOperationParticipantBatchInsertBuilder{} sequence := uint32(20) + s.lcm = xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(sequence), + }, + }, + }, + } s.addresses = []string{ "GA5WBPYA5Y4WAEHXWR2UKO2UO4BUGHUQ74EUPKON2QHV4WRHOIRNKKH2", @@ -76,15 +86,16 @@ func (s *ParticipantsProcessorTestSuiteLedger) SetupTest() { s.thirdTx.Envelope.V1.Tx.SourceAccount = aid.ToMuxedAccount() s.thirdTxID = toid.New(int32(sequence), 3, 0).ToInt64() - s.addressToID = map[string]int64{ - s.addresses[0]: 2, - s.addresses[1]: 20, - s.addresses[2]: 200, + s.accountLoader = history.NewAccountLoader() + s.addressToFuture = map[string]history.FutureAccountID{} + for _, address := range s.addresses { + s.addressToFuture[address] = s.accountLoader.GetFuture(address) } s.processor = NewParticipantsProcessor( - s.mockQ, - sequence, + s.accountLoader, + s.mockBatchInsertBuilder, + s.mockOperationsBatchInsertBuilder, ) s.txs = []ingest.LedgerTransaction{ @@ -95,44 +106,46 @@ func (s *ParticipantsProcessorTestSuiteLedger) SetupTest() { } func (s *ParticipantsProcessorTestSuiteLedger) TearDownTest() { - s.mockQ.AssertExpectations(s.T()) s.mockBatchInsertBuilder.AssertExpectations(s.T()) s.mockOperationsBatchInsertBuilder.AssertExpectations(s.T()) } func (s *ParticipantsProcessorTestSuiteLedger) mockSuccessfulTransactionBatchAdds() { s.mockBatchInsertBuilder.On( - "Add", s.ctx, s.firstTxID, s.addressToID[s.addresses[0]], + "Add", s.firstTxID, s.addressToFuture[s.addresses[0]], ).Return(nil).Once() s.mockBatchInsertBuilder.On( - "Add", s.ctx, s.secondTxID, s.addressToID[s.addresses[1]], + "Add", s.secondTxID, s.addressToFuture[s.addresses[1]], ).Return(nil).Once() s.mockBatchInsertBuilder.On( - "Add", s.ctx, s.secondTxID, s.addressToID[s.addresses[2]], + "Add", s.secondTxID, s.addressToFuture[s.addresses[2]], ).Return(nil).Once() s.mockBatchInsertBuilder.On( - "Add", s.ctx, s.thirdTxID, s.addressToID[s.addresses[0]], + "Add", s.thirdTxID, s.addressToFuture[s.addresses[0]], ).Return(nil).Once() } func (s *ParticipantsProcessorTestSuiteLedger) mockSuccessfulOperationBatchAdds() { s.mockOperationsBatchInsertBuilder.On( - "Add", s.ctx, s.firstTxID+1, s.addressToID[s.addresses[0]], + "Add", s.firstTxID+1, s.addressToFuture[s.addresses[0]], ).Return(nil).Once() s.mockOperationsBatchInsertBuilder.On( - "Add", s.ctx, s.secondTxID+1, s.addressToID[s.addresses[1]], + "Add", s.secondTxID+1, s.addressToFuture[s.addresses[1]], ).Return(nil).Once() s.mockOperationsBatchInsertBuilder.On( - "Add", s.ctx, s.secondTxID+1, s.addressToID[s.addresses[2]], + "Add", s.secondTxID+1, s.addressToFuture[s.addresses[2]], ).Return(nil).Once() s.mockOperationsBatchInsertBuilder.On( - "Add", s.ctx, s.thirdTxID+1, s.addressToID[s.addresses[0]], + "Add", s.thirdTxID+1, s.addressToFuture[s.addresses[0]], ).Return(nil).Once() } func (s *ParticipantsProcessorTestSuiteLedger) TestEmptyParticipants() { - err := s.processor.Commit(s.ctx) + s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() + s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() + + err := s.processor.Flush(s.ctx, s.mockSession) s.Assert().NoError(err) } @@ -166,198 +179,82 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestFeeBumptransaction() { feeBumpTx.Result.Result.Result.Results = nil feeBumpTxID := toid.New(20, 1, 0).ToInt64() - addresses := s.addresses[:2] - addressToID := map[string]int64{ - addresses[0]: s.addressToID[addresses[0]], - addresses[1]: s.addressToID[addresses[1]], - } - s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]string) - s.Assert().ElementsMatch( - addresses, - arg, - ) - }).Return(addressToID, nil).Once() - s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder", maxBatchSize). - Return(s.mockBatchInsertBuilder).Once() - s.mockQ.On("NewOperationParticipantBatchInsertBuilder", maxBatchSize). - Return(s.mockOperationsBatchInsertBuilder).Once() - s.mockBatchInsertBuilder.On( - "Add", s.ctx, feeBumpTxID, addressToID[addresses[0]], + "Add", feeBumpTxID, s.addressToFuture[s.addresses[0]], ).Return(nil).Once() s.mockBatchInsertBuilder.On( - "Add", s.ctx, feeBumpTxID, addressToID[addresses[1]], + "Add", feeBumpTxID, s.addressToFuture[s.addresses[1]], ).Return(nil).Once() - s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() - s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() + s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() + s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() - s.Assert().NoError(s.processor.ProcessTransaction(s.ctx, feeBumpTx)) - s.Assert().NoError(s.processor.Commit(s.ctx)) + s.Assert().NoError(s.processor.ProcessTransaction(s.lcm, feeBumpTx)) + s.Assert().NoError(s.processor.Flush(s.ctx, s.mockSession)) } func (s *ParticipantsProcessorTestSuiteLedger) TestIngestParticipantsSucceeds() { - s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]string) - s.Assert().ElementsMatch( - s.addresses, - arg, - ) - }).Return(s.addressToID, nil).Once() - s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder", maxBatchSize). - Return(s.mockBatchInsertBuilder).Once() - s.mockQ.On("NewOperationParticipantBatchInsertBuilder", maxBatchSize). - Return(s.mockOperationsBatchInsertBuilder).Once() - s.mockSuccessfulTransactionBatchAdds() s.mockSuccessfulOperationBatchAdds() - s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() - s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() + s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() + s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() for _, tx := range s.txs { - err := s.processor.ProcessTransaction(s.ctx, tx) + err := s.processor.ProcessTransaction(s.lcm, tx) s.Assert().NoError(err) } - err := s.processor.Commit(s.ctx) + err := s.processor.Flush(s.ctx, s.mockSession) s.Assert().NoError(err) } -func (s *ParticipantsProcessorTestSuiteLedger) TestCreateAccountsFails() { - s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Return(s.addressToID, errors.New("transient error")).Once() - for _, tx := range s.txs { - err := s.processor.ProcessTransaction(s.ctx, tx) - s.Assert().NoError(err) - } - err := s.processor.Commit(s.ctx) - s.Assert().EqualError(err, "Could not create account ids: transient error") -} - func (s *ParticipantsProcessorTestSuiteLedger) TestBatchAddFails() { - s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]string) - s.Assert().ElementsMatch( - s.addresses, - arg, - ) - }).Return(s.addressToID, nil).Once() - s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder", maxBatchSize). - Return(s.mockBatchInsertBuilder).Once() - s.mockBatchInsertBuilder.On( - "Add", s.ctx, s.firstTxID, s.addressToID[s.addresses[0]], + "Add", s.firstTxID, s.addressToFuture[s.addresses[0]], ).Return(errors.New("transient error")).Once() - s.mockBatchInsertBuilder.On( - "Add", s.ctx, s.secondTxID, s.addressToID[s.addresses[1]], - ).Return(nil).Maybe() - s.mockBatchInsertBuilder.On( - "Add", s.ctx, s.secondTxID, s.addressToID[s.addresses[2]], - ).Return(nil).Maybe() - - s.mockBatchInsertBuilder.On( - "Add", s.ctx, s.thirdTxID, s.addressToID[s.addresses[0]], - ).Return(nil).Maybe() - for _, tx := range s.txs { - err := s.processor.ProcessTransaction(s.ctx, tx) - s.Assert().NoError(err) - } - err := s.processor.Commit(s.ctx) - s.Assert().EqualError(err, "Could not insert transaction participant in db: transient error") + err := s.processor.ProcessTransaction(s.lcm, s.txs[0]) + s.Assert().EqualError(err, "transient error") } func (s *ParticipantsProcessorTestSuiteLedger) TestOperationParticipantsBatchAddFails() { - s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]string) - s.Assert().ElementsMatch( - s.addresses, - arg, - ) - }).Return(s.addressToID, nil).Once() - s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder", maxBatchSize). - Return(s.mockBatchInsertBuilder).Once() - s.mockQ.On("NewOperationParticipantBatchInsertBuilder", maxBatchSize). - Return(s.mockOperationsBatchInsertBuilder).Once() - - s.mockSuccessfulTransactionBatchAdds() + s.mockBatchInsertBuilder.On( + "Add", s.firstTxID, s.addressToFuture[s.addresses[0]], + ).Return(nil).Once() s.mockOperationsBatchInsertBuilder.On( - "Add", s.ctx, s.firstTxID+1, s.addressToID[s.addresses[0]], + "Add", s.firstTxID+1, s.addressToFuture[s.addresses[0]], ).Return(errors.New("transient error")).Once() - s.mockOperationsBatchInsertBuilder.On( - "Add", s.ctx, s.secondTxID+1, s.addressToID[s.addresses[1]], - ).Return(nil).Maybe() - s.mockOperationsBatchInsertBuilder.On( - "Add", s.ctx, s.secondTxID+1, s.addressToID[s.addresses[2]], - ).Return(nil).Maybe() - s.mockOperationsBatchInsertBuilder.On( - "Add", s.ctx, s.thirdTxID+1, s.addressToID[s.addresses[0]], - ).Return(nil).Maybe() - - s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() - for _, tx := range s.txs { - err := s.processor.ProcessTransaction(s.ctx, tx) - s.Assert().NoError(err) - } - err := s.processor.Commit(s.ctx) - s.Assert().EqualError(err, "could not insert operation participant in db: transient error") + err := s.processor.ProcessTransaction(s.lcm, s.txs[0]) + s.Assert().EqualError(err, "transient error") } func (s *ParticipantsProcessorTestSuiteLedger) TestBatchAddExecFails() { - s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]string) - s.Assert().ElementsMatch( - s.addresses, - arg, - ) - }).Return(s.addressToID, nil).Once() - s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder", maxBatchSize). - Return(s.mockBatchInsertBuilder).Once() - s.mockSuccessfulTransactionBatchAdds() + s.mockSuccessfulOperationBatchAdds() - s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(errors.New("transient error")).Once() + s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(errors.New("transient error")).Once() for _, tx := range s.txs { - err := s.processor.ProcessTransaction(s.ctx, tx) + err := s.processor.ProcessTransaction(s.lcm, tx) s.Assert().NoError(err) } - err := s.processor.Commit(s.ctx) + err := s.processor.Flush(s.ctx, s.mockSession) s.Assert().EqualError(err, "Could not flush transaction participants to db: transient error") } -func (s *ParticipantsProcessorTestSuiteLedger) TestOpeartionBatchAddExecFails() { - s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]string) - s.Assert().ElementsMatch( - s.addresses, - arg, - ) - }).Return(s.addressToID, nil).Once() - s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder", maxBatchSize). - Return(s.mockBatchInsertBuilder).Once() - s.mockQ.On("NewOperationParticipantBatchInsertBuilder", maxBatchSize). - Return(s.mockOperationsBatchInsertBuilder).Once() - +func (s *ParticipantsProcessorTestSuiteLedger) TestOperationBatchAddExecFails() { s.mockSuccessfulTransactionBatchAdds() s.mockSuccessfulOperationBatchAdds() - s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() - s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx).Return(errors.New("transient error")).Once() + s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() + s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(errors.New("transient error")).Once() for _, tx := range s.txs { - err := s.processor.ProcessTransaction(s.ctx, tx) + err := s.processor.ProcessTransaction(s.lcm, tx) s.Assert().NoError(err) } - err := s.processor.Commit(s.ctx) - s.Assert().EqualError(err, "could not flush operation participants to db: transient error") + err := s.processor.Flush(s.ctx, s.mockSession) + s.Assert().EqualError(err, "Could not flush operation participants to db: transient error") } diff --git a/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor.go b/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor.go index 115c15f874..1937331d6a 100644 --- a/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor.go +++ b/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/stellar/go/ingest" + "github.com/stellar/go/support/db" "github.com/stellar/go/xdr" ) @@ -54,7 +55,11 @@ type StatsLedgerTransactionProcessorResults struct { OperationsRestoreFootprint int64 } -func (p *StatsLedgerTransactionProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error { +func (p *StatsLedgerTransactionProcessor) Flush(ctx context.Context, session db.SessionInterface) error { + return nil +} + +func (p *StatsLedgerTransactionProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error { p.results.Transactions++ ops := int64(len(transaction.Envelope.Operations())) p.results.Operations += ops diff --git a/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor_test.go b/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor_test.go index f2bc2a5040..c7fc6d7967 100644 --- a/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor_test.go +++ b/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor_test.go @@ -1,7 +1,6 @@ package processors import ( - "context" "testing" "github.com/stretchr/testify/assert" @@ -23,12 +22,14 @@ func TestStatsLedgerTransactionProcessoAllOpTypesCovered(t *testing.T) { }, }, } + lcm := xdr.LedgerCloseMeta{} + for typ, s := range xdr.OperationTypeToStringMap { tx := txTemplate txTemplate.Envelope.V1.Tx.Operations[0].Body.Type = xdr.OperationType(typ) f := func() { var p StatsLedgerTransactionProcessor - p.ProcessTransaction(context.Background(), tx) + p.ProcessTransaction(lcm, tx) } assert.NotPanics(t, f, s) } @@ -38,16 +39,17 @@ func TestStatsLedgerTransactionProcessoAllOpTypesCovered(t *testing.T) { txTemplate.Envelope.V1.Tx.Operations[0].Body.Type = 20000 f := func() { var p StatsLedgerTransactionProcessor - p.ProcessTransaction(context.Background(), tx) + p.ProcessTransaction(lcm, tx) } assert.Panics(t, f) } func TestStatsLedgerTransactionProcessor(t *testing.T) { processor := &StatsLedgerTransactionProcessor{} + lcm := xdr.LedgerCloseMeta{} // Successful - assert.NoError(t, processor.ProcessTransaction(context.Background(), ingest.LedgerTransaction{ + assert.NoError(t, processor.ProcessTransaction(lcm, ingest.LedgerTransaction{ Result: xdr.TransactionResultPair{ Result: xdr.TransactionResult{ Result: xdr.TransactionResultResult{ @@ -88,7 +90,7 @@ func TestStatsLedgerTransactionProcessor(t *testing.T) { })) // Failed - assert.NoError(t, processor.ProcessTransaction(context.Background(), ingest.LedgerTransaction{ + assert.NoError(t, processor.ProcessTransaction(lcm, ingest.LedgerTransaction{ Result: xdr.TransactionResultPair{ Result: xdr.TransactionResult{ Result: xdr.TransactionResultResult{ diff --git a/services/horizon/internal/ingest/processors/trades_processor.go b/services/horizon/internal/ingest/processors/trades_processor.go index b536b15bc4..2cb702e14b 100644 --- a/services/horizon/internal/ingest/processors/trades_processor.go +++ b/services/horizon/internal/ingest/processors/trades_processor.go @@ -11,6 +11,7 @@ import ( "github.com/stellar/go/exp/orderbook" "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/toid" "github.com/stellar/go/xdr" @@ -18,16 +19,25 @@ import ( // TradeProcessor operations processor type TradeProcessor struct { - tradesQ history.QTrades - ledger xdr.LedgerHeaderHistoryEntry - trades []ingestTrade - stats TradeStats + accountLoader *history.AccountLoader + lpLoader *history.LiquidityPoolLoader + assetLoader *history.AssetLoader + batch history.TradeBatchInsertBuilder + trades []ingestTrade + stats TradeStats } -func NewTradeProcessor(tradesQ history.QTrades, ledger xdr.LedgerHeaderHistoryEntry) *TradeProcessor { +func NewTradeProcessor( + accountLoader *history.AccountLoader, + lpLoader *history.LiquidityPoolLoader, + assetLoader *history.AssetLoader, + batch history.TradeBatchInsertBuilder, +) *TradeProcessor { return &TradeProcessor{ - tradesQ: tradesQ, - ledger: ledger, + accountLoader: accountLoader, + lpLoader: lpLoader, + assetLoader: assetLoader, + batch: batch, } } @@ -45,79 +55,75 @@ func (stats *TradeStats) Map() map[string]interface{} { } // ProcessTransaction process the given transaction -func (p *TradeProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) (err error) { +func (p *TradeProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) (err error) { if !transaction.Result.Successful() { return nil } - trades, err := p.extractTrades(p.ledger, transaction) + trades, err := p.extractTrades(lcm.LedgerHeaderHistoryEntry(), transaction) if err != nil { return err } + for _, trade := range trades { + if trade.buyerAccount != "" { + p.accountLoader.GetFuture(trade.buyerAccount) + } + if trade.sellerAccount != "" { + p.accountLoader.GetFuture(trade.sellerAccount) + } + if trade.liquidityPoolID != "" { + p.lpLoader.GetFuture(trade.liquidityPoolID) + } + p.assetLoader.GetFuture(history.AssetKeyFromXDR(trade.boughtAsset)) + p.assetLoader.GetFuture(history.AssetKeyFromXDR(trade.soldAsset)) + } + p.trades = append(p.trades, trades...) p.stats.count += int64(len(trades)) return nil } -func (p *TradeProcessor) Commit(ctx context.Context) error { +func (p *TradeProcessor) Flush(ctx context.Context, session db.SessionInterface) error { if len(p.trades) == 0 { return nil } - batch := p.tradesQ.NewTradeBatchInsertBuilder(maxBatchSize) - var poolIDs, accounts []string - var assets []xdr.Asset for _, trade := range p.trades { - if trade.buyerAccount != "" { - accounts = append(accounts, trade.buyerAccount) - } + row := trade.row if trade.sellerAccount != "" { - accounts = append(accounts, trade.sellerAccount) + val, err := p.accountLoader.GetNow(trade.sellerAccount) + if err != nil { + return err + } + row.BaseAccountID = null.IntFrom(val) + } + if trade.buyerAccount != "" { + val, err := p.accountLoader.GetNow(trade.buyerAccount) + if err != nil { + return err + } + row.CounterAccountID = null.IntFrom(val) } if trade.liquidityPoolID != "" { - poolIDs = append(poolIDs, trade.liquidityPoolID) + val, err := p.lpLoader.GetNow(trade.liquidityPoolID) + if err != nil { + return err + } + row.BaseLiquidityPoolID = null.IntFrom(val) } - assets = append(assets, trade.boughtAsset) - assets = append(assets, trade.soldAsset) - } - - accountSet, err := p.tradesQ.CreateAccounts(ctx, accounts, maxBatchSize) - if err != nil { - return errors.Wrap(err, "Error creating account ids") - } - - var assetMap map[string]history.Asset - assetMap, err = p.tradesQ.CreateAssets(ctx, assets, maxBatchSize) - if err != nil { - return errors.Wrap(err, "Error creating asset ids") - } - - var poolMap map[string]int64 - poolMap, err = p.tradesQ.CreateHistoryLiquidityPools(ctx, poolIDs, maxBatchSize) - if err != nil { - return errors.Wrap(err, "Error creating pool ids") - } - for _, trade := range p.trades { - row := trade.row - if id, ok := accountSet[trade.sellerAccount]; ok { - row.BaseAccountID = null.IntFrom(id) - } else if len(trade.sellerAccount) > 0 { - return errors.Errorf("Could not find history account id for %s", trade.sellerAccount) + val, err := p.assetLoader.GetNow(history.AssetKeyFromXDR(trade.soldAsset)) + if err != nil { + return err } - if id, ok := accountSet[trade.buyerAccount]; ok { - row.CounterAccountID = null.IntFrom(id) - } else if len(trade.buyerAccount) > 0 { - return errors.Errorf("Could not find history account id for %s", trade.buyerAccount) - } - if id, ok := poolMap[trade.liquidityPoolID]; ok { - row.BaseLiquidityPoolID = null.IntFrom(id) - } else if len(trade.liquidityPoolID) > 0 { - return errors.Errorf("Could not find history liquidity pool id for %s", trade.liquidityPoolID) + row.BaseAssetID = val + + val, err = p.assetLoader.GetNow(history.AssetKeyFromXDR(trade.boughtAsset)) + if err != nil { + return err } - row.BaseAssetID = assetMap[trade.soldAsset.String()].ID - row.CounterAssetID = assetMap[trade.boughtAsset.String()].ID + row.CounterAssetID = val if row.BaseAssetID > row.CounterAssetID { row.BaseIsSeller = false @@ -133,12 +139,12 @@ func (p *TradeProcessor) Commit(ctx context.Context) error { } } - if err = batch.Add(ctx, row); err != nil { + if err := p.batch.Add(row); err != nil { return errors.Wrap(err, "Error adding trade to batch") } } - if err = batch.Exec(ctx); err != nil { + if err := p.batch.Exec(ctx, session); err != nil { return errors.Wrap(err, "Error flushing operation batch") } return nil diff --git a/services/horizon/internal/ingest/processors/trades_processor_test.go b/services/horizon/internal/ingest/processors/trades_processor_test.go index e11fdf370f..5b2a2f20e3 100644 --- a/services/horizon/internal/ingest/processors/trades_processor_test.go +++ b/services/horizon/internal/ingest/processors/trades_processor_test.go @@ -10,18 +10,23 @@ import ( "github.com/guregu/null" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/suite" + "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" + "github.com/stellar/go/support/db" "github.com/stellar/go/toid" "github.com/stellar/go/xdr" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/suite" ) type TradeProcessorTestSuiteLedger struct { suite.Suite processor *TradeProcessor - mockQ *history.MockQTrades + mockSession *db.MockSession + accountLoader history.AccountLoaderStub + lpLoader history.LiquidityPoolLoaderStub + assetLoader history.AssetLoaderStub mockBatchInsertBuilder *history.MockTradeBatchInsertBuilder unmuxedSourceAccount xdr.AccountId @@ -43,9 +48,10 @@ type TradeProcessorTestSuiteLedger struct { lpToID map[xdr.PoolId]int64 unmuxedAccountToID map[string]int64 - assetToID map[string]history.Asset + assetToID map[history.AssetKey]history.Asset txs []ingest.LedgerTransaction + lcm xdr.LedgerCloseMeta } func TestTradeProcessorTestSuiteLedger(t *testing.T) { @@ -53,7 +59,6 @@ func TestTradeProcessorTestSuiteLedger(t *testing.T) { } func (s *TradeProcessorTestSuiteLedger) SetupTest() { - s.mockQ = &history.MockQTrades{} s.mockBatchInsertBuilder = &history.MockTradeBatchInsertBuilder{} s.unmuxedSourceAccount = xdr.MustAddress("GAUJETIZVEP2NRYLUESJ3LS66NVCEGMON4UDCBCSBEVPIID773P2W6AY") @@ -163,7 +168,7 @@ func (s *TradeProcessorTestSuiteLedger) SetupTest() { s.unmuxedSourceAccount.Address(): 1000, s.unmuxedOpSourceAccount.Address(): 1001, } - s.assetToID = map[string]history.Asset{} + s.assetToID = map[history.AssetKey]history.Asset{} s.allTrades = []xdr.ClaimAtom{ s.strictReceiveTrade, s.strictSendTrade, @@ -188,42 +193,51 @@ func (s *TradeProcessorTestSuiteLedger) SetupTest() { s.sellPrices = append(s.sellPrices, xdr.Price{N: xdr.Int32(trade.AmountBought()), D: xdr.Int32(trade.AmountSold())}) } if i%2 == 0 { - s.assetToID[trade.AssetSold().String()] = history.Asset{ID: int64(10000 + i)} - s.assetToID[trade.AssetBought().String()] = history.Asset{ID: int64(100 + i)} + s.assetToID[history.AssetKeyFromXDR(trade.AssetSold())] = history.Asset{ID: int64(10000 + i)} + s.assetToID[history.AssetKeyFromXDR(trade.AssetBought())] = history.Asset{ID: int64(100 + i)} } else { - s.assetToID[trade.AssetSold().String()] = history.Asset{ID: int64(100 + i)} - s.assetToID[trade.AssetBought().String()] = history.Asset{ID: int64(10000 + i)} + s.assetToID[history.AssetKeyFromXDR(trade.AssetSold())] = history.Asset{ID: int64(100 + i)} + s.assetToID[history.AssetKeyFromXDR(trade.AssetBought())] = history.Asset{ID: int64(10000 + i)} } s.assets = append(s.assets, trade.AssetSold(), trade.AssetBought()) } - s.processor = NewTradeProcessor( - s.mockQ, - xdr.LedgerHeaderHistoryEntry{ - Header: xdr.LedgerHeader{ - LedgerSeq: 100, + s.lcm = xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(100), + }, }, }, + } + + s.accountLoader = history.NewAccountLoaderStub() + s.assetLoader = history.NewAssetLoaderStub() + s.lpLoader = history.NewLiquidityPoolLoaderStub() + s.processor = NewTradeProcessor( + s.accountLoader.Loader, + s.lpLoader.Loader, + s.assetLoader.Loader, + s.mockBatchInsertBuilder, ) } func (s *TradeProcessorTestSuiteLedger) TearDownTest() { - s.mockQ.AssertExpectations(s.T()) s.mockBatchInsertBuilder.AssertExpectations(s.T()) } func (s *TradeProcessorTestSuiteLedger) TestIgnoreFailedTransactions() { ctx := context.Background() - err := s.processor.ProcessTransaction(ctx, createTransaction(false, 1)) + err := s.processor.ProcessTransaction(s.lcm, createTransaction(false, 1)) s.Assert().NoError(err) - err = s.processor.Commit(ctx) + err = s.processor.Flush(ctx, s.mockSession) s.Assert().NoError(err) } -func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions( - ledger xdr.LedgerHeaderHistoryEntry, -) []history.InsertTrade { +func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions() []history.InsertTrade { + ledger := s.lcm.LedgerHeaderHistoryEntry() closeTime := time.Unix(int64(ledger.Header.ScpValue.CloseTime), 0).UTC() inserts := []history.InsertTrade{ { @@ -232,11 +246,11 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions( LedgerCloseTime: closeTime, BaseAmount: int64(s.strictReceiveTrade.AmountBought()), BaseAccountID: null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]), - BaseAssetID: s.assetToID[s.strictReceiveTrade.AssetBought().String()].ID, + BaseAssetID: s.assetToID[history.AssetKeyFromXDR(s.strictReceiveTrade.AssetBought())].ID, BaseOfferID: null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 2).ToInt64()), TOIDType)), CounterAmount: int64(s.strictReceiveTrade.AmountSold()), CounterAccountID: null.IntFrom(s.unmuxedAccountToID[s.strictReceiveTrade.SellerId().Address()]), - CounterAssetID: s.assetToID[s.strictReceiveTrade.AssetSold().String()].ID, + CounterAssetID: s.assetToID[history.AssetKeyFromXDR(s.strictReceiveTrade.AssetSold())].ID, CounterOfferID: null.IntFrom(int64(s.strictReceiveTrade.OfferId())), BaseIsSeller: false, BaseIsExact: null.BoolFrom(false), @@ -250,11 +264,11 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions( LedgerCloseTime: closeTime, CounterAmount: int64(s.strictSendTrade.AmountBought()), CounterAccountID: null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]), - CounterAssetID: s.assetToID[s.strictSendTrade.AssetBought().String()].ID, + CounterAssetID: s.assetToID[history.AssetKeyFromXDR(s.strictSendTrade.AssetBought())].ID, CounterOfferID: null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 3).ToInt64()), TOIDType)), BaseAmount: int64(s.strictSendTrade.AmountSold()), BaseAccountID: null.IntFrom(s.unmuxedAccountToID[s.strictSendTrade.SellerId().Address()]), - BaseAssetID: s.assetToID[s.strictSendTrade.AssetSold().String()].ID, + BaseAssetID: s.assetToID[history.AssetKeyFromXDR(s.strictSendTrade.AssetSold())].ID, BaseIsSeller: true, BaseIsExact: null.BoolFrom(false), BaseOfferID: null.IntFrom(int64(s.strictSendTrade.OfferId())), @@ -269,10 +283,10 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions( BaseOfferID: null.IntFrom(879136), BaseAmount: int64(s.buyOfferTrade.AmountBought()), BaseAccountID: null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]), - BaseAssetID: s.assetToID[s.buyOfferTrade.AssetBought().String()].ID, + BaseAssetID: s.assetToID[history.AssetKeyFromXDR(s.buyOfferTrade.AssetBought())].ID, CounterAmount: int64(s.buyOfferTrade.AmountSold()), CounterAccountID: null.IntFrom(s.unmuxedAccountToID[s.buyOfferTrade.SellerId().Address()]), - CounterAssetID: s.assetToID[s.buyOfferTrade.AssetSold().String()].ID, + CounterAssetID: s.assetToID[history.AssetKeyFromXDR(s.buyOfferTrade.AssetSold())].ID, BaseIsSeller: false, CounterOfferID: null.IntFrom(int64(s.buyOfferTrade.OfferId())), PriceN: int64(s.sellPrices[2].D), @@ -284,12 +298,12 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions( Order: 2, LedgerCloseTime: closeTime, CounterAmount: int64(s.sellOfferTrade.AmountBought()), - CounterAssetID: s.assetToID[s.sellOfferTrade.AssetBought().String()].ID, + CounterAssetID: s.assetToID[history.AssetKeyFromXDR(s.sellOfferTrade.AssetBought())].ID, CounterAccountID: null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]), CounterOfferID: null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 5).ToInt64()), TOIDType)), BaseAmount: int64(s.sellOfferTrade.AmountSold()), BaseAccountID: null.IntFrom(s.unmuxedAccountToID[s.sellOfferTrade.SellerId().Address()]), - BaseAssetID: s.assetToID[s.sellOfferTrade.AssetSold().String()].ID, + BaseAssetID: s.assetToID[history.AssetKeyFromXDR(s.sellOfferTrade.AssetSold())].ID, BaseIsSeller: true, BaseOfferID: null.IntFrom(int64(s.sellOfferTrade.OfferId())), PriceN: int64(s.sellPrices[3].N), @@ -301,12 +315,12 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions( Order: 0, LedgerCloseTime: closeTime, BaseAmount: int64(s.passiveSellOfferTrade.AmountBought()), - BaseAssetID: s.assetToID[s.passiveSellOfferTrade.AssetBought().String()].ID, + BaseAssetID: s.assetToID[history.AssetKeyFromXDR(s.passiveSellOfferTrade.AssetBought())].ID, BaseAccountID: null.IntFrom(s.unmuxedAccountToID[s.unmuxedSourceAccount.Address()]), BaseOfferID: null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 6).ToInt64()), TOIDType)), CounterAmount: int64(s.passiveSellOfferTrade.AmountSold()), CounterAccountID: null.IntFrom(s.unmuxedAccountToID[s.passiveSellOfferTrade.SellerId().Address()]), - CounterAssetID: s.assetToID[s.passiveSellOfferTrade.AssetSold().String()].ID, + CounterAssetID: s.assetToID[history.AssetKeyFromXDR(s.passiveSellOfferTrade.AssetSold())].ID, BaseIsSeller: false, CounterOfferID: null.IntFrom(int64(s.passiveSellOfferTrade.OfferId())), PriceN: int64(s.sellPrices[4].D), @@ -320,12 +334,12 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions( LedgerCloseTime: closeTime, CounterAmount: int64(s.otherPassiveSellOfferTrade.AmountBought()), - CounterAssetID: s.assetToID[s.otherPassiveSellOfferTrade.AssetBought().String()].ID, + CounterAssetID: s.assetToID[history.AssetKeyFromXDR(s.otherPassiveSellOfferTrade.AssetBought())].ID, CounterAccountID: null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]), CounterOfferID: null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 7).ToInt64()), TOIDType)), BaseAmount: int64(s.otherPassiveSellOfferTrade.AmountSold()), BaseAccountID: null.IntFrom(s.unmuxedAccountToID[s.otherPassiveSellOfferTrade.SellerId().Address()]), - BaseAssetID: s.assetToID[s.otherPassiveSellOfferTrade.AssetSold().String()].ID, + BaseAssetID: s.assetToID[history.AssetKeyFromXDR(s.otherPassiveSellOfferTrade.AssetSold())].ID, BaseIsSeller: true, BaseOfferID: null.IntFrom(int64(s.otherPassiveSellOfferTrade.OfferId())), PriceN: int64(s.sellPrices[5].N), @@ -337,12 +351,12 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions( Order: 1, LedgerCloseTime: closeTime, BaseAmount: int64(s.strictReceiveTradeLP.AmountBought()), - BaseAssetID: s.assetToID[s.strictReceiveTradeLP.AssetBought().String()].ID, + BaseAssetID: s.assetToID[history.AssetKeyFromXDR(s.strictReceiveTradeLP.AssetBought())].ID, BaseAccountID: null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]), BaseOfferID: null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 8).ToInt64()), TOIDType)), CounterAmount: int64(s.strictReceiveTradeLP.AmountSold()), CounterLiquidityPoolID: null.IntFrom(s.lpToID[s.strictReceiveTradeLP.MustLiquidityPool().LiquidityPoolId]), - CounterAssetID: s.assetToID[s.strictReceiveTradeLP.AssetSold().String()].ID, + CounterAssetID: s.assetToID[history.AssetKeyFromXDR(s.strictReceiveTradeLP.AssetSold())].ID, BaseIsSeller: false, BaseIsExact: null.BoolFrom(false), LiquidityPoolFee: null.IntFrom(int64(xdr.LiquidityPoolFeeV18)), @@ -356,12 +370,12 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions( Order: 0, LedgerCloseTime: closeTime, CounterAmount: int64(s.strictSendTradeLP.AmountBought()), - CounterAssetID: s.assetToID[s.strictSendTradeLP.AssetBought().String()].ID, + CounterAssetID: s.assetToID[history.AssetKeyFromXDR(s.strictSendTradeLP.AssetBought())].ID, CounterAccountID: null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]), CounterOfferID: null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 9).ToInt64()), TOIDType)), BaseAmount: int64(s.strictSendTradeLP.AmountSold()), BaseLiquidityPoolID: null.IntFrom(s.lpToID[s.strictSendTradeLP.MustLiquidityPool().LiquidityPoolId]), - BaseAssetID: s.assetToID[s.strictSendTradeLP.AssetSold().String()].ID, + BaseAssetID: s.assetToID[history.AssetKeyFromXDR(s.strictSendTradeLP.AssetSold())].ID, BaseIsSeller: true, BaseIsExact: null.BoolFrom(false), LiquidityPoolFee: null.IntFrom(int64(xdr.LiquidityPoolFeeV18)), @@ -709,243 +723,75 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions( tx, } - s.mockQ.On("NewTradeBatchInsertBuilder", maxBatchSize). - Return(s.mockBatchInsertBuilder).Once() - return inserts } -func mapKeysToList(set map[string]int64) []string { - keys := make([]string, 0, len(set)) - for key := range set { - keys = append(keys, key) +func (s *TradeProcessorTestSuiteLedger) stubLoaders() { + for key, id := range s.unmuxedAccountToID { + s.accountLoader.Insert(key, id) } - return keys -} - -func uniq(list []string) []string { - var deduped []string - set := map[string]bool{} - for _, s := range list { - if set[s] { - continue - } - deduped = append(deduped, s) - set[s] = true + for key, id := range s.assetToID { + s.assetLoader.Insert(key, id.ID) + } + for key, id := range s.lpToID { + s.lpLoader.Insert(PoolIDToString(key), id) } - return deduped } func (s *TradeProcessorTestSuiteLedger) TestIngestTradesSucceeds() { ctx := context.Background() - inserts := s.mockReadTradeTransactions(s.processor.ledger) - - s.mockCreateAccounts(ctx) - - s.mockCreateAssets(ctx) + inserts := s.mockReadTradeTransactions() - s.mockCreateHistoryLiquidityPools(ctx) + for _, tx := range s.txs { + err := s.processor.ProcessTransaction(s.lcm, tx) + s.Assert().NoError(err) + } for _, insert := range inserts { - s.mockBatchInsertBuilder.On("Add", ctx, []history.InsertTrade{ + s.mockBatchInsertBuilder.On("Add", []history.InsertTrade{ insert, }).Return(nil).Once() } + s.mockBatchInsertBuilder.On("Exec", ctx, s.mockSession).Return(nil).Once() + s.stubLoaders() - s.mockBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() - - for _, tx := range s.txs { - err := s.processor.ProcessTransaction(ctx, tx) - s.Assert().NoError(err) - } - - err := s.processor.Commit(ctx) + err := s.processor.Flush(ctx, s.mockSession) s.Assert().NoError(err) } -func (s *TradeProcessorTestSuiteLedger) mockCreateHistoryLiquidityPools(ctx context.Context) { - lpIDs, lpStrToID := s.extractLpIDs() - s.mockQ.On("CreateHistoryLiquidityPools", ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]string) - s.Assert().ElementsMatch( - lpIDs, - arg, - ) - }).Return(lpStrToID, nil).Once() -} - -func (s *TradeProcessorTestSuiteLedger) extractLpIDs() ([]string, map[string]int64) { - var lpIDs []string - lpStrToID := map[string]int64{} - for lpID, id := range s.lpToID { - lpIDStr := PoolIDToString(lpID) - lpIDs = append(lpIDs, lpIDStr) - lpStrToID[lpIDStr] = id - } - return lpIDs, lpStrToID -} - -func (s *TradeProcessorTestSuiteLedger) TestCreateAccountsError() { - ctx := context.Background() - s.mockReadTradeTransactions(s.processor.ledger) - - s.mockQ.On("CreateAccounts", ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]string) - s.Assert().ElementsMatch( - mapKeysToList(s.unmuxedAccountToID), - uniq(arg), - ) - }).Return(map[string]int64{}, fmt.Errorf("create accounts error")).Once() - - for _, tx := range s.txs { - err := s.processor.ProcessTransaction(ctx, tx) - s.Assert().NoError(err) - } - - err := s.processor.Commit(ctx) - - s.Assert().EqualError(err, "Error creating account ids: create accounts error") -} - -func (s *TradeProcessorTestSuiteLedger) TestCreateAssetsError() { - ctx := context.Background() - s.mockReadTradeTransactions(s.processor.ledger) - - s.mockCreateAccounts(ctx) - - s.mockQ.On("CreateAssets", ctx, mock.AnythingOfType("[]xdr.Asset"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]xdr.Asset) - s.Assert().ElementsMatch( - s.assets, - arg, - ) - }).Return(s.assetToID, fmt.Errorf("create assets error")).Once() - - for _, tx := range s.txs { - err := s.processor.ProcessTransaction(ctx, tx) - s.Assert().NoError(err) - } - - err := s.processor.Commit(ctx) - s.Assert().EqualError(err, "Error creating asset ids: create assets error") -} - -func (s *TradeProcessorTestSuiteLedger) TestCreateHistoryLiquidityPoolsError() { +func (s *TradeProcessorTestSuiteLedger) TestBatchAddError() { ctx := context.Background() - s.mockReadTradeTransactions(s.processor.ledger) - - s.mockCreateAccounts(ctx) - - s.mockCreateAssets(ctx) - - lpIDs, lpStrToID := s.extractLpIDs() - s.mockQ.On("CreateHistoryLiquidityPools", ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]string) - s.Assert().ElementsMatch( - lpIDs, - arg, - ) - }).Return(lpStrToID, fmt.Errorf("create liqudity pool id error")).Once() + s.mockReadTradeTransactions() for _, tx := range s.txs { - err := s.processor.ProcessTransaction(ctx, tx) + err := s.processor.ProcessTransaction(s.lcm, tx) s.Assert().NoError(err) } - err := s.processor.Commit(ctx) - s.Assert().EqualError(err, "Error creating pool ids: create liqudity pool id error") -} - -func (s *TradeProcessorTestSuiteLedger) mockCreateAssets(ctx context.Context) { - s.mockQ.On("CreateAssets", ctx, mock.AnythingOfType("[]xdr.Asset"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]xdr.Asset) - s.Assert().ElementsMatch( - s.assets, - arg, - ) - }).Return(s.assetToID, nil).Once() -} - -func (s *TradeProcessorTestSuiteLedger) mockCreateAccounts(ctx context.Context) { - s.mockQ.On("CreateAccounts", ctx, mock.AnythingOfType("[]string"), maxBatchSize). - Run(func(args mock.Arguments) { - arg := args.Get(1).([]string) - s.Assert().ElementsMatch( - mapKeysToList(s.unmuxedAccountToID), - uniq(arg), - ) - }).Return(s.unmuxedAccountToID, nil).Once() -} - -func (s *TradeProcessorTestSuiteLedger) TestBatchAddError() { - ctx := context.Background() - s.mockReadTradeTransactions(s.processor.ledger) - - s.mockCreateAccounts(ctx) - - s.mockCreateAssets(ctx) - - s.mockCreateHistoryLiquidityPools(ctx) - - s.mockBatchInsertBuilder.On("Add", ctx, mock.AnythingOfType("[]history.InsertTrade")). + s.stubLoaders() + s.mockBatchInsertBuilder.On("Add", mock.AnythingOfType("[]history.InsertTrade")). Return(fmt.Errorf("batch add error")).Once() - for _, tx := range s.txs { - err := s.processor.ProcessTransaction(ctx, tx) - s.Assert().NoError(err) - } - - err := s.processor.Commit(ctx) + err := s.processor.Flush(ctx, s.mockSession) s.Assert().EqualError(err, "Error adding trade to batch: batch add error") } func (s *TradeProcessorTestSuiteLedger) TestBatchExecError() { ctx := context.Background() - insert := s.mockReadTradeTransactions(s.processor.ledger) - - s.mockCreateAccounts(ctx) - - s.mockCreateAssets(ctx) + insert := s.mockReadTradeTransactions() - s.mockCreateHistoryLiquidityPools(ctx) - - s.mockBatchInsertBuilder.On("Add", ctx, mock.AnythingOfType("[]history.InsertTrade")). - Return(nil).Times(len(insert)) - s.mockBatchInsertBuilder.On("Exec", ctx).Return(fmt.Errorf("exec error")).Once() for _, tx := range s.txs { - err := s.processor.ProcessTransaction(ctx, tx) + err := s.processor.ProcessTransaction(s.lcm, tx) s.Assert().NoError(err) } - err := s.processor.Commit(ctx) - s.Assert().EqualError(err, "Error flushing operation batch: exec error") -} - -func (s *TradeProcessorTestSuiteLedger) TestIgnoreCheckIfSmallLedger() { - ctx := context.Background() - insert := s.mockReadTradeTransactions(s.processor.ledger) - - s.mockCreateAccounts(ctx) - - s.mockCreateAssets(ctx) - - s.mockCreateHistoryLiquidityPools(ctx) - s.mockBatchInsertBuilder.On("Add", ctx, mock.AnythingOfType("[]history.InsertTrade")). + s.stubLoaders() + s.mockBatchInsertBuilder.On("Add", mock.AnythingOfType("[]history.InsertTrade")). Return(nil).Times(len(insert)) - s.mockBatchInsertBuilder.On("Exec", ctx).Return(nil).Once() - - for _, tx := range s.txs { - err := s.processor.ProcessTransaction(ctx, tx) - s.Assert().NoError(err) - } + s.mockBatchInsertBuilder.On("Exec", ctx, s.mockSession).Return(fmt.Errorf("exec error")).Once() - err := s.processor.Commit(ctx) - s.Assert().NoError(err) + err := s.processor.Flush(ctx, s.mockSession) + s.Assert().EqualError(err, "Error flushing operation batch: exec error") } func TestTradeProcessor_ProcessTransaction_MuxedAccount(t *testing.T) { @@ -973,7 +819,7 @@ func TestTradeProcessor_RoundingSlippage_Big(t *testing.T) { s := &TradeProcessorTestSuiteLedger{} s.SetT(t) s.SetupTest() - s.mockReadTradeTransactions(s.processor.ledger) + s.mockReadTradeTransactions() assetDeposited := xdr.MustNewCreditAsset("MAD", s.unmuxedSourceAccount.Address()) assetDisbursed := xdr.MustNewCreditAsset("GRE", s.unmuxedSourceAccount.Address()) @@ -1005,7 +851,7 @@ func TestTradeProcessor_RoundingSlippage_Small(t *testing.T) { s := &TradeProcessorTestSuiteLedger{} s.SetT(t) s.SetupTest() - s.mockReadTradeTransactions(s.processor.ledger) + s.mockReadTradeTransactions() assetDeposited := xdr.MustNewCreditAsset("MAD", s.unmuxedSourceAccount.Address()) assetDisbursed := xdr.MustNewCreditAsset("GRE", s.unmuxedSourceAccount.Address()) diff --git a/services/horizon/internal/ingest/processors/transactions_processor.go b/services/horizon/internal/ingest/processors/transactions_processor.go index e2a880f296..871c72624a 100644 --- a/services/horizon/internal/ingest/processors/transactions_processor.go +++ b/services/horizon/internal/ingest/processors/transactions_processor.go @@ -5,43 +5,35 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/services/horizon/internal/db2/history" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" + "github.com/stellar/go/xdr" ) type TransactionProcessor struct { - transactionsQ history.QTransactions - sequence uint32 - batch history.TransactionBatchInsertBuilder + batch history.TransactionBatchInsertBuilder } -func NewTransactionFilteredTmpProcessor(transactionsQ history.QTransactions, sequence uint32) *TransactionProcessor { +func NewTransactionFilteredTmpProcessor(batch history.TransactionBatchInsertBuilder) *TransactionProcessor { return &TransactionProcessor{ - transactionsQ: transactionsQ, - sequence: sequence, - batch: transactionsQ.NewTransactionFilteredTmpBatchInsertBuilder(maxBatchSize), + batch: batch, } } -func NewTransactionProcessor(transactionsQ history.QTransactions, sequence uint32) *TransactionProcessor { +func NewTransactionProcessor(batch history.TransactionBatchInsertBuilder) *TransactionProcessor { return &TransactionProcessor{ - transactionsQ: transactionsQ, - sequence: sequence, - batch: transactionsQ.NewTransactionBatchInsertBuilder(maxBatchSize), + batch: batch, } } -func (p *TransactionProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error { - if err := p.batch.Add(ctx, transaction, p.sequence); err != nil { +func (p *TransactionProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error { + if err := p.batch.Add(transaction, lcm.LedgerSequence()); err != nil { return errors.Wrap(err, "Error batch inserting transaction rows") } return nil } -func (p *TransactionProcessor) Commit(ctx context.Context) error { - if err := p.batch.Exec(ctx); err != nil { - return errors.Wrap(err, "Error flushing transaction batch") - } - - return nil +func (p *TransactionProcessor) Flush(ctx context.Context, session db.SessionInterface) error { + return p.batch.Exec(ctx, session) } diff --git a/services/horizon/internal/ingest/processors/transactions_processor_test.go b/services/horizon/internal/ingest/processors/transactions_processor_test.go index ec1cf105e5..987e8ce6f9 100644 --- a/services/horizon/internal/ingest/processors/transactions_processor_test.go +++ b/services/horizon/internal/ingest/processors/transactions_processor_test.go @@ -7,7 +7,10 @@ import ( "testing" "github.com/stellar/go/services/horizon/internal/db2/history" + "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" + "github.com/stellar/go/xdr" + "github.com/stretchr/testify/suite" ) @@ -15,7 +18,7 @@ type TransactionsProcessorTestSuiteLedger struct { suite.Suite ctx context.Context processor *TransactionProcessor - mockQ *history.MockQTransactions + mockSession *db.MockSession mockBatchInsertBuilder *history.MockTransactionsBatchInsertBuilder } @@ -25,66 +28,81 @@ func TestTransactionsProcessorTestSuiteLedger(t *testing.T) { func (s *TransactionsProcessorTestSuiteLedger) SetupTest() { s.ctx = context.Background() - s.mockQ = &history.MockQTransactions{} s.mockBatchInsertBuilder = &history.MockTransactionsBatchInsertBuilder{} - - s.mockQ. - On("NewTransactionBatchInsertBuilder", maxBatchSize). - Return(s.mockBatchInsertBuilder).Once() - - s.processor = NewTransactionProcessor(s.mockQ, 20) + s.processor = NewTransactionProcessor(s.mockBatchInsertBuilder) } func (s *TransactionsProcessorTestSuiteLedger) TearDownTest() { - s.mockQ.AssertExpectations(s.T()) s.mockBatchInsertBuilder.AssertExpectations(s.T()) } func (s *TransactionsProcessorTestSuiteLedger) TestAddTransactionsSucceeds() { sequence := uint32(20) - + lcm := xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(sequence), + }, + }, + }, + } firstTx := createTransaction(true, 1) secondTx := createTransaction(false, 3) thirdTx := createTransaction(true, 4) - s.mockBatchInsertBuilder.On("Add", s.ctx, firstTx, sequence).Return(nil).Once() - s.mockBatchInsertBuilder.On("Add", s.ctx, secondTx, sequence).Return(nil).Once() - s.mockBatchInsertBuilder.On("Add", s.ctx, thirdTx, sequence).Return(nil).Once() - s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once() - s.Assert().NoError(s.processor.Commit(s.ctx)) - - err := s.processor.ProcessTransaction(s.ctx, firstTx) - s.Assert().NoError(err) + s.mockBatchInsertBuilder.On("Add", firstTx, sequence).Return(nil).Once() + s.mockBatchInsertBuilder.On("Add", secondTx, sequence).Return(nil).Once() + s.mockBatchInsertBuilder.On("Add", thirdTx, sequence+1).Return(nil).Once() + s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once() - err = s.processor.ProcessTransaction(s.ctx, secondTx) - s.Assert().NoError(err) + s.Assert().NoError(s.processor.ProcessTransaction(lcm, firstTx)) + s.Assert().NoError(s.processor.ProcessTransaction(lcm, secondTx)) + lcm.V0.LedgerHeader.Header.LedgerSeq++ + s.Assert().NoError(s.processor.ProcessTransaction(lcm, thirdTx)) - err = s.processor.ProcessTransaction(s.ctx, thirdTx) - s.Assert().NoError(err) + s.Assert().NoError(s.processor.Flush(s.ctx, s.mockSession)) } func (s *TransactionsProcessorTestSuiteLedger) TestAddTransactionsFails() { sequence := uint32(20) + lcm := xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(sequence), + }, + }, + }, + } firstTx := createTransaction(true, 1) - s.mockBatchInsertBuilder.On("Add", s.ctx, firstTx, sequence). + s.mockBatchInsertBuilder.On("Add", firstTx, sequence). Return(errors.New("transient error")).Once() - err := s.processor.ProcessTransaction(s.ctx, firstTx) + err := s.processor.ProcessTransaction(lcm, firstTx) s.Assert().Error(err) s.Assert().EqualError(err, "Error batch inserting transaction rows: transient error") } func (s *TransactionsProcessorTestSuiteLedger) TestExecFails() { sequence := uint32(20) + lcm := xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(sequence), + }, + }, + }, + } firstTx := createTransaction(true, 1) - s.mockBatchInsertBuilder.On("Add", s.ctx, firstTx, sequence).Return(nil).Once() - s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(errors.New("transient error")).Once() + s.mockBatchInsertBuilder.On("Add", firstTx, sequence).Return(nil).Once() + s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(errors.New("transient error")).Once() - err := s.processor.ProcessTransaction(s.ctx, firstTx) - s.Assert().NoError(err) + s.Assert().NoError(s.processor.ProcessTransaction(lcm, firstTx)) - err = s.processor.Commit(s.ctx) + err := s.processor.Flush(s.ctx, s.mockSession) s.Assert().Error(err) - s.Assert().EqualError(err, "Error flushing transaction batch: transient error") + s.Assert().EqualError(err, "transient error") } diff --git a/services/horizon/internal/integration/trade_aggregations_test.go b/services/horizon/internal/integration/trade_aggregations_test.go index 43248bed73..ff3837444f 100644 --- a/services/horizon/internal/integration/trade_aggregations_test.go +++ b/services/horizon/internal/integration/trade_aggregations_test.go @@ -277,9 +277,9 @@ func TestTradeAggregations(t *testing.T) { assert.NoError(t, historyQ.Rollback()) }() - batch := historyQ.NewTradeBatchInsertBuilder(1000) - batch.Add(ctx, scenario.trades...) - assert.NoError(t, batch.Exec(ctx)) + batch := historyQ.NewTradeBatchInsertBuilder() + assert.NoError(t, batch.Add(scenario.trades...)) + assert.NoError(t, batch.Exec(ctx, historyQ)) // Rebuild the aggregates. for _, trade := range scenario.trades { diff --git a/services/horizon/internal/middleware_test.go b/services/horizon/internal/middleware_test.go index 08b90465f3..40a74ea69e 100644 --- a/services/horizon/internal/middleware_test.go +++ b/services/horizon/internal/middleware_test.go @@ -284,7 +284,10 @@ func TestStateMiddleware(t *testing.T) { t.Run(testCase.name, func(t *testing.T) { stateMiddleware.NoStateVerification = testCase.noStateVerification tt.Assert.NoError(q.UpdateExpStateInvalid(context.Background(), testCase.stateInvalid)) - _, err = q.InsertLedger(context.Background(), xdr.LedgerHeaderHistoryEntry{ + + tt.Assert.NoError(q.Begin(tt.Ctx)) + ledgerBatch := q.NewLedgerBatchInsertBuilder() + err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{ Hash: xdr.Hash{byte(i)}, Header: xdr.LedgerHeader{ LedgerSeq: testCase.latestHistoryLedger, @@ -292,6 +295,9 @@ func TestStateMiddleware(t *testing.T) { }, }, 0, 0, 0, 0, 0) tt.Assert.NoError(err) + tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q)) + tt.Assert.NoError(q.Commit()) + tt.Assert.NoError(q.UpdateLastLedgerIngest(context.Background(), testCase.lastIngestedLedger)) tt.Assert.NoError(q.UpdateIngestVersion(context.Background(), testCase.ingestionVersion)) diff --git a/support/db/batch_insert_builder_test.go b/support/db/batch_insert_builder_test.go index e283e8bf57..a7a772757f 100644 --- a/support/db/batch_insert_builder_test.go +++ b/support/db/batch_insert_builder_test.go @@ -13,6 +13,7 @@ import ( type hungerRow struct { Name string `db:"name"` HungerLevel string `db:"hunger_level"` + JsonValue string `db:"json_value"` } type invalidHungerRow struct { diff --git a/support/db/fast_batch_insert_builder.go b/support/db/fast_batch_insert_builder.go new file mode 100644 index 0000000000..ec235ee31d --- /dev/null +++ b/support/db/fast_batch_insert_builder.go @@ -0,0 +1,150 @@ +package db + +import ( + "context" + "reflect" + "sort" + + "github.com/lib/pq" + + "github.com/stellar/go/support/errors" +) + +// ErrSealed is returned when trying to add rows to the FastBatchInsertBuilder after Exec() is called. +// Once Exec() is called no more rows can be added to the FastBatchInsertBuilder unless you call Reset() +// which clears out the old rows from the FastBatchInsertBuilder. +var ErrSealed = errors.New("cannot add more rows after Exec() without calling Reset() first") + +// ErrNoTx is returned when Exec() is called outside of a transaction. +var ErrNoTx = errors.New("cannot call Exec() outside of a transaction") + +// FastBatchInsertBuilder works like sq.InsertBuilder but has a better support for batching +// large number of rows. +// It is NOT safe for concurrent use. +// It does NOT support updating existing rows. +type FastBatchInsertBuilder struct { + columns []string + rows [][]interface{} + rowStructType reflect.Type + sealed bool +} + +// Row adds a new row to the batch. All rows must have exactly the same columns +// (map keys). Otherwise, error will be returned. Please note that rows are not +// added one by one but in batches when `Exec` is called. +func (b *FastBatchInsertBuilder) Row(row map[string]interface{}) error { + if b.sealed { + return ErrSealed + } + + if b.columns == nil { + b.columns = make([]string, 0, len(row)) + b.rows = make([][]interface{}, 0) + + for column := range row { + b.columns = append(b.columns, column) + } + + sort.Strings(b.columns) + } + + if len(b.columns) != len(row) { + return errors.Errorf("invalid number of columns (expected=%d, actual=%d)", len(b.columns), len(row)) + } + + rowSlice := make([]interface{}, 0, len(b.columns)) + for _, column := range b.columns { + val, ok := row[column] + if !ok { + return errors.Errorf(`column "%s" does not exist`, column) + } + rowSlice = append(rowSlice, val) + } + b.rows = append(b.rows, rowSlice) + + return nil +} + +// RowStruct adds a new row to the batch. All rows must have exactly the same columns +// (map keys). Otherwise, error will be returned. Please note that rows are not +// added one by one but in batches when `Exec` is called. +func (b *FastBatchInsertBuilder) RowStruct(row interface{}) error { + if b.sealed { + return ErrSealed + } + + if b.columns == nil { + b.columns = ColumnsForStruct(row) + b.rows = make([][]interface{}, 0) + } + + rowType := reflect.TypeOf(row) + if b.rowStructType == nil { + b.rowStructType = rowType + } else if b.rowStructType != rowType { + return errors.Errorf(`expected value of type "%s" but got "%s" value`, b.rowStructType.String(), rowType.String()) + } + + rrow := reflect.ValueOf(row) + rvals := mapper.FieldsByName(rrow, b.columns) + + // convert fields values to interface{} + columnValues := make([]interface{}, len(b.columns)) + for i, rval := range rvals { + columnValues[i] = rval.Interface() + } + + b.rows = append(b.rows, columnValues) + + return nil +} + +// Len returns the number of rows held in memory by the FastBatchInsertBuilder. +func (b *FastBatchInsertBuilder) Len() int { + return len(b.rows) +} + +// Exec inserts rows in a single COPY statement. Once Exec is called no more rows +// can be added to the FastBatchInsertBuilder unless Reset is called. +// Exec must be called within a transaction. +func (b *FastBatchInsertBuilder) Exec(ctx context.Context, session SessionInterface, tableName string) error { + b.sealed = true + if session.GetTx() == nil { + return ErrNoTx + } + + if len(b.rows) == 0 { + return nil + } + + tx := session.GetTx() + stmt, err := tx.PrepareContext(ctx, pq.CopyIn(tableName, b.columns...)) + if err != nil { + return err + } + + for _, row := range b.rows { + if _, err = stmt.ExecContext(ctx, row...); err != nil { + // we need to close the statement otherwise the session + // will always return bad connection errors when executing + // any other sql statements, + // see https://github.com/stellar/go/pull/316#issuecomment-368990324 + stmt.Close() + return err + } + } + + if err = stmt.Close(); err != nil { + return err + } + return nil +} + +// Reset clears out all the rows contained in the FastBatchInsertBuilder. +// After Reset is called new rows can be added to the FastBatchInsertBuilder. +func (b *FastBatchInsertBuilder) Reset() { + b.sealed = false + b.columns = nil + b.rows = nil + b.rowStructType = nil +} diff --git a/support/db/fast_batch_insert_builder_test.go b/support/db/fast_batch_insert_builder_test.go new file mode 100644 index 0000000000..4acc09369c --- /dev/null +++ b/support/db/fast_batch_insert_builder_test.go @@ -0,0 +1,132 @@ +package db + +import ( + "context" + "testing" + + "github.com/guregu/null" + "github.com/stretchr/testify/assert" + + "github.com/stellar/go/support/db/dbtest" +) + +func TestFastBatchInsertBuilder(t *testing.T) { + db := dbtest.Postgres(t).Load(testSchema) + defer db.Close() + sess := &Session{DB: db.Open()} + defer sess.DB.Close() + + insertBuilder := &FastBatchInsertBuilder{} + + assert.NoError(t, + insertBuilder.Row(map[string]interface{}{ + "name": "bubba", + "hunger_level": "1", + "json_value": `{"bump_to": "97"}`, + }), + ) + + assert.EqualError(t, + insertBuilder.Row(map[string]interface{}{ + "name": "bubba", + }), + "invalid number of columns (expected=3, actual=1)", + ) + + assert.EqualError(t, + insertBuilder.Row(map[string]interface{}{ + "name": "bubba", + "city": "London", + "json_value": `{"bump_to": "98"}`, + }), + "column \"hunger_level\" does not exist", + ) + + assert.NoError(t, + insertBuilder.RowStruct(hungerRow{ + Name: "bubba2", + HungerLevel: "9", + JsonValue: `{"bump_to": "98"}`, + }), + ) + + assert.EqualError(t, + insertBuilder.RowStruct(invalidHungerRow{ + Name: "bubba", + HungerLevel: "2", + LastName: "b", + }), + "expected value of type \"db.hungerRow\" but got \"db.invalidHungerRow\" value", + ) + assert.Equal(t, 2, insertBuilder.Len()) + assert.Equal(t, false, insertBuilder.sealed) + + ctx := context.Background() + assert.EqualError(t, + insertBuilder.Exec(ctx, sess, "people"), + "cannot call Exec() outside of a transaction", + ) + assert.Equal(t, true, insertBuilder.sealed) + + assert.NoError(t, sess.Begin(ctx)) + assert.NoError(t, insertBuilder.Exec(ctx, sess, "people")) + assert.Equal(t, 2, insertBuilder.Len()) + assert.Equal(t, true, insertBuilder.sealed) + + var found []person + assert.NoError(t, sess.SelectRaw(context.Background(), &found, `SELECT * FROM people WHERE name like 'bubba%'`)) + assert.Equal( + t, + found, + []person{ + {Name: "bubba", HungerLevel: "1", JsonValue: null.NewString(`{"bump_to": "97"}`, true)}, + {Name: "bubba2", HungerLevel: "9", JsonValue: null.NewString(`{"bump_to": "98"}`, true)}, + }, + ) + + assert.EqualError(t, + insertBuilder.Row(map[string]interface{}{ + "name": "bubba3", + "hunger_level": "100", + }), + "cannot add more rows after Exec() without calling Reset() first", + ) + assert.Equal(t, 2, insertBuilder.Len()) + assert.Equal(t, true, insertBuilder.sealed) + + insertBuilder.Reset() + assert.Equal(t, 0, insertBuilder.Len()) + assert.Equal(t, false, insertBuilder.sealed) + + assert.NoError(t, + insertBuilder.Row(map[string]interface{}{ + "name": "bubba3", + "hunger_level": "3", + }), + ) + assert.Equal(t, 1, insertBuilder.Len()) + assert.Equal(t, false, insertBuilder.sealed) + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + assert.EqualError(t, + insertBuilder.Exec(ctx, sess, "people"), + "context canceled", + ) + assert.Equal(t, 1, insertBuilder.Len()) + assert.Equal(t, true, insertBuilder.sealed) + + assert.NoError(t, sess.SelectRaw(context.Background(), &found, `SELECT * FROM people WHERE name like 'bubba%'`)) + assert.Equal( + t, + found, + []person{ + {Name: "bubba", HungerLevel: "1", JsonValue: null.NewString(`{"bump_to": "97"}`, true)}, + {Name: "bubba2", HungerLevel: "9", JsonValue: null.NewString(`{"bump_to": "98"}`, true)}, + }, + ) + assert.NoError(t, sess.Rollback()) + + assert.NoError(t, sess.SelectRaw(context.Background(), &found, `SELECT * FROM people WHERE name like 'bubba%'`)) + assert.Empty(t, found) +} diff --git a/support/db/internal_test.go b/support/db/internal_test.go index 8ce0370a92..3e1a06dabc 100644 --- a/support/db/internal_test.go +++ b/support/db/internal_test.go @@ -7,6 +7,7 @@ const testSchema = ` CREATE TABLE IF NOT EXISTS people ( name character varying NOT NULL, hunger_level integer NOT NULL, + json_value jsonb, PRIMARY KEY (name) ); DELETE FROM people; diff --git a/support/db/main_test.go b/support/db/main_test.go index 68724d197d..301b533aa4 100644 --- a/support/db/main_test.go +++ b/support/db/main_test.go @@ -4,15 +4,16 @@ import ( "testing" "time" + "github.com/guregu/null" "github.com/stellar/go/support/db/dbtest" "github.com/stretchr/testify/assert" ) type person struct { - Name string `db:"name"` - HungerLevel string `db:"hunger_level"` - - SomethingIgnored int `db:"-"` + Name string `db:"name"` + HungerLevel string `db:"hunger_level"` + JsonValue null.String `db:"json_value"` + SomethingIgnored int `db:"-"` } func TestGetTable(t *testing.T) {