From 4e9ae5a2f14776e6a13dcc390174fbb9471443ae Mon Sep 17 00:00:00 2001 From: Bartek Nowotarski Date: Mon, 20 Sep 2021 20:39:42 +0200 Subject: [PATCH 1/4] services/horizon: Batch remove Accounts, move ingestion to AccountsProcessor --- .vscode/tasks.json | 16 +- .../horizon/internal/actions/account_test.go | 193 ++++------ .../horizon/internal/actions/asset_test.go | 22 +- .../horizon/internal/actions/offer_test.go | 4 +- .../horizon/internal/actions_path_test.go | 80 ++-- .../horizon/internal/db2/history/accounts.go | 202 +++------- .../history/accounts_batch_insert_builder.go | 21 - .../internal/db2/history/accounts_test.go | 363 ++++++------------ services/horizon/internal/db2/history/main.go | 19 +- .../internal/db2/history/mock_q_accounts.go | 14 +- .../internal/db2/history/offers_test.go | 4 +- .../db2/history/sequence_provider_test.go | 10 +- .../internal/db2/history/trust_lines_test.go | 15 +- .../internal/ingest/processor_runner_test.go | 34 +- .../ingest/processors/accounts_processor.go | 65 +++- .../processors/accounts_processor_test.go | 75 ++-- 16 files changed, 402 insertions(+), 735 deletions(-) delete mode 100644 services/horizon/internal/db2/history/accounts_batch_insert_builder.go diff --git a/.vscode/tasks.json b/.vscode/tasks.json index b6291051e5..0079459abc 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,23 +1,23 @@ { // See https://go.microsoft.com/fwlink/?LinkId=733558 // for the documentation about the tasks.json format - "version": "0.1.0", + "version": "2.0.0", "command": "go", "isShellCommand": true, - "showOutput": "always", "suppressTaskName": true, "options": { "cwd": "${workspaceRoot}" }, "tasks": [ { - "label": "test all", + "label": "go", "type": "shell", - "args": [ - "test", - "./..." - ], - "problemMatcher": [] + "command": "go", + "problemMatcher": [], + "group": { + "_id": "build", + "isDefault": false + } } ] } diff --git a/services/horizon/internal/actions/account_test.go b/services/horizon/internal/actions/account_test.go index fbb8f56ec3..b5519ab2ea 100644 --- a/services/horizon/internal/actions/account_test.go +++ b/services/horizon/internal/actions/account_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/guregu/null" "github.com/stretchr/testify/assert" protocol "github.com/stellar/go/protocols/horizon" @@ -20,94 +21,58 @@ var ( accountOne = "GABGMPEKKDWR2WFH5AJOZV5PDKLJEHGCR3Q24ALETWR5H3A7GI3YTS7V" accountTwo = "GADTXHUTHIAESMMQ2ZWSTIIGBZRLHUCBLCHPLLUEIAWDEFRDC4SYDKOZ" signer = "GCXKG6RN4ONIEPCMNFB732A436Z5PNDSRLGWK7GBLCMQLIFO4S7EYWVU" - sponsor = xdr.MustAddress("GCO26ZSBD63TKYX45H2C7D2WOFWOUSG5BMTNC3BG4QMXM3PAYI6WHKVZ") + sponsor = "GCO26ZSBD63TKYX45H2C7D2WOFWOUSG5BMTNC3BG4QMXM3PAYI6WHKVZ" usd = xdr.MustNewCreditAsset("USD", trustLineIssuer) euro = xdr.MustNewCreditAsset("EUR", trustLineIssuer) - account1 = xdr.LedgerEntry{ - LastModifiedLedgerSeq: 1234, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &xdr.AccountEntry{ - AccountId: xdr.MustAddress(accountOne), - Balance: 20000, - SeqNum: 223456789, - NumSubEntries: 10, - Flags: 1, - HomeDomain: "stellar.org", - Thresholds: xdr.Thresholds{1, 2, 3, 4}, - Ext: xdr.AccountEntryExt{ - V: 1, - V1: &xdr.AccountEntryExtensionV1{ - Liabilities: xdr.Liabilities{ - Buying: 3, - Selling: 4, - }, - }, - }, - }, - }, + account1 = history.AccountEntry{ + LastModifiedLedger: 1234, + AccountID: accountOne, + Balance: 20000, + SequenceNumber: 223456789, + NumSubEntries: 10, + Flags: 1, + HomeDomain: "stellar.org", + MasterWeight: 1, + ThresholdLow: 2, + ThresholdMedium: 3, + ThresholdHigh: 4, + BuyingLiabilities: 3, + SellingLiabilities: 3, } - account2 = xdr.LedgerEntry{ - LastModifiedLedgerSeq: 1234, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &xdr.AccountEntry{ - AccountId: xdr.MustAddress(accountTwo), - Balance: 50000, - SeqNum: 648736, - NumSubEntries: 10, - Flags: 2, - HomeDomain: "meridian.stellar.org", - Thresholds: xdr.Thresholds{5, 6, 7, 8}, - Ext: xdr.AccountEntryExt{ - V: 1, - V1: &xdr.AccountEntryExtensionV1{ - Liabilities: xdr.Liabilities{ - Buying: 30, - Selling: 40, - }, - }, - }, - }, - }, + account2 = history.AccountEntry{ + LastModifiedLedger: 1234, + AccountID: accountTwo, + Balance: 50000, + SequenceNumber: 648736, + NumSubEntries: 10, + Flags: 2, + HomeDomain: "meridian.stellar.org", + MasterWeight: 5, + ThresholdLow: 6, + ThresholdMedium: 7, + ThresholdHigh: 8, + BuyingLiabilities: 30, + SellingLiabilities: 40, } - account3 = xdr.LedgerEntry{ - LastModifiedLedgerSeq: 1234, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &xdr.AccountEntry{ - AccountId: xdr.MustAddress(signer), - Balance: 50000, - SeqNum: 648736, - NumSubEntries: 10, - Flags: 2, - Thresholds: xdr.Thresholds{5, 6, 7, 8}, - Ext: xdr.AccountEntryExt{ - V: 1, - V1: &xdr.AccountEntryExtensionV1{ - Liabilities: xdr.Liabilities{ - Buying: 30, - Selling: 40, - }, - Ext: xdr.AccountEntryExtensionV1Ext{ - V2: &xdr.AccountEntryExtensionV2{ - NumSponsored: 1, - NumSponsoring: 2, - }, - }, - }, - }, - }, - }, - Ext: xdr.LedgerEntryExt{ - V: 1, - V1: &xdr.LedgerEntryExtensionV1{ - SponsoringId: &sponsor, - }, - }, + account3 = history.AccountEntry{ + LastModifiedLedger: 1234, + AccountID: signer, + Balance: 50000, + SequenceNumber: 648736, + NumSubEntries: 10, + Flags: 2, + MasterWeight: 5, + ThresholdLow: 6, + ThresholdMedium: 7, + ThresholdHigh: 8, + BuyingLiabilities: 30, + SellingLiabilities: 40, + NumSponsored: 1, + NumSponsoring: 2, + Sponsor: null.StringFrom(sponsor), } eurTrustLine = xdr.LedgerEntry{ @@ -220,29 +185,23 @@ func TestAccountInfo(t *testing.T) { tt.Assert.NoError( xdr.SafeUnmarshalBase64("AQAAAA==", &thresholds), ) - accountID := xdr.MustAddress( - "GCXKG6RN4ONIEPCMNFB732A436Z5PNDSRLGWK7GBLCMQLIFO4S7EYWVU", - ) - accountEntry := xdr.LedgerEntry{ - LastModifiedLedgerSeq: 4, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &xdr.AccountEntry{ - AccountId: accountID, - Balance: 9999999900, - SeqNum: 8589934593, - NumSubEntries: 1, - InflationDest: nil, - HomeDomain: "", - Thresholds: thresholds, - Flags: 0, - }, - }, + accountID := "GCXKG6RN4ONIEPCMNFB732A436Z5PNDSRLGWK7GBLCMQLIFO4S7EYWVU" + accountEntry := history.AccountEntry{ + LastModifiedLedger: 4, + AccountID: accountID, + Balance: 9999999900, + SequenceNumber: 8589934593, + NumSubEntries: 1, + InflationDestination: "", + HomeDomain: "", + MasterWeight: thresholds[0], + ThresholdLow: thresholds[1], + ThresholdMedium: thresholds[2], + ThresholdHigh: thresholds[3], + Flags: 0, } - batch := q.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, accountEntry) + err := q.UpsertAccounts(tt.Ctx, []history.AccountEntry{accountEntry}) assert.NoError(t, err) - assert.NoError(t, batch.Exec(tt.Ctx)) tt.Assert.NoError(err) @@ -251,7 +210,7 @@ func TestAccountInfo(t *testing.T) { Data: xdr.LedgerEntryData{ Type: xdr.LedgerEntryTypeTrustline, TrustLine: &xdr.TrustLineEntry{ - AccountId: accountID, + AccountId: xdr.MustAddress(accountID), Asset: xdr.MustNewCreditAsset( "USD", "GC23QF2HUE52AMXUFUH3AYJAXXGXXV2VHXYYR6EYXETPKDXZSAW67XO4", @@ -269,7 +228,7 @@ func TestAccountInfo(t *testing.T) { Data: xdr.LedgerEntryData{ Type: xdr.LedgerEntryTypeTrustline, TrustLine: &xdr.TrustLineEntry{ - AccountId: accountID, + AccountId: xdr.MustAddress(accountID), Asset: xdr.MustNewCreditAsset( "EUR", "GC23QF2HUE52AMXUFUH3AYJAXXGXXV2VHXYYR6EYXETPKDXZSAW67XO4", @@ -293,7 +252,7 @@ func TestAccountInfo(t *testing.T) { }, 0, 0, 0, 0, 0) assert.NoError(t, err) - account, err := AccountInfo(tt.Ctx, &history.Q{tt.HorizonSession()}, accountID.Address()) + account, err := AccountInfo(tt.Ctx, &history.Q{tt.HorizonSession()}, accountID) tt.Assert.NoError(err) tt.Assert.Equal("8589934593", account.Sequence) @@ -354,14 +313,8 @@ func TestGetAccountsHandlerPageResultsBySigner(t *testing.T) { q := &history.Q{tt.HorizonSession()} handler := &GetAccountsHandler{} - batch := q.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, account1) + err := q.UpsertAccounts(tt.Ctx, []history.AccountEntry{account1, account2, account3}) assert.NoError(t, err) - err = batch.Add(tt.Ctx, account2) - assert.NoError(t, err) - err = batch.Add(tt.Ctx, account3) - assert.NoError(t, err) - assert.NoError(t, batch.Exec(tt.Ctx)) for _, row := range accountSigners { q.CreateAccountSigner(tt.Ctx, row.Account, row.Signer, row.Weight, nil) @@ -434,14 +387,8 @@ func TestGetAccountsHandlerPageResultsBySponsor(t *testing.T) { q := &history.Q{tt.HorizonSession()} handler := &GetAccountsHandler{} - batch := q.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, account1) - assert.NoError(t, err) - err = batch.Add(tt.Ctx, account2) + err := q.UpsertAccounts(tt.Ctx, []history.AccountEntry{account1, account2, account3}) assert.NoError(t, err) - err = batch.Add(tt.Ctx, account3) - assert.NoError(t, err) - assert.NoError(t, batch.Exec(tt.Ctx)) for _, row := range accountSigners { q.CreateAccountSigner(tt.Ctx, row.Account, row.Signer, row.Weight, nil) @@ -452,7 +399,7 @@ func TestGetAccountsHandlerPageResultsBySponsor(t *testing.T) { makeRequest( t, map[string]string{ - "sponsor": sponsor.Address(), + "sponsor": sponsor, }, map[string]string{}, q, @@ -472,12 +419,8 @@ func TestGetAccountsHandlerPageResultsByAsset(t *testing.T) { q := &history.Q{tt.HorizonSession()} handler := &GetAccountsHandler{} - batch := q.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, account1) - assert.NoError(t, err) - err = batch.Add(tt.Ctx, account2) + err := q.UpsertAccounts(tt.Ctx, []history.AccountEntry{account1, account2}) assert.NoError(t, err) - assert.NoError(t, batch.Exec(tt.Ctx)) ledgerCloseTime := time.Now().Unix() _, err = q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ diff --git a/services/horizon/internal/actions/asset_test.go b/services/horizon/internal/actions/asset_test.go index e0233fedc7..504f467d9d 100644 --- a/services/horizon/internal/actions/asset_test.go +++ b/services/horizon/internal/actions/asset_test.go @@ -312,23 +312,15 @@ func TestAssetStats(t *testing.T) { issuer, otherIssuer, } { - accountEntry := xdr.LedgerEntry{ - LastModifiedLedgerSeq: 100, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &xdr.AccountEntry{ - Flags: xdr.Uint32(account.Flags), - HomeDomain: xdr.String32(account.HomeDomain), - }, - }, - } - if err := accountEntry.Data.Account.AccountId.SetAddress(account.AccountID); err != nil { - t.Fatalf("unexpected error %v", err) + accountEntry := history.AccountEntry{ + LastModifiedLedger: 100, + AccountID: account.AccountID, + Flags: account.Flags, + HomeDomain: account.HomeDomain, } - batch := q.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, accountEntry) + + err := q.UpsertAccounts(tt.Ctx, []history.AccountEntry{accountEntry}) tt.Assert.NoError(err) - tt.Assert.NoError(batch.Exec(tt.Ctx)) } for _, testCase := range []struct { diff --git a/services/horizon/internal/actions/offer_test.go b/services/horizon/internal/actions/offer_test.go index 108af3236d..41663c65d5 100644 --- a/services/horizon/internal/actions/offer_test.go +++ b/services/horizon/internal/actions/offer_test.go @@ -52,7 +52,7 @@ var ( Price: float64(2), Flags: 2, LastModifiedLedger: uint32(4), - Sponsor: null.StringFrom(sponsor.Address()), + Sponsor: null.StringFrom(sponsor), } usdOffer = history.Offer{ SellerID: issuer.Address(), @@ -268,7 +268,7 @@ func TestGetOffersHandler(t *testing.T) { makeRequest( t, map[string]string{ - "sponsor": sponsor.Address(), + "sponsor": sponsor, }, map[string]string{}, q, diff --git a/services/horizon/internal/actions_path_test.go b/services/horizon/internal/actions_path_test.go index 96e83b6f5a..4ab8f5b836 100644 --- a/services/horizon/internal/actions_path_test.go +++ b/services/horizon/internal/actions_path_test.go @@ -143,34 +143,22 @@ func TestPathActionsStrictReceive(t *testing.T) { q := &history.Q{tt.HorizonSession()} - account := xdr.LedgerEntry{ - LastModifiedLedgerSeq: 1234, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &xdr.AccountEntry{ - AccountId: xdr.MustAddress(sourceAccount), - Balance: 20000, - SeqNum: 223456789, - NumSubEntries: 10, - Flags: 1, - Thresholds: xdr.Thresholds{1, 2, 3, 4}, - Ext: xdr.AccountEntryExt{ - V: 1, - V1: &xdr.AccountEntryExtensionV1{ - Liabilities: xdr.Liabilities{ - Buying: 3, - Selling: 4, - }, - }, - }, - }, - }, + account := history.AccountEntry{ + LastModifiedLedger: 1234, + AccountID: sourceAccount, + Balance: 20000, + SequenceNumber: 223456789, + NumSubEntries: 10, + Flags: 1, + MasterWeight: 1, + ThresholdLow: 2, + ThresholdMedium: 3, + ThresholdHigh: 4, + BuyingLiabilities: 3, + SellingLiabilities: 4, } - batch := q.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, account) - assert.NoError(t, err) - err = batch.Exec(tt.Ctx) + err := q.UpsertAccounts(tt.Ctx, []history.AccountEntry{account}) assert.NoError(t, err) assetsByKeys := map[string]xdr.Asset{} @@ -501,34 +489,22 @@ func TestPathActionsStrictSend(t *testing.T) { xdr.MustNewNativeAsset(), } - account := xdr.LedgerEntry{ - LastModifiedLedgerSeq: 1234, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &xdr.AccountEntry{ - AccountId: xdr.MustAddress(destinationAccount), - Balance: 20000, - SeqNum: 223456789, - NumSubEntries: 10, - Flags: 1, - Thresholds: xdr.Thresholds{1, 2, 3, 4}, - Ext: xdr.AccountEntryExt{ - V: 1, - V1: &xdr.AccountEntryExtensionV1{ - Liabilities: xdr.Liabilities{ - Buying: 3, - Selling: 4, - }, - }, - }, - }, - }, + account := history.AccountEntry{ + LastModifiedLedger: 1234, + AccountID: destinationAccount, + Balance: 20000, + SequenceNumber: 223456789, + NumSubEntries: 10, + Flags: 1, + MasterWeight: 1, + ThresholdLow: 2, + ThresholdMedium: 3, + ThresholdHigh: 4, + BuyingLiabilities: 3, + SellingLiabilities: 4, } - batch := historyQ.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, account) - assert.NoError(t, err) - err = batch.Exec(tt.Ctx) + err := historyQ.UpsertAccounts(tt.Ctx, []history.AccountEntry{account}) assert.NoError(t, err) assetsByKeys := map[string]xdr.Asset{} diff --git a/services/horizon/internal/db2/history/accounts.go b/services/horizon/internal/db2/history/accounts.go index a4b60fd382..695374d3c7 100644 --- a/services/horizon/internal/db2/history/accounts.go +++ b/services/horizon/internal/db2/history/accounts.go @@ -4,11 +4,8 @@ import ( "context" sq "github.com/Masterminds/squirrel" - "github.com/guregu/null" - "github.com/lib/pq" "github.com/stellar/go/services/horizon/internal/db2" - "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" ) @@ -62,156 +59,57 @@ func (q *Q) GetAccountsByIDs(ctx context.Context, ids []string) ([]AccountEntry, return accounts, err } -func accountToMap(entry xdr.LedgerEntry) map[string]interface{} { - account := entry.Data.MustAccount() - liabilities := account.Liabilities() - - var inflationDestination = "" - if account.InflationDest != nil { - inflationDestination = account.InflationDest.Address() - } - - return map[string]interface{}{ - "account_id": account.AccountId.Address(), - "balance": account.Balance, - "buying_liabilities": liabilities.Buying, - "selling_liabilities": liabilities.Selling, - "sequence_number": account.SeqNum, - "num_subentries": account.NumSubEntries, - "inflation_destination": inflationDestination, - "flags": account.Flags, - "home_domain": account.HomeDomain, - "master_weight": account.MasterKeyWeight(), - "threshold_low": account.ThresholdLow(), - "threshold_medium": account.ThresholdMedium(), - "threshold_high": account.ThresholdHigh(), - "last_modified_ledger": entry.LastModifiedLedgerSeq, - "sponsor": ledgerEntrySponsorToNullString(entry), - "num_sponsored": account.NumSponsored(), - "num_sponsoring": account.NumSponsoring(), - } -} - // UpsertAccounts upserts a batch of accounts in the accounts table. // There's currently no limit of the number of accounts this method can // accept other than 2GB limit of the query string length what should be enough // for each ledger with the current limits. -func (q *Q) UpsertAccounts(ctx context.Context, accounts []xdr.LedgerEntry) error { - var accountID, inflationDestination []string - var homeDomain []xdr.String32 - var balance, buyingLiabilities, sellingLiabilities []xdr.Int64 - var sequenceNumber []xdr.SequenceNumber - var numSubEntries, flags, lastModifiedLedger, numSponsored, numSponsoring []xdr.Uint32 - var masterWeight, thresholdLow, thresholdMedium, thresholdHigh []uint8 - var sponsor []null.String - - for _, entry := range accounts { - if entry.Data.Type != xdr.LedgerEntryTypeAccount { - return errors.Errorf("Invalid entry type: %d", entry.Data.Type) - } +func (q *Q) UpsertAccounts(ctx context.Context, accounts []AccountEntry) error { + var accountID, inflationDestination, homeDomain, balance, buyingLiabilities, + sellingLiabilities, sequenceNumber, numSubEntries, flags, lastModifiedLedger, + numSponsored, numSponsoring, masterWeight, thresholdLow, thresholdMedium, + thresholdHigh, sponsor []interface{} + + for _, account := range accounts { + accountID = append(accountID, account.AccountID) + balance = append(balance, account.Balance) + buyingLiabilities = append(buyingLiabilities, account.BuyingLiabilities) + sellingLiabilities = append(sellingLiabilities, account.SellingLiabilities) + sequenceNumber = append(sequenceNumber, account.SequenceNumber) + numSubEntries = append(numSubEntries, account.NumSubEntries) + inflationDestination = append(inflationDestination, account.InflationDestination) + homeDomain = append(homeDomain, account.HomeDomain) + flags = append(flags, account.Flags) + masterWeight = append(masterWeight, account.MasterWeight) + thresholdLow = append(thresholdLow, account.ThresholdLow) + thresholdMedium = append(thresholdMedium, account.ThresholdMedium) + thresholdHigh = append(thresholdHigh, account.ThresholdHigh) + lastModifiedLedger = append(lastModifiedLedger, account.LastModifiedLedger) + sponsor = append(sponsor, account.Sponsor) + numSponsored = append(numSponsored, account.NumSponsored) + numSponsoring = append(numSponsoring, account.NumSponsoring) + } - m := accountToMap(entry) - accountID = append(accountID, m["account_id"].(string)) - balance = append(balance, m["balance"].(xdr.Int64)) - buyingLiabilities = append(buyingLiabilities, m["buying_liabilities"].(xdr.Int64)) - sellingLiabilities = append(sellingLiabilities, m["selling_liabilities"].(xdr.Int64)) - sequenceNumber = append(sequenceNumber, m["sequence_number"].(xdr.SequenceNumber)) - numSubEntries = append(numSubEntries, m["num_subentries"].(xdr.Uint32)) - inflationDestination = append(inflationDestination, m["inflation_destination"].(string)) - flags = append(flags, m["flags"].(xdr.Uint32)) - homeDomain = append(homeDomain, m["home_domain"].(xdr.String32)) - masterWeight = append(masterWeight, m["master_weight"].(uint8)) - thresholdLow = append(thresholdLow, m["threshold_low"].(uint8)) - thresholdMedium = append(thresholdMedium, m["threshold_medium"].(uint8)) - thresholdHigh = append(thresholdHigh, m["threshold_high"].(uint8)) - lastModifiedLedger = append(lastModifiedLedger, m["last_modified_ledger"].(xdr.Uint32)) - sponsor = append(sponsor, m["sponsor"].(null.String)) - numSponsored = append(numSponsored, m["num_sponsored"].(xdr.Uint32)) - numSponsoring = append(numSponsoring, m["num_sponsoring"].(xdr.Uint32)) + upsertFields := []upsertField{ + {"account_id", "text", accountID}, + {"balance", "bigint", balance}, + {"buying_liabilities", "bigint", buyingLiabilities}, + {"selling_liabilities", "bigint", sellingLiabilities}, + {"sequence_number", "bigint", sequenceNumber}, + {"num_subentries", "int", numSubEntries}, + {"inflation_destination", "text", inflationDestination}, + {"flags", "int", flags}, + {"home_domain", "text", homeDomain}, + {"master_weight", "int", masterWeight}, + {"threshold_low", "int", thresholdLow}, + {"threshold_medium", "int", thresholdMedium}, + {"threshold_high", "int", thresholdHigh}, + {"last_modified_ledger", "int", lastModifiedLedger}, + {"sponsor", "text", sponsor}, + {"num_sponsored", "int", numSponsored}, + {"num_sponsoring", "int", numSponsoring}, } - sql := ` - WITH r AS - (SELECT - unnest(?::text[]), /* account_id */ - unnest(?::bigint[]), /* balance */ - unnest(?::bigint[]), /* buying_liabilities */ - unnest(?::bigint[]), /* selling_liabilities */ - unnest(?::bigint[]), /* sequence_number */ - unnest(?::int[]), /* num_subentries */ - unnest(?::text[]), /* inflation_destination */ - unnest(?::int[]), /* flags */ - unnest(?::text[]), /* home_domain */ - unnest(?::int[]), /* master_weight */ - unnest(?::int[]), /* threshold_low */ - unnest(?::int[]), /* threshold_medium */ - unnest(?::int[]), /* threshold_high */ - unnest(?::int[]), /* last_modified_ledger */ - unnest(?::text[]), /* sponsor */ - unnest(?::int[]), /* num_sponsored */ - unnest(?::int[]) /* num_sponsoring */ - ) - INSERT INTO accounts ( - account_id, - balance, - buying_liabilities, - selling_liabilities, - sequence_number, - num_subentries, - inflation_destination, - flags, - home_domain, - master_weight, - threshold_low, - threshold_medium, - threshold_high, - last_modified_ledger, - sponsor, - num_sponsored, - num_sponsoring - ) - SELECT * from r - ON CONFLICT (account_id) DO UPDATE SET - account_id = excluded.account_id, - balance = excluded.balance, - buying_liabilities = excluded.buying_liabilities, - selling_liabilities = excluded.selling_liabilities, - sequence_number = excluded.sequence_number, - num_subentries = excluded.num_subentries, - inflation_destination = excluded.inflation_destination, - flags = excluded.flags, - home_domain = excluded.home_domain, - master_weight = excluded.master_weight, - threshold_low = excluded.threshold_low, - threshold_medium = excluded.threshold_medium, - threshold_high = excluded.threshold_high, - last_modified_ledger = excluded.last_modified_ledger, - sponsor = excluded.sponsor, - num_sponsored = excluded.num_sponsored, - num_sponsoring = excluded.num_sponsoring` - - _, err := q.ExecRaw( - context.WithValue(ctx, &db.QueryTypeContextKey, db.UpsertQueryType), - sql, - pq.Array(accountID), - pq.Array(balance), - pq.Array(buyingLiabilities), - pq.Array(sellingLiabilities), - pq.Array(sequenceNumber), - pq.Array(numSubEntries), - pq.Array(inflationDestination), - pq.Array(flags), - pq.Array(homeDomain), - pq.Array(masterWeight), - pq.Array(thresholdLow), - pq.Array(thresholdMedium), - pq.Array(thresholdHigh), - pq.Array(lastModifiedLedger), - pq.Array(sponsor), - pq.Array(numSponsored), - pq.Array(numSponsoring), - ) - return err + return q.upsertRows(ctx, "accounts", "account_id", upsertFields) } // RemoveAccount deletes a row in the accounts table. @@ -226,6 +124,18 @@ func (q *Q) RemoveAccount(ctx context.Context, accountID string) (int64, error) return result.RowsAffected() } +// RemoveAccounts deletes a row in the accounts table. +// Returns number of rows affected and error. +func (q *Q) RemoveAccounts(ctx context.Context, accountIDs []string) (int64, error) { + sql := sq.Delete("accounts").Where(sq.Eq{"account_id": accountIDs}) + result, err := q.Exec(ctx, sql) + if err != nil { + return 0, err + } + + return result.RowsAffected() +} + // AccountsForAsset returns a list of `AccountEntry` rows who are trustee to an // asset func (q *Q) AccountsForAsset(ctx context.Context, asset xdr.Asset, page db2.PageQuery) ([]AccountEntry, error) { diff --git a/services/horizon/internal/db2/history/accounts_batch_insert_builder.go b/services/horizon/internal/db2/history/accounts_batch_insert_builder.go deleted file mode 100644 index 7c4e0abddc..0000000000 --- a/services/horizon/internal/db2/history/accounts_batch_insert_builder.go +++ /dev/null @@ -1,21 +0,0 @@ -package history - -import ( - "context" - - "github.com/stellar/go/support/db" - "github.com/stellar/go/xdr" -) - -// accountsBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder -type accountsBatchInsertBuilder struct { - builder db.BatchInsertBuilder -} - -func (i *accountsBatchInsertBuilder) Add(ctx context.Context, entry xdr.LedgerEntry) error { - return i.builder.Row(ctx, accountToMap(entry)) -} - -func (i *accountsBatchInsertBuilder) Exec(ctx context.Context) error { - return i.builder.Exec(ctx) -} diff --git a/services/horizon/internal/db2/history/accounts_test.go b/services/horizon/internal/db2/history/accounts_test.go index 461f4ca687..55f390cc46 100644 --- a/services/horizon/internal/db2/history/accounts_test.go +++ b/services/horizon/internal/db2/history/accounts_test.go @@ -11,96 +11,60 @@ import ( ) var ( - inflationDest = xdr.MustAddress("GBUH7T6U36DAVEKECMKN5YEBQYZVRBPNSZAAKBCO6P5HBMDFSQMQL4Z4") - sponsor = xdr.MustAddress("GCO26ZSBD63TKYX45H2C7D2WOFWOUSG5BMTNC3BG4QMXM3PAYI6WHKVZ") - - account1 = xdr.LedgerEntry{ - LastModifiedLedgerSeq: 1234, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &xdr.AccountEntry{ - AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), - Balance: 20000, - SeqNum: 223456789, - NumSubEntries: 10, - InflationDest: &inflationDest, - Flags: 1, - HomeDomain: "stellar.org", - Thresholds: xdr.Thresholds{1, 2, 3, 4}, - Ext: xdr.AccountEntryExt{ - V: 1, - V1: &xdr.AccountEntryExtensionV1{ - Liabilities: xdr.Liabilities{ - Buying: 3, - Selling: 4, - }, - }, - }, - }, - }, + inflationDest = "GBUH7T6U36DAVEKECMKN5YEBQYZVRBPNSZAAKBCO6P5HBMDFSQMQL4Z4" + sponsor = "GCO26ZSBD63TKYX45H2C7D2WOFWOUSG5BMTNC3BG4QMXM3PAYI6WHKVZ" + + account1 = AccountEntry{ + LastModifiedLedger: 1234, + AccountID: "GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB", + Balance: 20000, + SequenceNumber: 223456789, + NumSubEntries: 10, + InflationDestination: inflationDest, + Flags: 1, + HomeDomain: "stellar.org", + MasterWeight: 1, + ThresholdLow: 2, + ThresholdMedium: 3, + ThresholdHigh: 4, + BuyingLiabilities: 3, + SellingLiabilities: 4, } - account2 = xdr.LedgerEntry{ - LastModifiedLedgerSeq: 1235, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &xdr.AccountEntry{ - AccountId: xdr.MustAddress("GCT2NQM5KJJEF55NPMY444C6M6CA7T33HRNCMA6ZFBIIXKNCRO6J25K7"), - Balance: 50000, - SeqNum: 648736, - NumSubEntries: 10, - InflationDest: &inflationDest, - Flags: 2, - HomeDomain: "meridian.stellar.org", - Thresholds: xdr.Thresholds{5, 6, 7, 8}, - Ext: xdr.AccountEntryExt{ - V: 1, - V1: &xdr.AccountEntryExtensionV1{ - Liabilities: xdr.Liabilities{ - Buying: 30, - Selling: 40, - }, - Ext: xdr.AccountEntryExtensionV1Ext{ - V2: &xdr.AccountEntryExtensionV2{ - NumSponsored: 1, - NumSponsoring: 2, - }, - }, - }, - }, - }, - }, - Ext: xdr.LedgerEntryExt{ - V: 1, - V1: &xdr.LedgerEntryExtensionV1{ - SponsoringId: &sponsor, - }, - }, + account2 = AccountEntry{ + LastModifiedLedger: 1235, + AccountID: "GCT2NQM5KJJEF55NPMY444C6M6CA7T33HRNCMA6ZFBIIXKNCRO6J25K7", + Balance: 50000, + SequenceNumber: 648736, + NumSubEntries: 10, + InflationDestination: inflationDest, + Flags: 2, + HomeDomain: "meridian.stellar.org", + MasterWeight: 5, + ThresholdLow: 6, + ThresholdMedium: 7, + ThresholdHigh: 8, + BuyingLiabilities: 30, + SellingLiabilities: 40, + NumSponsored: 1, + NumSponsoring: 2, + Sponsor: null.StringFrom(sponsor), } - account3 = xdr.LedgerEntry{ - LastModifiedLedgerSeq: 1235, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &xdr.AccountEntry{ - AccountId: xdr.MustAddress("GDPGOMFSP4IF7A4P7UBKA4UC4QTRLEHGBD6IMDIS3W3KBDNBFAQ7FXDY"), - Balance: 50000, - SeqNum: 648736, - NumSubEntries: 10, - InflationDest: &inflationDest, - Flags: 2, - Thresholds: xdr.Thresholds{5, 6, 7, 8}, - Ext: xdr.AccountEntryExt{ - V: 1, - V1: &xdr.AccountEntryExtensionV1{ - Liabilities: xdr.Liabilities{ - Buying: 30, - Selling: 40, - }, - }, - }, - }, - }, + account3 = AccountEntry{ + LastModifiedLedger: 1235, + AccountID: "GDPGOMFSP4IF7A4P7UBKA4UC4QTRLEHGBD6IMDIS3W3KBDNBFAQ7FXDY", + Balance: 50000, + SequenceNumber: 648736, + NumSubEntries: 10, + InflationDestination: inflationDest, + Flags: 2, + MasterWeight: 5, + ThresholdLow: 6, + ThresholdMedium: 7, + ThresholdHigh: 8, + BuyingLiabilities: 30, + SellingLiabilities: 40, } ) @@ -110,12 +74,8 @@ func TestInsertAccount(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - batch := q.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, account1) + err := q.UpsertAccounts(tt.Ctx, []AccountEntry{account1, account2}) assert.NoError(t, err) - err = batch.Add(tt.Ctx, account2) - assert.NoError(t, err) - assert.NoError(t, batch.Exec(tt.Ctx)) accounts, err := q.GetAccountsByIDs(tt.Ctx, []string{ "GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB", @@ -143,7 +103,7 @@ func TestInsertAccount(t *testing.T) { assert.Equal(t, uint32(1), accounts[1].NumSponsored) assert.Equal(t, uint32(2), accounts[1].NumSponsoring) - assert.Equal(t, null.StringFrom(sponsor.Address()), accounts[1].Sponsor) + assert.Equal(t, null.StringFrom(sponsor), accounts[1].Sponsor) } func TestUpsertAccount(t *testing.T) { @@ -152,42 +112,33 @@ func TestUpsertAccount(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - ledgerEntries := []xdr.LedgerEntry{account1, account2} + ledgerEntries := []AccountEntry{account1, account2} err := q.UpsertAccounts(tt.Ctx, ledgerEntries) assert.NoError(t, err) - modifiedAccount := xdr.LedgerEntry{ - LastModifiedLedgerSeq: 1234, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &xdr.AccountEntry{ - AccountId: xdr.MustAddress("GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"), - Balance: 32847893, - SeqNum: 223456789, - NumSubEntries: 10, - InflationDest: &inflationDest, - Flags: 1, - HomeDomain: "stellar.org", - Thresholds: xdr.Thresholds{1, 2, 3, 4}, - Ext: xdr.AccountEntryExt{ - V: 1, - V1: &xdr.AccountEntryExtensionV1{ - Liabilities: xdr.Liabilities{ - Buying: 3, - Selling: 4, - }, - }, - }, - }, - }, + modifiedAccount := AccountEntry{ + LastModifiedLedger: 1234, + AccountID: "GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB", + Balance: 32847893, + SequenceNumber: 223456789, + NumSubEntries: 10, + InflationDestination: inflationDest, + Flags: 1, + HomeDomain: "stellar.org", + MasterWeight: 1, + ThresholdLow: 2, + ThresholdMedium: 3, + ThresholdHigh: 4, + BuyingLiabilities: 3, + SellingLiabilities: 4, } - err = q.UpsertAccounts(tt.Ctx, []xdr.LedgerEntry{modifiedAccount}) + err = q.UpsertAccounts(tt.Ctx, []AccountEntry{modifiedAccount}) assert.NoError(t, err) keys := []string{ - account1.Data.Account.AccountId.Address(), - account2.Data.Account.AccountId.Address(), + account1.AccountID, + account2.AccountID, } accounts, err := q.GetAccountsByIDs(tt.Ctx, keys) assert.NoError(t, err) @@ -195,84 +146,24 @@ func TestUpsertAccount(t *testing.T) { assert.Equal(t, uint32(1), accounts[0].NumSponsored) assert.Equal(t, uint32(2), accounts[0].NumSponsoring) - assert.Equal(t, null.StringFrom(sponsor.Address()), accounts[0].Sponsor) + assert.Equal(t, null.StringFrom(sponsor), accounts[0].Sponsor) assert.Equal(t, uint32(0), accounts[1].NumSponsored) assert.Equal(t, uint32(0), accounts[1].NumSponsoring) assert.Equal(t, null.String{}, accounts[1].Sponsor) - accounts, err = q.GetAccountsByIDs(tt.Ctx, []string{account1.Data.Account.AccountId.Address()}) + accounts, err = q.GetAccountsByIDs(tt.Ctx, []string{account1.AccountID}) assert.NoError(t, err) assert.Len(t, accounts, 1) - expectedBinary, err := modifiedAccount.Data.Account.MarshalBinary() - assert.NoError(t, err) - - dbEntry := xdr.AccountEntry{ - AccountId: xdr.MustAddress(accounts[0].AccountID), - Balance: xdr.Int64(accounts[0].Balance), - SeqNum: xdr.SequenceNumber(accounts[0].SequenceNumber), - NumSubEntries: xdr.Uint32(accounts[0].NumSubEntries), - InflationDest: &inflationDest, - Flags: xdr.Uint32(accounts[0].Flags), - HomeDomain: xdr.String32(accounts[0].HomeDomain), - Thresholds: xdr.Thresholds{ - accounts[0].MasterWeight, - accounts[0].ThresholdLow, - accounts[0].ThresholdMedium, - accounts[0].ThresholdHigh, - }, - Ext: xdr.AccountEntryExt{ - V: 1, - V1: &xdr.AccountEntryExtensionV1{ - Liabilities: xdr.Liabilities{ - Buying: xdr.Int64(accounts[0].BuyingLiabilities), - Selling: xdr.Int64(accounts[0].SellingLiabilities), - }, - }, - }, - } - - actualBinary, err := dbEntry.MarshalBinary() - assert.NoError(t, err) - assert.Equal(t, expectedBinary, actualBinary) + assert.Equal(t, modifiedAccount, accounts[0]) assert.Equal(t, uint32(1234), accounts[0].LastModifiedLedger) - accounts, err = q.GetAccountsByIDs(tt.Ctx, []string{account2.Data.Account.AccountId.Address()}) + accounts, err = q.GetAccountsByIDs(tt.Ctx, []string{account2.AccountID}) assert.NoError(t, err) assert.Len(t, accounts, 1) - expectedBinary, err = account2.Data.Account.MarshalBinary() - assert.NoError(t, err) - - dbEntry = xdr.AccountEntry{ - AccountId: xdr.MustAddress(accounts[0].AccountID), - Balance: xdr.Int64(accounts[0].Balance), - SeqNum: xdr.SequenceNumber(accounts[0].SequenceNumber), - NumSubEntries: xdr.Uint32(accounts[0].NumSubEntries), - InflationDest: &inflationDest, - Flags: xdr.Uint32(accounts[0].Flags), - HomeDomain: xdr.String32(accounts[0].HomeDomain), - Thresholds: xdr.Thresholds{ - accounts[0].MasterWeight, - accounts[0].ThresholdLow, - accounts[0].ThresholdMedium, - accounts[0].ThresholdHigh, - }, - Ext: xdr.AccountEntryExt{ - V: 1, - V1: &xdr.AccountEntryExtensionV1{ - Liabilities: xdr.Liabilities{ - Buying: xdr.Int64(accounts[0].BuyingLiabilities), - Selling: xdr.Int64(accounts[0].SellingLiabilities), - }, - }, - }, - } - - actualBinary, err = dbEntry.MarshalBinary() - assert.NoError(t, err) - assert.Equal(t, expectedBinary, actualBinary) + assert.Equal(t, account2, accounts[0]) assert.Equal(t, uint32(1235), accounts[0].LastModifiedLedger) } @@ -282,13 +173,11 @@ func TestRemoveAccount(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - batch := q.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, account1) + err := q.UpsertAccounts(tt.Ctx, []AccountEntry{account1}) assert.NoError(t, err) - assert.NoError(t, batch.Exec(tt.Ctx)) var rows int64 - rows, err = q.RemoveAccount(tt.Ctx, "GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB") + rows, err = q.RemoveAccounts(tt.Ctx, []string{"GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"}) assert.NoError(t, err) assert.Equal(t, int64(1), rows) @@ -297,7 +186,7 @@ func TestRemoveAccount(t *testing.T) { assert.Len(t, accounts, 0) // Doesn't exist anymore - rows, err = q.RemoveAccount(tt.Ctx, "GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB") + rows, err = q.RemoveAccounts(tt.Ctx, []string{"GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"}) assert.NoError(t, err) assert.Equal(t, int64(0), rows) } @@ -308,15 +197,11 @@ func TestAccountsForAsset(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - eurTrustLine.Data.TrustLine.AccountId = account1.Data.Account.AccountId - usdTrustLine.Data.TrustLine.AccountId = account2.Data.Account.AccountId + eurTrustLine.Data.TrustLine.AccountId = xdr.MustAddress(account1.AccountID) + usdTrustLine.Data.TrustLine.AccountId = xdr.MustAddress(account2.AccountID) - batch := q.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, account1) - assert.NoError(t, err) - err = batch.Add(tt.Ctx, account2) + err := q.UpsertAccounts(tt.Ctx, []AccountEntry{account1, account2}) assert.NoError(t, err) - assert.NoError(t, batch.Exec(tt.Ctx)) _, err = q.InsertTrustLine(tt.Ctx, eurTrustLine) tt.Assert.NoError(err) @@ -332,14 +217,14 @@ func TestAccountsForAsset(t *testing.T) { accounts, err := q.AccountsForAsset(tt.Ctx, eurTrustLine.Data.TrustLine.Asset.ToAsset(), pq) assert.NoError(t, err) tt.Assert.Len(accounts, 1) - tt.Assert.Equal(account1.Data.Account.AccountId.Address(), accounts[0].AccountID) + tt.Assert.Equal(account1.AccountID, accounts[0].AccountID) accounts, err = q.AccountsForAsset(tt.Ctx, usdTrustLine.Data.TrustLine.Asset.ToAsset(), pq) assert.NoError(t, err) tt.Assert.Len(accounts, 1) - tt.Assert.Equal(account2.Data.Account.AccountId.Address(), accounts[0].AccountID) + tt.Assert.Equal(account2.AccountID, accounts[0].AccountID) - pq.Cursor = account2.Data.Account.AccountId.Address() + pq.Cursor = account2.AccountID accounts, err = q.AccountsForAsset(tt.Ctx, usdTrustLine.Data.TrustLine.Asset.ToAsset(), pq) assert.NoError(t, err) tt.Assert.Len(accounts, 0) @@ -361,32 +246,26 @@ func TestAccountsForSponsor(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - eurTrustLine.Data.TrustLine.AccountId = account1.Data.Account.AccountId - usdTrustLine.Data.TrustLine.AccountId = account2.Data.Account.AccountId + eurTrustLine.Data.TrustLine.AccountId = xdr.MustAddress(account1.AccountID) + usdTrustLine.Data.TrustLine.AccountId = xdr.MustAddress(account2.AccountID) - batch := q.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, account1) + err := q.UpsertAccounts(tt.Ctx, []AccountEntry{account1, account2, account3}) assert.NoError(t, err) - err = batch.Add(tt.Ctx, account2) - assert.NoError(t, err) - err = batch.Add(tt.Ctx, account3) - assert.NoError(t, err) - assert.NoError(t, batch.Exec(tt.Ctx)) _, err = q.InsertTrustLine(tt.Ctx, eurTrustLine) tt.Assert.NoError(err) _, err = q.InsertTrustLine(tt.Ctx, usdTrustLine) tt.Assert.NoError(err) - _, err = q.CreateAccountSigner(tt.Ctx, account1.Data.Account.AccountId.Address(), account1.Data.Account.AccountId.Address(), 1, nil) + _, err = q.CreateAccountSigner(tt.Ctx, account1.AccountID, account1.AccountID, 1, nil) tt.Assert.NoError(err) - _, err = q.CreateAccountSigner(tt.Ctx, account2.Data.Account.AccountId.Address(), account2.Data.Account.AccountId.Address(), 1, nil) + _, err = q.CreateAccountSigner(tt.Ctx, account2.AccountID, account2.AccountID, 1, nil) tt.Assert.NoError(err) - _, err = q.CreateAccountSigner(tt.Ctx, account3.Data.Account.AccountId.Address(), account3.Data.Account.AccountId.Address(), 1, nil) + _, err = q.CreateAccountSigner(tt.Ctx, account3.AccountID, account3.AccountID, 1, nil) tt.Assert.NoError(err) - _, err = q.CreateAccountSigner(tt.Ctx, account1.Data.Account.AccountId.Address(), account3.Data.Account.AccountId.Address(), 1, nil) + _, err = q.CreateAccountSigner(tt.Ctx, account1.AccountID, account3.AccountID, 1, nil) tt.Assert.NoError(err) - _, err = q.CreateAccountSigner(tt.Ctx, account2.Data.Account.AccountId.Address(), account3.Data.Account.AccountId.Address(), 1, nil) + _, err = q.CreateAccountSigner(tt.Ctx, account2.AccountID, account3.AccountID, 1, nil) tt.Assert.NoError(err) pq := db2.PageQuery{ @@ -395,11 +274,11 @@ func TestAccountsForSponsor(t *testing.T) { Cursor: "", } - accounts, err := q.AccountsForSponsor(tt.Ctx, sponsor.Address(), pq) + accounts, err := q.AccountsForSponsor(tt.Ctx, sponsor, pq) assert.NoError(t, err) tt.Assert.Len(accounts, 2) - tt.Assert.Equal(account1.Data.Account.AccountId.Address(), accounts[0].AccountID) - tt.Assert.Equal(account2.Data.Account.AccountId.Address(), accounts[1].AccountID) + tt.Assert.Equal(account1.AccountID, accounts[0].AccountID) + tt.Assert.Equal(account2.AccountID, accounts[1].AccountID) } func TestAccountEntriesForSigner(t *testing.T) { @@ -408,32 +287,26 @@ func TestAccountEntriesForSigner(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - eurTrustLine.Data.TrustLine.AccountId = account1.Data.Account.AccountId - usdTrustLine.Data.TrustLine.AccountId = account2.Data.Account.AccountId + eurTrustLine.Data.TrustLine.AccountId = xdr.MustAddress(account1.AccountID) + usdTrustLine.Data.TrustLine.AccountId = xdr.MustAddress(account2.AccountID) - batch := q.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, account1) - assert.NoError(t, err) - err = batch.Add(tt.Ctx, account2) - assert.NoError(t, err) - err = batch.Add(tt.Ctx, account3) + err := q.UpsertAccounts(tt.Ctx, []AccountEntry{account1, account2, account3}) assert.NoError(t, err) - assert.NoError(t, batch.Exec(tt.Ctx)) _, err = q.InsertTrustLine(tt.Ctx, eurTrustLine) tt.Assert.NoError(err) _, err = q.InsertTrustLine(tt.Ctx, usdTrustLine) tt.Assert.NoError(err) - _, err = q.CreateAccountSigner(tt.Ctx, account1.Data.Account.AccountId.Address(), account1.Data.Account.AccountId.Address(), 1, nil) + _, err = q.CreateAccountSigner(tt.Ctx, account1.AccountID, account1.AccountID, 1, nil) tt.Assert.NoError(err) - _, err = q.CreateAccountSigner(tt.Ctx, account2.Data.Account.AccountId.Address(), account2.Data.Account.AccountId.Address(), 1, nil) + _, err = q.CreateAccountSigner(tt.Ctx, account2.AccountID, account2.AccountID, 1, nil) tt.Assert.NoError(err) - _, err = q.CreateAccountSigner(tt.Ctx, account3.Data.Account.AccountId.Address(), account3.Data.Account.AccountId.Address(), 1, nil) + _, err = q.CreateAccountSigner(tt.Ctx, account3.AccountID, account3.AccountID, 1, nil) tt.Assert.NoError(err) - _, err = q.CreateAccountSigner(tt.Ctx, account1.Data.Account.AccountId.Address(), account3.Data.Account.AccountId.Address(), 1, nil) + _, err = q.CreateAccountSigner(tt.Ctx, account1.AccountID, account3.AccountID, 1, nil) tt.Assert.NoError(err) - _, err = q.CreateAccountSigner(tt.Ctx, account2.Data.Account.AccountId.Address(), account3.Data.Account.AccountId.Address(), 1, nil) + _, err = q.CreateAccountSigner(tt.Ctx, account2.AccountID, account3.AccountID, 1, nil) tt.Assert.NoError(err) pq := db2.PageQuery{ @@ -442,23 +315,23 @@ func TestAccountEntriesForSigner(t *testing.T) { Cursor: "", } - accounts, err := q.AccountEntriesForSigner(tt.Ctx, account1.Data.Account.AccountId.Address(), pq) + accounts, err := q.AccountEntriesForSigner(tt.Ctx, account1.AccountID, pq) assert.NoError(t, err) tt.Assert.Len(accounts, 1) - tt.Assert.Equal(account1.Data.Account.AccountId.Address(), accounts[0].AccountID) + tt.Assert.Equal(account1.AccountID, accounts[0].AccountID) - accounts, err = q.AccountEntriesForSigner(tt.Ctx, account2.Data.Account.AccountId.Address(), pq) + accounts, err = q.AccountEntriesForSigner(tt.Ctx, account2.AccountID, pq) assert.NoError(t, err) tt.Assert.Len(accounts, 1) - tt.Assert.Equal(account2.Data.Account.AccountId.Address(), accounts[0].AccountID) + tt.Assert.Equal(account2.AccountID, accounts[0].AccountID) want := map[string]bool{ - account1.Data.Account.AccountId.Address(): true, - account2.Data.Account.AccountId.Address(): true, - account3.Data.Account.AccountId.Address(): true, + account1.AccountID: true, + account2.AccountID: true, + account3.AccountID: true, } - accounts, err = q.AccountEntriesForSigner(tt.Ctx, account3.Data.Account.AccountId.Address(), pq) + accounts, err = q.AccountEntriesForSigner(tt.Ctx, account3.AccountID, pq) assert.NoError(t, err) tt.Assert.Len(accounts, 3) @@ -470,12 +343,12 @@ func TestAccountEntriesForSigner(t *testing.T) { tt.Assert.Len(want, 0) pq.Cursor = accounts[len(accounts)-1].AccountID - accounts, err = q.AccountEntriesForSigner(tt.Ctx, account3.Data.Account.AccountId.Address(), pq) + accounts, err = q.AccountEntriesForSigner(tt.Ctx, account3.AccountID, pq) assert.NoError(t, err) tt.Assert.Len(accounts, 0) pq.Order = "desc" - accounts, err = q.AccountEntriesForSigner(tt.Ctx, account3.Data.Account.AccountId.Address(), pq) + accounts, err = q.AccountEntriesForSigner(tt.Ctx, account3.AccountID, pq) assert.NoError(t, err) tt.Assert.Len(accounts, 2) @@ -485,7 +358,7 @@ func TestAccountEntriesForSigner(t *testing.T) { Cursor: "", } - accounts, err = q.AccountEntriesForSigner(tt.Ctx, account1.Data.Account.AccountId.Address(), pq) + accounts, err = q.AccountEntriesForSigner(tt.Ctx, account1.AccountID, pq) assert.NoError(t, err) tt.Assert.Len(accounts, 1) } @@ -496,12 +369,10 @@ func TestGetAccountByID(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - batch := q.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, account1) + err := q.UpsertAccounts(tt.Ctx, []AccountEntry{account1}) assert.NoError(t, err) - assert.NoError(t, batch.Exec(tt.Ctx)) - resultAccount, err := q.GetAccountByID(tt.Ctx, account1.Data.Account.AccountId.Address()) + resultAccount, err := q.GetAccountByID(tt.Ctx, account1.AccountID) assert.NoError(t, err) assert.Equal(t, "GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB", resultAccount.AccountID) diff --git a/services/horizon/internal/db2/history/main.go b/services/horizon/internal/db2/history/main.go index 4ec9aa36d9..ec496c7ff1 100644 --- a/services/horizon/internal/db2/history/main.go +++ b/services/horizon/internal/db2/history/main.go @@ -217,11 +217,6 @@ type AccountEntry struct { NumSponsoring uint32 `db:"num_sponsoring"` } -type AccountsBatchInsertBuilder interface { - Add(ctx context.Context, entry xdr.LedgerEntry) error - Exec(ctx context.Context) error -} - type IngestionQ interface { QAccounts QAssetStats @@ -264,10 +259,9 @@ type IngestionQ interface { // QAccounts defines account related queries. type QAccounts interface { - NewAccountsBatchInsertBuilder(maxBatchSize int) AccountsBatchInsertBuilder GetAccountsByIDs(ctx context.Context, ids []string) ([]AccountEntry, error) - UpsertAccounts(ctx context.Context, accounts []xdr.LedgerEntry) error - RemoveAccount(ctx context.Context, accountID string) (int64, error) + UpsertAccounts(ctx context.Context, accounts []AccountEntry) error + RemoveAccounts(ctx context.Context, accountIDs []string) (int64, error) } // AccountSigner is a row of data from the `accounts_signers` table @@ -738,15 +732,6 @@ type trustLinesBatchInsertBuilder struct { builder db.BatchInsertBuilder } -func (q *Q) NewAccountsBatchInsertBuilder(maxBatchSize int) AccountsBatchInsertBuilder { - return &accountsBatchInsertBuilder{ - builder: db.BatchInsertBuilder{ - Table: q.GetTable("accounts"), - MaxBatchSize: maxBatchSize, - }, - } -} - func (q *Q) NewAccountSignersBatchInsertBuilder(maxBatchSize int) AccountSignersBatchInsertBuilder { return &accountSignersBatchInsertBuilder{ builder: db.BatchInsertBuilder{ diff --git a/services/horizon/internal/db2/history/mock_q_accounts.go b/services/horizon/internal/db2/history/mock_q_accounts.go index 3db8f7c91d..99f793d147 100644 --- a/services/horizon/internal/db2/history/mock_q_accounts.go +++ b/services/horizon/internal/db2/history/mock_q_accounts.go @@ -2,9 +2,8 @@ package history import ( "context" - "github.com/stretchr/testify/mock" - "github.com/stellar/go/xdr" + "github.com/stretchr/testify/mock" ) // MockQAccounts is a mock implementation of the QAccounts interface @@ -17,17 +16,12 @@ func (m *MockQAccounts) GetAccountsByIDs(ctx context.Context, ids []string) ([]A return a.Get(0).([]AccountEntry), a.Error(1) } -func (m *MockQAccounts) NewAccountsBatchInsertBuilder(maxBatchSize int) AccountsBatchInsertBuilder { - a := m.Called(maxBatchSize) - return a.Get(0).(AccountsBatchInsertBuilder) -} - -func (m *MockQAccounts) UpsertAccounts(ctx context.Context, accounts []xdr.LedgerEntry) error { +func (m *MockQAccounts) UpsertAccounts(ctx context.Context, accounts []AccountEntry) error { a := m.Called(ctx, accounts) return a.Error(0) } -func (m *MockQAccounts) RemoveAccount(ctx context.Context, accountID string) (int64, error) { - a := m.Called(ctx, accountID) +func (m *MockQAccounts) RemoveAccounts(ctx context.Context, accountIDs []string) (int64, error) { + a := m.Called(ctx, accountIDs) return a.Get(0).(int64), a.Error(1) } diff --git a/services/horizon/internal/db2/history/offers_test.go b/services/horizon/internal/db2/history/offers_test.go index 61f228e33a..733e61ce88 100644 --- a/services/horizon/internal/db2/history/offers_test.go +++ b/services/horizon/internal/db2/history/offers_test.go @@ -31,7 +31,7 @@ var ( Price: float64(1), Flags: 1, LastModifiedLedger: uint32(1234), - Sponsor: null.StringFrom(sponsor.Address()), + Sponsor: null.StringFrom(sponsor), } twoEurOffer = Offer{ SellerID: twoEurOfferSeller.Address(), @@ -359,7 +359,7 @@ func TestGetOffers(t *testing.T) { t.Run("Filter by sponsor", func(t *testing.T) { query := OffersQuery{ PageQuery: pageQuery, - Sponsor: sponsor.Address(), + Sponsor: sponsor, } offers, err := q.GetOffers(tt.Ctx, query) diff --git a/services/horizon/internal/db2/history/sequence_provider_test.go b/services/horizon/internal/db2/history/sequence_provider_test.go index 5ed67d7731..086c471be1 100644 --- a/services/horizon/internal/db2/history/sequence_provider_test.go +++ b/services/horizon/internal/db2/history/sequence_provider_test.go @@ -28,12 +28,8 @@ func TestSequenceProviderGet(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - batch := q.NewAccountsBatchInsertBuilder(0) - err := batch.Add(tt.Ctx, account1) + err := q.UpsertAccounts(tt.Ctx, []AccountEntry{account1, account2}) assert.NoError(t, err) - err = batch.Add(tt.Ctx, account2) - assert.NoError(t, err) - assert.NoError(t, batch.Exec(tt.Ctx)) results, err := q.GetSequenceNumbers(tt.Ctx, []string{ "GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB", @@ -42,6 +38,6 @@ func TestSequenceProviderGet(t *testing.T) { }) assert.NoError(t, err) assert.Len(t, results, 2) - assert.Equal(t, uint64(account1.Data.Account.SeqNum), results["GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"]) - assert.Equal(t, uint64(account2.Data.Account.SeqNum), results["GCT2NQM5KJJEF55NPMY444C6M6CA7T33HRNCMA6ZFBIIXKNCRO6J25K7"]) + assert.Equal(t, uint64(account1.SequenceNumber), results["GAOQJGUAB7NI7K7I62ORBXMN3J4SSWQUQ7FOEPSDJ322W2HMCNWPHXFB"]) + assert.Equal(t, uint64(account2.SequenceNumber), results["GCT2NQM5KJJEF55NPMY444C6M6CA7T33HRNCMA6ZFBIIXKNCRO6J25K7"]) } diff --git a/services/horizon/internal/db2/history/trust_lines_test.go b/services/horizon/internal/db2/history/trust_lines_test.go index 289898a62b..54880a27c5 100644 --- a/services/horizon/internal/db2/history/trust_lines_test.go +++ b/services/horizon/internal/db2/history/trust_lines_test.go @@ -13,13 +13,14 @@ import ( var ( trustLineIssuer = xdr.MustAddress("GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H") + sponsorXDR = xdr.MustAddress(sponsor) eurTrustLine = xdr.LedgerEntry{ LastModifiedLedgerSeq: 1234, Data: xdr.LedgerEntryData{ Type: xdr.LedgerEntryTypeTrustline, TrustLine: &xdr.TrustLineEntry{ - AccountId: account1.Data.Account.AccountId, + AccountId: xdr.MustAddress(account1.AccountID), Asset: xdr.MustNewCreditAsset("EUR", trustLineIssuer.Address()).ToTrustLineAsset(), Balance: 30000, Limit: 223456789, @@ -38,7 +39,7 @@ var ( Ext: xdr.LedgerEntryExt{ V: 1, V1: &xdr.LedgerEntryExtensionV1{ - SponsoringId: &sponsor, + SponsoringId: &sponsorXDR, }, }, } @@ -125,7 +126,7 @@ func TestInsertTrustLine(t *testing.T) { assert.NoError(t, err) assert.Len(t, lines, 2) - assert.Equal(t, null.StringFrom(sponsor.Address()), lines[0].Sponsor) + assert.Equal(t, null.StringFrom(sponsor), lines[0].Sponsor) } func TestUpdateTrustLine(t *testing.T) { @@ -253,7 +254,7 @@ func TestUpsertTrustLines(t *testing.T) { Ext: xdr.LedgerEntryExt{ V: 1, V1: &xdr.LedgerEntryExtensionV1{ - SponsoringId: &sponsor, + SponsoringId: &sponsorXDR, }, }, } @@ -417,7 +418,7 @@ func TestAssetsForAddress(t *testing.T) { test.ResetHorizonDB(t, tt.HorizonDB) q := &Q{tt.HorizonSession()} - ledgerEntries := []xdr.LedgerEntry{account1} + ledgerEntries := []AccountEntry{account1} err := q.UpsertAccounts(tt.Ctx, ledgerEntries) assert.NoError(t, err) @@ -430,7 +431,7 @@ func TestAssetsForAddress(t *testing.T) { Data: xdr.LedgerEntryData{ Type: xdr.LedgerEntryTypeTrustline, TrustLine: &xdr.TrustLineEntry{ - AccountId: account1.Data.Account.AccountId, + AccountId: xdr.MustAddress(account1.AccountID), Asset: xdr.MustNewCreditAsset("BRL", trustLineIssuer.Address()).ToTrustLineAsset(), Balance: 1000, Limit: 20000, @@ -463,7 +464,7 @@ func TestAssetsForAddress(t *testing.T) { tt.Assert.Empty(assets) tt.Assert.Empty(balances) - assets, balances, err = q.AssetsForAddress(tt.Ctx, account1.Data.Account.AccountId.Address()) + assets, balances, err = q.AssetsForAddress(tt.Ctx, account1.AccountID) tt.Assert.NoError(err) assetsToBalance := map[string]xdr.Int64{} diff --git a/services/horizon/internal/ingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go index a2c4975bd4..f3090c0d5d 100644 --- a/services/horizon/internal/ingest/processor_runner_test.go +++ b/services/horizon/internal/ingest/processor_runner_test.go @@ -36,18 +36,13 @@ func TestProcessorRunnerRunHistoryArchiveIngestionGenesis(t *testing.T) { q.MockQClaimableBalances.On("NewClaimableBalancesBatchInsertBuilder", maxBatchSize). Return(mockClaimableBalancesBatchInsertBuilder).Once() - q.MockQAccounts.On("UpsertAccounts", ctx, []xdr.LedgerEntry{ + q.MockQAccounts.On("UpsertAccounts", ctx, []history.AccountEntry{ { - LastModifiedLedgerSeq: 1, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &xdr.AccountEntry{ - AccountId: xdr.MustAddress("GAAZI4TCR3TY5OJHCTJC2A4QSY6CJWJH5IAJTGKIN2ER7LBNVKOCCWN7"), - Balance: xdr.Int64(1000000000000000000), - SeqNum: xdr.SequenceNumber(0), - Thresholds: [4]byte{1, 0, 0, 0}, - }, - }, + LastModifiedLedger: 1, + AccountID: "GAAZI4TCR3TY5OJHCTJC2A4QSY6CJWJH5IAJTGKIN2ER7LBNVKOCCWN7", + Balance: int64(1000000000000000000), + SequenceNumber: 0, + MasterWeight: 1, }, }).Return(nil).Once() @@ -119,18 +114,13 @@ func TestProcessorRunnerRunHistoryArchiveIngestionHistoryArchive(t *testing.T) { q.MockQClaimableBalances.On("NewClaimableBalancesBatchInsertBuilder", maxBatchSize). Return(mockClaimableBalancesBatchInsertBuilder).Once() - q.MockQAccounts.On("UpsertAccounts", ctx, []xdr.LedgerEntry{ + q.MockQAccounts.On("UpsertAccounts", ctx, []history.AccountEntry{ { - LastModifiedLedgerSeq: 1, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &xdr.AccountEntry{ - AccountId: xdr.MustAddress("GAAZI4TCR3TY5OJHCTJC2A4QSY6CJWJH5IAJTGKIN2ER7LBNVKOCCWN7"), - Balance: xdr.Int64(1000000000000000000), - SeqNum: xdr.SequenceNumber(0), - Thresholds: [4]byte{1, 0, 0, 0}, - }, - }, + LastModifiedLedger: 1, + AccountID: "GAAZI4TCR3TY5OJHCTJC2A4QSY6CJWJH5IAJTGKIN2ER7LBNVKOCCWN7", + Balance: int64(1000000000000000000), + SequenceNumber: 0, + MasterWeight: 1, }, }).Return(nil).Once() diff --git a/services/horizon/internal/ingest/processors/accounts_processor.go b/services/horizon/internal/ingest/processors/accounts_processor.go index 289ad23de5..7ea97bc6f1 100644 --- a/services/horizon/internal/ingest/processors/accounts_processor.go +++ b/services/horizon/internal/ingest/processors/accounts_processor.go @@ -47,7 +47,8 @@ func (p *AccountsProcessor) ProcessChange(ctx context.Context, change ingest.Cha } func (p *AccountsProcessor) Commit(ctx context.Context) error { - batchUpsertAccounts := []xdr.LedgerEntry{} + batchUpsertAccounts := []history.AccountEntry{} + removeBatch := []string{} changes := p.cache.GetChanges() for _, change := range changes { @@ -63,24 +64,13 @@ func (p *AccountsProcessor) Commit(ctx context.Context) error { switch { case change.Post != nil: // Created and updated - batchUpsertAccounts = append(batchUpsertAccounts, *change.Post) + row := p.ledgerEntryToRow(*change.Post) + batchUpsertAccounts = append(batchUpsertAccounts, row) case change.Pre != nil && change.Post == nil: // Removed account := change.Pre.Data.MustAccount() accountID := account.AccountId.Address() - rowsAffected, err := p.accountsQ.RemoveAccount(ctx, accountID) - - if err != nil { - return err - } - - if rowsAffected != 1 { - return ingest.NewStateError(errors.Errorf( - "%d No rows affected when removing account %s", - rowsAffected, - accountID, - )) - } + removeBatch = append(removeBatch, accountID) default: return errors.New("Invalid io.Change: change.Pre == nil && change.Post == nil") } @@ -94,5 +84,50 @@ func (p *AccountsProcessor) Commit(ctx context.Context) error { } } + if len(removeBatch) > 0 { + rowsAffected, err := p.accountsQ.RemoveAccounts(ctx, removeBatch) + if err != nil { + return errors.Wrap(err, "error in RemoveAccounts") + } + + if rowsAffected != int64(len(removeBatch)) { + return ingest.NewStateError(errors.Errorf( + "%d rows affected when removing %d accounts", + rowsAffected, + len(removeBatch), + )) + } + } + return nil } + +func (p *AccountsProcessor) ledgerEntryToRow(entry xdr.LedgerEntry) history.AccountEntry { + account := entry.Data.MustAccount() + liabilities := account.Liabilities() + + var inflationDestination = "" + if account.InflationDest != nil { + inflationDestination = account.InflationDest.Address() + } + + return history.AccountEntry{ + AccountID: account.AccountId.Address(), + Balance: int64(account.Balance), + BuyingLiabilities: int64(liabilities.Buying), + SellingLiabilities: int64(liabilities.Selling), + SequenceNumber: int64(account.SeqNum), + NumSubEntries: uint32(account.NumSubEntries), + InflationDestination: inflationDestination, + Flags: uint32(account.Flags), + HomeDomain: string(account.HomeDomain), + MasterWeight: account.MasterKeyWeight(), + ThresholdLow: account.ThresholdLow(), + ThresholdMedium: account.ThresholdMedium(), + ThresholdHigh: account.ThresholdHigh(), + LastModifiedLedger: uint32(entry.LastModifiedLedgerSeq), + Sponsor: ledgerEntrySponsorToNullString(entry), + NumSponsored: uint32(account.NumSponsored()), + NumSponsoring: uint32(account.NumSponsoring()), + } +} diff --git a/services/horizon/internal/ingest/processors/accounts_processor_test.go b/services/horizon/internal/ingest/processors/accounts_processor_test.go index 65dc2997e9..ea9fe91125 100644 --- a/services/horizon/internal/ingest/processors/accounts_processor_test.go +++ b/services/horizon/internal/ingest/processors/accounts_processor_test.go @@ -40,22 +40,17 @@ func (s *AccountsProcessorTestSuiteState) TestNoEntries() { } func (s *AccountsProcessorTestSuiteState) TestCreatesAccounts() { - account := xdr.AccountEntry{ - AccountId: xdr.MustAddress("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), - Thresholds: [4]byte{1, 1, 1, 1}, - } - lastModifiedLedgerSeq := xdr.Uint32(123) - // We use LedgerEntryChangesCache so all changes are squashed s.mockQ.On( "UpsertAccounts", s.ctx, - []xdr.LedgerEntry{ + []history.AccountEntry{ { - LastModifiedLedgerSeq: lastModifiedLedgerSeq, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &account, - }, + LastModifiedLedger: 123, + AccountID: "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML", + MasterWeight: 1, + ThresholdLow: 1, + ThresholdMedium: 1, + ThresholdHigh: 1, }, }, ).Return(nil).Once() @@ -65,10 +60,13 @@ func (s *AccountsProcessorTestSuiteState) TestCreatesAccounts() { Pre: nil, Post: &xdr.LedgerEntry{ Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &account, + Type: xdr.LedgerEntryTypeAccount, + Account: &xdr.AccountEntry{ + AccountId: xdr.MustAddress("GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"), + Thresholds: [4]byte{1, 1, 1, 1}, + }, }, - LastModifiedLedgerSeq: lastModifiedLedgerSeq, + LastModifiedLedgerSeq: xdr.Uint32(123), }, }) s.Assert().NoError(err) @@ -150,13 +148,15 @@ func (s *AccountsProcessorTestSuiteLedger) TestNewAccount() { s.mockQ.On( "UpsertAccounts", s.ctx, - []xdr.LedgerEntry{ + []history.AccountEntry{ { - LastModifiedLedgerSeq: lastModifiedLedgerSeq, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &updatedAccount, - }, + LastModifiedLedger: 123, + AccountID: "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML", + MasterWeight: 0, + ThresholdLow: 1, + ThresholdMedium: 2, + ThresholdHigh: 3, + HomeDomain: "stellar.org", }, }, ).Return(nil).Once() @@ -164,9 +164,9 @@ func (s *AccountsProcessorTestSuiteLedger) TestNewAccount() { func (s *AccountsProcessorTestSuiteLedger) TestRemoveAccount() { s.mockQ.On( - "RemoveAccount", + "RemoveAccounts", s.ctx, - "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML", + []string{"GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML"}, ).Return(int64(1), nil).Once() err := s.processor.ProcessChange(s.ctx, ingest.Change{ @@ -233,13 +233,15 @@ func (s *AccountsProcessorTestSuiteLedger) TestProcessUpgradeChange() { s.mockQ.On( "UpsertAccounts", s.ctx, - []xdr.LedgerEntry{ + []history.AccountEntry{ { - LastModifiedLedgerSeq: lastModifiedLedgerSeq + 1, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &updatedAccount, - }, + LastModifiedLedger: uint32(lastModifiedLedgerSeq) + 1, + AccountID: "GC3C4AKRBQLHOJ45U4XG35ESVWRDECWO5XLDGYADO6DPR3L7KIDVUMML", + MasterWeight: 0, + ThresholdLow: 1, + ThresholdMedium: 2, + ThresholdHigh: 3, + HomeDomain: "stellar.org", }, }, ).Return(nil).Once() @@ -292,21 +294,14 @@ func (s *AccountsProcessorTestSuiteLedger) TestFeeProcessedBeforeEverythingElse( }) s.Assert().NoError(err) - expectedAccount := xdr.AccountEntry{ - AccountId: xdr.MustAddress("GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A"), - Balance: 300, - } - s.mockQ.On( "UpsertAccounts", s.ctx, - []xdr.LedgerEntry{ + []history.AccountEntry{ { - LastModifiedLedgerSeq: 0, - Data: xdr.LedgerEntryData{ - Type: xdr.LedgerEntryTypeAccount, - Account: &expectedAccount, - }, + LastModifiedLedger: 0, + AccountID: "GAHK7EEG2WWHVKDNT4CEQFZGKF2LGDSW2IVM4S5DP42RBW3K6BTODB4A", + Balance: 300, }, }, ).Return(nil).Once() From 139a2bd9e5e9013e8877c7de3e1d9a61f40b4f35 Mon Sep 17 00:00:00 2001 From: Bartek Nowotarski Date: Mon, 20 Sep 2021 20:54:41 +0200 Subject: [PATCH 2/4] fix --- .vscode/tasks.json | 18 +++++++++--------- .../horizon/internal/db2/history/accounts.go | 12 ------------ 2 files changed, 9 insertions(+), 21 deletions(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 0079459abc..d862336cf2 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -1,23 +1,23 @@ { // See https://go.microsoft.com/fwlink/?LinkId=733558 // for the documentation about the tasks.json format - "version": "2.0.0", + "version": "0.1.0", "command": "go", "isShellCommand": true, + "showOutput": "always", "suppressTaskName": true, "options": { "cwd": "${workspaceRoot}" }, "tasks": [ { - "label": "go", + "label": "test all", "type": "shell", - "command": "go", - "problemMatcher": [], - "group": { - "_id": "build", - "isDefault": false - } + "args": [ + "test", + "./..." + ], + "problemMatcher": [] } ] -} +} \ No newline at end of file diff --git a/services/horizon/internal/db2/history/accounts.go b/services/horizon/internal/db2/history/accounts.go index 695374d3c7..f52070c03e 100644 --- a/services/horizon/internal/db2/history/accounts.go +++ b/services/horizon/internal/db2/history/accounts.go @@ -112,18 +112,6 @@ func (q *Q) UpsertAccounts(ctx context.Context, accounts []AccountEntry) error { return q.upsertRows(ctx, "accounts", "account_id", upsertFields) } -// RemoveAccount deletes a row in the accounts table. -// Returns number of rows affected and error. -func (q *Q) RemoveAccount(ctx context.Context, accountID string) (int64, error) { - sql := sq.Delete("accounts").Where(sq.Eq{"account_id": accountID}) - result, err := q.Exec(ctx, sql) - if err != nil { - return 0, err - } - - return result.RowsAffected() -} - // RemoveAccounts deletes a row in the accounts table. // Returns number of rows affected and error. func (q *Q) RemoveAccounts(ctx context.Context, accountIDs []string) (int64, error) { From 6956ea006eeae5c95f6a63bb9b02c63eeaa96523 Mon Sep 17 00:00:00 2001 From: Bartek Nowotarski Date: Mon, 20 Sep 2021 20:56:28 +0200 Subject: [PATCH 3/4] fix --- .vscode/tasks.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index d862336cf2..470c1cd251 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -20,4 +20,5 @@ "problemMatcher": [] } ] -} \ No newline at end of file +} + From 4bb720bb73c24d2380ff7722ede537e4866db332 Mon Sep 17 00:00:00 2001 From: Bartek Nowotarski Date: Mon, 20 Sep 2021 20:57:29 +0200 Subject: [PATCH 4/4] fix --- .vscode/tasks.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.vscode/tasks.json b/.vscode/tasks.json index 470c1cd251..b6291051e5 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -21,4 +21,3 @@ } ] } -