From 9bc197d1de32e98c0360946aefd1f46804cd6119 Mon Sep 17 00:00:00 2001
From: tamirms <tamirms@gmail.com>
Date: Wed, 5 Jul 2023 09:47:36 +0100
Subject: [PATCH 01/17] support/db: Add batch insert builder which uses COPY to
 insert rows (#4916)

* Add batch insert builder which uses COPY to insert rows

* Update support/db/fast_batch_insert_builder.go

Co-authored-by: George <Shaptic@users.noreply.github.com>

* Update fast_batch_insert_builder_test.go

---------

Co-authored-by: George <Shaptic@users.noreply.github.com>
---
 support/db/fast_batch_insert_builder.go      | 150 +++++++++++++++++++
 support/db/fast_batch_insert_builder_test.go | 127 ++++++++++++++++
 2 files changed, 277 insertions(+)
 create mode 100644 support/db/fast_batch_insert_builder.go
 create mode 100644 support/db/fast_batch_insert_builder_test.go

diff --git a/support/db/fast_batch_insert_builder.go b/support/db/fast_batch_insert_builder.go
new file mode 100644
index 0000000000..ec235ee31d
--- /dev/null
+++ b/support/db/fast_batch_insert_builder.go
@@ -0,0 +1,150 @@
+package db
+
+import (
+	"context"
+	"reflect"
+	"sort"
+
+	"github.com/lib/pq"
+
+	"github.com/stellar/go/support/errors"
+)
+
+// ErrSealed is returned when trying to add rows to the FastBatchInsertBuilder after Exec() is called.
+// Once Exec() is called no more rows can be added to the FastBatchInsertBuilder unless you call Reset()
+// which clears out the old rows from the FastBatchInsertBuilder.
+var ErrSealed = errors.New("cannot add more rows after Exec() without calling Reset() first")
+
+// ErrNoTx is returned when Exec() is called outside of a transaction.
+var ErrNoTx = errors.New("cannot call Exec() outside of a transaction")
+
+// FastBatchInsertBuilder works like sq.InsertBuilder but has a better support for batching
+// large number of rows.
+// It is NOT safe for concurrent use.
+// It does NOT support updating existing rows.
+type FastBatchInsertBuilder struct {
+	columns       []string
+	rows          [][]interface{}
+	rowStructType reflect.Type
+	sealed        bool
+}
+
+// Row adds a new row to the batch. All rows must have exactly the same columns
+// (map keys). Otherwise, error will be returned. Please note that rows are not
+// added one by one but in batches when `Exec` is called.
+func (b *FastBatchInsertBuilder) Row(row map[string]interface{}) error {
+	if b.sealed {
+		return ErrSealed
+	}
+
+	if b.columns == nil {
+		b.columns = make([]string, 0, len(row))
+		b.rows = make([][]interface{}, 0)
+
+		for column := range row {
+			b.columns = append(b.columns, column)
+		}
+
+		sort.Strings(b.columns)
+	}
+
+	if len(b.columns) != len(row) {
+		return errors.Errorf("invalid number of columns (expected=%d, actual=%d)", len(b.columns), len(row))
+	}
+
+	rowSlice := make([]interface{}, 0, len(b.columns))
+	for _, column := range b.columns {
+		val, ok := row[column]
+		if !ok {
+			return errors.Errorf(`column "%s" does not exist`, column)
+		}
+		rowSlice = append(rowSlice, val)
+	}
+	b.rows = append(b.rows, rowSlice)
+
+	return nil
+}
+
+// RowStruct adds a new row to the batch. All rows must have exactly the same columns
+// (map keys). Otherwise, error will be returned. Please note that rows are not
+// added one by one but in batches when `Exec` is called.
+func (b *FastBatchInsertBuilder) RowStruct(row interface{}) error {
+	if b.sealed {
+		return ErrSealed
+	}
+
+	if b.columns == nil {
+		b.columns = ColumnsForStruct(row)
+		b.rows = make([][]interface{}, 0)
+	}
+
+	rowType := reflect.TypeOf(row)
+	if b.rowStructType == nil {
+		b.rowStructType = rowType
+	} else if b.rowStructType != rowType {
+		return errors.Errorf(`expected value of type "%s" but got "%s" value`, b.rowStructType.String(), rowType.String())
+	}
+
+	rrow := reflect.ValueOf(row)
+	rvals := mapper.FieldsByName(rrow, b.columns)
+
+	// convert fields values to interface{}
+	columnValues := make([]interface{}, len(b.columns))
+	for i, rval := range rvals {
+		columnValues[i] = rval.Interface()
+	}
+
+	b.rows = append(b.rows, columnValues)
+
+	return nil
+}
+
+// Len returns the number of rows held in memory by the FastBatchInsertBuilder.
+func (b *FastBatchInsertBuilder) Len() int {
+	return len(b.rows)
+}
+
+// Exec inserts rows in a single COPY statement. Once Exec is called no more rows
+// can be added to the FastBatchInsertBuilder unless Reset is called.
+// Exec must be called within a transaction.
+func (b *FastBatchInsertBuilder) Exec(ctx context.Context, session SessionInterface, tableName string) error {
+	b.sealed = true
+	if session.GetTx() == nil {
+		return ErrNoTx
+	}
+
+	if len(b.rows) == 0 {
+		return nil
+	}
+
+	tx := session.GetTx()
+	stmt, err := tx.PrepareContext(ctx, pq.CopyIn(tableName, b.columns...))
+	if err != nil {
+		return err
+	}
+
+	for _, row := range b.rows {
+		if _, err = stmt.ExecContext(ctx, row...); err != nil {
+			// we need to close the statement otherwise the session
+			// will always return bad connection errors when executing
+			// any other sql statements,
+			// see https://github.com/stellar/go/pull/316#issuecomment-368990324
+			stmt.Close()
+			return err
+		}
+	}
+
+	if err = stmt.Close(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Reset clears out all the rows contained in the FastBatchInsertBuilder.
+// After Reset is called new rows can be added to the FastBatchInsertBuilder.
+func (b *FastBatchInsertBuilder) Reset() {
+	b.sealed = false
+	b.columns = nil
+	b.rows = nil
+	b.rowStructType = nil
+}
diff --git a/support/db/fast_batch_insert_builder_test.go b/support/db/fast_batch_insert_builder_test.go
new file mode 100644
index 0000000000..c31f502735
--- /dev/null
+++ b/support/db/fast_batch_insert_builder_test.go
@@ -0,0 +1,127 @@
+package db
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/stellar/go/support/db/dbtest"
+)
+
+func TestFastBatchInsertBuilder(t *testing.T) {
+	db := dbtest.Postgres(t).Load(testSchema)
+	defer db.Close()
+	sess := &Session{DB: db.Open()}
+	defer sess.DB.Close()
+
+	insertBuilder := &FastBatchInsertBuilder{}
+
+	assert.NoError(t,
+		insertBuilder.Row(map[string]interface{}{
+			"name":         "bubba",
+			"hunger_level": "1",
+		}),
+	)
+
+	assert.EqualError(t,
+		insertBuilder.Row(map[string]interface{}{
+			"name": "bubba",
+		}),
+		"invalid number of columns (expected=2, actual=1)",
+	)
+
+	assert.EqualError(t,
+		insertBuilder.Row(map[string]interface{}{
+			"name": "bubba",
+			"city": "London",
+		}),
+		"column \"hunger_level\" does not exist",
+	)
+
+	assert.NoError(t,
+		insertBuilder.RowStruct(hungerRow{
+			Name:        "bubba2",
+			HungerLevel: "9",
+		}),
+	)
+
+	assert.EqualError(t,
+		insertBuilder.RowStruct(invalidHungerRow{
+			Name:        "bubba",
+			HungerLevel: "2",
+			LastName:    "b",
+		}),
+		"expected value of type \"db.hungerRow\" but got \"db.invalidHungerRow\" value",
+	)
+	assert.Equal(t, 2, insertBuilder.Len())
+	assert.Equal(t, false, insertBuilder.sealed)
+
+	assert.EqualError(t,
+		insertBuilder.Exec(context.Background(), sess, "people"),
+		"cannot call Exec() outside of a transaction",
+	)
+	assert.Equal(t, true, insertBuilder.sealed)
+
+	assert.NoError(t, sess.Begin())
+	assert.NoError(t, insertBuilder.Exec(context.Background(), sess, "people"))
+	assert.Equal(t, 2, insertBuilder.Len())
+	assert.Equal(t, true, insertBuilder.sealed)
+
+	var found []person
+	assert.NoError(t, sess.SelectRaw(context.Background(), &found, `SELECT * FROM people WHERE name like 'bubba%'`))
+	assert.Equal(
+		t,
+		found,
+		[]person{
+			{Name: "bubba", HungerLevel: "1"},
+			{Name: "bubba2", HungerLevel: "9"},
+		},
+	)
+
+	assert.EqualError(t,
+		insertBuilder.Row(map[string]interface{}{
+			"name":         "bubba3",
+			"hunger_level": "100",
+		}),
+		"cannot add more rows after Exec() without calling Reset() first",
+	)
+	assert.Equal(t, 2, insertBuilder.Len())
+	assert.Equal(t, true, insertBuilder.sealed)
+
+	insertBuilder.Reset()
+	assert.Equal(t, 0, insertBuilder.Len())
+	assert.Equal(t, false, insertBuilder.sealed)
+
+	assert.NoError(t,
+		insertBuilder.Row(map[string]interface{}{
+			"name":         "bubba3",
+			"hunger_level": "3",
+		}),
+	)
+	assert.Equal(t, 1, insertBuilder.Len())
+	assert.Equal(t, false, insertBuilder.sealed)
+
+	ctx, cancel := context.WithCancel(context.Background())
+	cancel()
+	assert.EqualError(t,
+		insertBuilder.Exec(ctx, sess, "people"),
+		"context canceled",
+	)
+	assert.Equal(t, 1, insertBuilder.Len())
+	assert.Equal(t, true, insertBuilder.sealed)
+
+	assert.NoError(t, sess.SelectRaw(context.Background(), &found, `SELECT * FROM people WHERE name like 'bubba%'`))
+	assert.Equal(
+		t,
+		found,
+		[]person{
+			{Name: "bubba", HungerLevel: "1"},
+			{Name: "bubba2", HungerLevel: "9"},
+		},
+	)
+	assert.NoError(t, sess.Rollback())
+
+	assert.NoError(t, sess.SelectRaw(context.Background(), &found, `SELECT * FROM people WHERE name like 'bubba%'`))
+	assert.Empty(t, found)
+}

From b451d60d95f76e7a793f212e15d34f1718be4ef9 Mon Sep 17 00:00:00 2001
From: tamirms <tamirms@gmail.com>
Date: Tue, 11 Jul 2023 20:01:58 +0100
Subject: [PATCH 02/17] Use FastBatchInsertBuilder to insert ledgers into the
 history_ledgers table (#4947)

---
 .../horizon/internal/action_offers_test.go    |  6 ++-
 .../horizon/internal/actions/account_test.go  | 19 +++++--
 .../horizon/internal/actions/offer_test.go    | 16 ++++--
 .../horizon/internal/actions_account_test.go  |  7 ++-
 .../horizon/internal/actions_data_test.go     |  6 ++-
 .../horizon/internal/db2/history/ledger.go    | 46 +++++++++++------
 .../internal/db2/history/ledger_test.go       | 14 ++++--
 .../internal/db2/history/mock_q_ledgers.go    | 27 +++++++---
 .../internal/db2/history/transaction_test.go  |  6 ++-
 services/horizon/internal/ingest/main.go      |  1 +
 .../internal/ingest/processor_runner.go       |  4 +-
 .../internal/ingest/processor_runner_test.go  | 29 +++++++++--
 .../ingest/processors/ledgers_processor.go    | 27 +++-------
 .../processors/ledgers_processor_test.go      | 49 +++++++++----------
 services/horizon/internal/middleware_test.go  |  8 ++-
 15 files changed, 178 insertions(+), 87 deletions(-)

diff --git a/services/horizon/internal/action_offers_test.go b/services/horizon/internal/action_offers_test.go
index 13458db9fe..d10e720636 100644
--- a/services/horizon/internal/action_offers_test.go
+++ b/services/horizon/internal/action_offers_test.go
@@ -24,7 +24,9 @@ func TestOfferActions_Show(t *testing.T) {
 	ht.Assert.NoError(err)
 
 	ledgerCloseTime := time.Now().Unix()
-	_, err = q.InsertLedger(ctx, xdr.LedgerHeaderHistoryEntry{
+	ht.Assert.NoError(q.Begin())
+	ledgerBatch := q.NewLedgerBatchInsertBuilder()
+	err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
 			LedgerSeq: 100,
 			ScpValue: xdr.StellarValue{
@@ -33,6 +35,8 @@ func TestOfferActions_Show(t *testing.T) {
 		},
 	}, 0, 0, 0, 0, 0)
 	ht.Assert.NoError(err)
+	ht.Assert.NoError(ledgerBatch.Exec(ht.Ctx, q))
+	ht.Assert.NoError(q.Commit())
 
 	issuer := xdr.MustAddress("GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H")
 	nativeAsset := xdr.MustNewNativeAsset()
diff --git a/services/horizon/internal/actions/account_test.go b/services/horizon/internal/actions/account_test.go
index ed3d529575..ff265721da 100644
--- a/services/horizon/internal/actions/account_test.go
+++ b/services/horizon/internal/actions/account_test.go
@@ -228,7 +228,9 @@ func TestAccountInfo(t *testing.T) {
 	assert.NoError(t, err)
 
 	ledgerFourCloseTime := time.Now().Unix()
-	_, err = q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{
+	assert.NoError(t, q.Begin())
+	ledgerBatch := q.NewLedgerBatchInsertBuilder()
+	err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
 			LedgerSeq: 4,
 			ScpValue: xdr.StellarValue{
@@ -237,6 +239,8 @@ func TestAccountInfo(t *testing.T) {
 		},
 	}, 0, 0, 0, 0, 0)
 	assert.NoError(t, err)
+	assert.NoError(t, ledgerBatch.Exec(tt.Ctx, q))
+	assert.NoError(t, q.Commit())
 
 	account, err := AccountInfo(tt.Ctx, &history.Q{tt.HorizonSession()}, accountID)
 	tt.Assert.NoError(err)
@@ -408,7 +412,9 @@ func TestGetAccountsHandlerPageResultsByAsset(t *testing.T) {
 	err := q.UpsertAccounts(tt.Ctx, []history.AccountEntry{account1, account2})
 	assert.NoError(t, err)
 	ledgerCloseTime := time.Now().Unix()
-	_, err = q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{
+	assert.NoError(t, q.Begin())
+	ledgerBatch := q.NewLedgerBatchInsertBuilder()
+	err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
 			LedgerSeq: 1234,
 			ScpValue: xdr.StellarValue{
@@ -417,6 +423,8 @@ func TestGetAccountsHandlerPageResultsByAsset(t *testing.T) {
 		},
 	}, 0, 0, 0, 0, 0)
 	assert.NoError(t, err)
+	assert.NoError(t, ledgerBatch.Exec(tt.Ctx, q))
+	assert.NoError(t, q.Commit())
 
 	for _, row := range accountSigners {
 		_, err = q.CreateAccountSigner(tt.Ctx, row.Account, row.Signer, row.Weight, nil)
@@ -511,7 +519,9 @@ func TestGetAccountsHandlerPageResultsByLiquidityPool(t *testing.T) {
 	assert.NoError(t, err)
 
 	ledgerCloseTime := time.Now().Unix()
-	_, err = q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{
+	assert.NoError(t, q.Begin())
+	ledgerBatch := q.NewLedgerBatchInsertBuilder()
+	err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
 			LedgerSeq: 1234,
 			ScpValue: xdr.StellarValue{
@@ -520,6 +530,9 @@ func TestGetAccountsHandlerPageResultsByLiquidityPool(t *testing.T) {
 		},
 	}, 0, 0, 0, 0, 0)
 	assert.NoError(t, err)
+	assert.NoError(t, ledgerBatch.Exec(tt.Ctx, q))
+	assert.NoError(t, q.Commit())
+
 	var assetType, code, issuer string
 	usd.MustExtract(&assetType, &code, &issuer)
 	params := map[string]string{
diff --git a/services/horizon/internal/actions/offer_test.go b/services/horizon/internal/actions/offer_test.go
index 41663c65d5..578e284bc6 100644
--- a/services/horizon/internal/actions/offer_test.go
+++ b/services/horizon/internal/actions/offer_test.go
@@ -79,7 +79,9 @@ func TestGetOfferByIDHandler(t *testing.T) {
 	handler := GetOfferByID{}
 
 	ledgerCloseTime := time.Now().Unix()
-	_, err := q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{
+	assert.NoError(t, q.Begin())
+	ledgerBatch := q.NewLedgerBatchInsertBuilder()
+	err := ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
 			LedgerSeq: 3,
 			ScpValue: xdr.StellarValue{
@@ -87,7 +89,9 @@ func TestGetOfferByIDHandler(t *testing.T) {
 			},
 		},
 	}, 0, 0, 0, 0, 0)
-	tt.Assert.NoError(err)
+	assert.NoError(t, err)
+	assert.NoError(t, ledgerBatch.Exec(tt.Ctx, q))
+	assert.NoError(t, q.Commit())
 
 	err = q.UpsertOffers(tt.Ctx, []history.Offer{eurOffer, usdOffer})
 	tt.Assert.NoError(err)
@@ -186,7 +190,9 @@ func TestGetOffersHandler(t *testing.T) {
 	handler := GetOffersHandler{}
 
 	ledgerCloseTime := time.Now().Unix()
-	_, err := q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{
+	assert.NoError(t, q.Begin())
+	ledgerBatch := q.NewLedgerBatchInsertBuilder()
+	err := ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
 			LedgerSeq: 3,
 			ScpValue: xdr.StellarValue{
@@ -194,7 +200,9 @@ func TestGetOffersHandler(t *testing.T) {
 			},
 		},
 	}, 0, 0, 0, 0, 0)
-	tt.Assert.NoError(err)
+	assert.NoError(t, err)
+	assert.NoError(t, ledgerBatch.Exec(tt.Ctx, q))
+	assert.NoError(t, q.Commit())
 
 	err = q.UpsertOffers(tt.Ctx, []history.Offer{eurOffer, twoEurOffer, usdOffer})
 	tt.Assert.NoError(err)
diff --git a/services/horizon/internal/actions_account_test.go b/services/horizon/internal/actions_account_test.go
index 541300c3a6..e3e71e0b3a 100644
--- a/services/horizon/internal/actions_account_test.go
+++ b/services/horizon/internal/actions_account_test.go
@@ -18,12 +18,17 @@ func TestAccountActions_InvalidID(t *testing.T) {
 	ht.Assert.NoError(err)
 	err = q.UpdateIngestVersion(ht.Ctx, ingest.CurrentVersion)
 	ht.Assert.NoError(err)
-	_, err = q.InsertLedger(ht.Ctx, xdr.LedgerHeaderHistoryEntry{
+
+	ht.Assert.NoError(q.Begin())
+	ledgerBatch := q.NewLedgerBatchInsertBuilder()
+	err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
 			LedgerSeq: 100,
 		},
 	}, 0, 0, 0, 0, 0)
 	ht.Assert.NoError(err)
+	ht.Assert.NoError(ledgerBatch.Exec(ht.Ctx, q))
+	ht.Assert.NoError(q.Commit())
 
 	// existing account
 	w := ht.Get(
diff --git a/services/horizon/internal/actions_data_test.go b/services/horizon/internal/actions_data_test.go
index 3de82a915d..8cdc4a07db 100644
--- a/services/horizon/internal/actions_data_test.go
+++ b/services/horizon/internal/actions_data_test.go
@@ -44,12 +44,16 @@ func TestDataActions_Show(t *testing.T) {
 	ht.Assert.NoError(err)
 	err = q.UpdateIngestVersion(ht.Ctx, ingest.CurrentVersion)
 	ht.Assert.NoError(err)
-	_, err = q.InsertLedger(ht.Ctx, xdr.LedgerHeaderHistoryEntry{
+	ht.Assert.NoError(q.Begin())
+	ledgerBatch := q.NewLedgerBatchInsertBuilder()
+	err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
 			LedgerSeq: 100,
 		},
 	}, 0, 0, 0, 0, 0)
 	ht.Assert.NoError(err)
+	ht.Assert.NoError(ledgerBatch.Exec(ht.Ctx, q))
+	ht.Assert.NoError(q.Commit())
 
 	err = q.UpsertAccountData(ht.Ctx, []history.Data{data1, data2})
 	assert.NoError(t, err)
diff --git a/services/horizon/internal/db2/history/ledger.go b/services/horizon/internal/db2/history/ledger.go
index 7d367a8464..ca89534702 100644
--- a/services/horizon/internal/db2/history/ledger.go
+++ b/services/horizon/internal/db2/history/ledger.go
@@ -10,6 +10,7 @@ import (
 	sq "github.com/Masterminds/squirrel"
 	"github.com/guregu/null"
 	"github.com/stellar/go/services/horizon/internal/db2"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/support/ordered"
 	"github.com/stellar/go/toid"
@@ -90,27 +91,46 @@ func (q *LedgersQ) Select(ctx context.Context, dest interface{}) error {
 
 // QLedgers defines ingestion ledger related queries.
 type QLedgers interface {
-	InsertLedger(
-		ctx context.Context,
+	NewLedgerBatchInsertBuilder() LedgerBatchInsertBuilder
+}
+
+// LedgerBatchInsertBuilder is used to insert ledgers into the
+// history_ledgers table
+type LedgerBatchInsertBuilder interface {
+	Add(
 		ledger xdr.LedgerHeaderHistoryEntry,
 		successTxsCount int,
 		failedTxsCount int,
 		opCount int,
 		txSetOpCount int,
 		ingestVersion int,
-	) (int64, error)
+	) error
+	Exec(ctx context.Context, session db.SessionInterface) error
+}
+
+// ledgerBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder
+type ledgerBatchInsertBuilder struct {
+	builder db.FastBatchInsertBuilder
+	table   string
+}
+
+// NewLedgerBatchInsertBuilder constructs a new EffectBatchInsertBuilder instance
+func (q *Q) NewLedgerBatchInsertBuilder() LedgerBatchInsertBuilder {
+	return &ledgerBatchInsertBuilder{
+		table:   "history_ledgers",
+		builder: db.FastBatchInsertBuilder{},
+	}
 }
 
-// InsertLedger creates a row in the history_ledgers table.
-// Returns number of rows affected and error.
-func (q *Q) InsertLedger(ctx context.Context,
+// Add adds a effect to the batch
+func (i *ledgerBatchInsertBuilder) Add(
 	ledger xdr.LedgerHeaderHistoryEntry,
 	successTxsCount int,
 	failedTxsCount int,
 	opCount int,
 	txSetOpCount int,
 	ingestVersion int,
-) (int64, error) {
+) error {
 	m, err := ledgerHeaderToMap(
 		ledger,
 		successTxsCount,
@@ -120,16 +140,14 @@ func (q *Q) InsertLedger(ctx context.Context,
 		ingestVersion,
 	)
 	if err != nil {
-		return 0, err
+		return err
 	}
 
-	sql := sq.Insert("history_ledgers").SetMap(m)
-	result, err := q.Exec(ctx, sql)
-	if err != nil {
-		return 0, err
-	}
+	return i.builder.Row(m)
+}
 
-	return result.RowsAffected()
+func (i *ledgerBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	return i.builder.Exec(ctx, session, i.table)
 }
 
 // GetLedgerGaps obtains ingestion gaps in the history_ledgers table.
diff --git a/services/horizon/internal/db2/history/ledger_test.go b/services/horizon/internal/db2/history/ledger_test.go
index 4bf6d7b058..c8526fddd6 100644
--- a/services/horizon/internal/db2/history/ledger_test.go
+++ b/services/horizon/internal/db2/history/ledger_test.go
@@ -119,7 +119,8 @@ func TestInsertLedger(t *testing.T) {
 	tt.Assert.NoError(err)
 	expectedLedger.LedgerHeaderXDR = null.NewString(ledgerHeaderBase64, true)
 
-	rowsAffected, err := q.InsertLedger(tt.Ctx,
+	ledgerBatch := q.NewLedgerBatchInsertBuilder()
+	err = ledgerBatch.Add(
 		ledgerEntry,
 		12,
 		3,
@@ -128,7 +129,9 @@ func TestInsertLedger(t *testing.T) {
 		int(expectedLedger.ImporterVersion),
 	)
 	tt.Assert.NoError(err)
-	tt.Assert.Equal(rowsAffected, int64(1))
+	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q.SessionInterface))
+	tt.Assert.NoError(q.Commit())
 
 	err = q.LedgerBySequence(tt.Ctx, &ledgerFromDB, 69859)
 	tt.Assert.NoError(err)
@@ -204,7 +207,8 @@ func insertLedgerWithSequence(tt *test.T, q *Q, seq uint32) {
 	ledgerHeaderBase64, err := xdr.MarshalBase64(ledgerEntry.Header)
 	tt.Assert.NoError(err)
 	expectedLedger.LedgerHeaderXDR = null.NewString(ledgerHeaderBase64, true)
-	rowsAffected, err := q.InsertLedger(tt.Ctx,
+	ledgerBatch := q.NewLedgerBatchInsertBuilder()
+	err = ledgerBatch.Add(
 		ledgerEntry,
 		12,
 		3,
@@ -213,7 +217,9 @@ func insertLedgerWithSequence(tt *test.T, q *Q, seq uint32) {
 		int(expectedLedger.ImporterVersion),
 	)
 	tt.Assert.NoError(err)
-	tt.Assert.Equal(rowsAffected, int64(1))
+	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q.SessionInterface))
+	tt.Assert.NoError(q.Commit())
 }
 
 func TestGetLedgerGaps(t *testing.T) {
diff --git a/services/horizon/internal/db2/history/mock_q_ledgers.go b/services/horizon/internal/db2/history/mock_q_ledgers.go
index 16d3ef5524..f02cd7517c 100644
--- a/services/horizon/internal/db2/history/mock_q_ledgers.go
+++ b/services/horizon/internal/db2/history/mock_q_ledgers.go
@@ -3,23 +3,38 @@ package history
 import (
 	"context"
 
-	"github.com/stretchr/testify/mock"
-
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/xdr"
+
+	"github.com/stretchr/testify/mock"
 )
 
 type MockQLedgers struct {
 	mock.Mock
 }
 
-func (m *MockQLedgers) InsertLedger(ctx context.Context,
+func (m *MockQLedgers) NewLedgerBatchInsertBuilder() LedgerBatchInsertBuilder {
+	a := m.Called()
+	return a.Get(0).(LedgerBatchInsertBuilder)
+}
+
+type MockLedgersBatchInsertBuilder struct {
+	mock.Mock
+}
+
+func (m *MockLedgersBatchInsertBuilder) Add(
 	ledger xdr.LedgerHeaderHistoryEntry,
 	successTxsCount int,
 	failedTxsCount int,
 	opCount int,
 	txSetOpCount int,
 	ingestVersion int,
-) (int64, error) {
-	a := m.Called(ctx, ledger, successTxsCount, failedTxsCount, opCount, txSetOpCount, ingestVersion)
-	return a.Get(0).(int64), a.Error(1)
+) error {
+	a := m.Called(ledger, successTxsCount, failedTxsCount, opCount, txSetOpCount, ingestVersion)
+	return a.Error(0)
+}
+
+func (m *MockLedgersBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	a := m.Called(ctx, session)
+	return a.Error(0)
 }
diff --git a/services/horizon/internal/db2/history/transaction_test.go b/services/horizon/internal/db2/history/transaction_test.go
index 0f3592c439..576b93ffb9 100644
--- a/services/horizon/internal/db2/history/transaction_test.go
+++ b/services/horizon/internal/db2/history/transaction_test.go
@@ -45,7 +45,8 @@ func TestTransactionByLiquidityPool(t *testing.T) {
 
 	// Insert a phony ledger
 	ledgerCloseTime := time.Now().Unix()
-	_, err := q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{
+	ledgerBatch := q.NewLedgerBatchInsertBuilder()
+	err := ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
 			LedgerSeq: xdr.Uint32(sequence),
 			ScpValue: xdr.StellarValue{
@@ -54,6 +55,9 @@ func TestTransactionByLiquidityPool(t *testing.T) {
 		},
 	}, 0, 0, 0, 0, 0)
 	tt.Assert.NoError(err)
+	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q.SessionInterface))
+	tt.Assert.NoError(q.Commit())
 
 	// Insert a phony transaction
 	transactionBuilder := q.NewTransactionBatchInsertBuilder(2)
diff --git a/services/horizon/internal/ingest/main.go b/services/horizon/internal/ingest/main.go
index dc6dc8dd46..783cc29681 100644
--- a/services/horizon/internal/ingest/main.go
+++ b/services/horizon/internal/ingest/main.go
@@ -310,6 +310,7 @@ func NewSystem(config Config) (System, error) {
 			ctx:            ctx,
 			config:         config,
 			historyQ:       historyQ,
+			session:        historyQ,
 			historyAdapter: historyAdapter,
 			filters:        filters,
 		},
diff --git a/services/horizon/internal/ingest/processor_runner.go b/services/horizon/internal/ingest/processor_runner.go
index 29635dce66..c6fcd75f2c 100644
--- a/services/horizon/internal/ingest/processor_runner.go
+++ b/services/horizon/internal/ingest/processor_runner.go
@@ -10,6 +10,7 @@ import (
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	"github.com/stellar/go/services/horizon/internal/ingest/filters"
 	"github.com/stellar/go/services/horizon/internal/ingest/processors"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/xdr"
 )
@@ -88,6 +89,7 @@ type ProcessorRunner struct {
 
 	ctx                   context.Context
 	historyQ              history.IngestionQ
+	session               db.SessionInterface
 	historyAdapter        historyArchiveAdapterInterface
 	logMemoryStats        bool
 	filters               filters.Filters
@@ -143,7 +145,7 @@ func (s *ProcessorRunner) buildTransactionProcessor(
 	return newGroupTransactionProcessors([]horizonTransactionProcessor{
 		statsLedgerTransactionProcessor,
 		processors.NewEffectProcessor(s.historyQ, sequence),
-		processors.NewLedgerProcessor(s.historyQ, ledger, CurrentVersion),
+		processors.NewLedgerProcessor(s.session, s.historyQ, ledger, CurrentVersion),
 		processors.NewOperationProcessor(s.historyQ, sequence),
 		tradeProcessor,
 		processors.NewParticipantsProcessor(s.historyQ, sequence),
diff --git a/services/horizon/internal/ingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go
index f1fa7747fb..14c1daf0df 100644
--- a/services/horizon/internal/ingest/processor_runner_test.go
+++ b/services/horizon/internal/ingest/processor_runner_test.go
@@ -16,6 +16,7 @@ import (
 	"github.com/stellar/go/network"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	"github.com/stellar/go/services/horizon/internal/ingest/processors"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/xdr"
 )
 
@@ -316,13 +317,23 @@ func TestProcessorRunnerWithFilterEnabled(t *testing.T) {
 	q.On("DeleteTransactionsFilteredTmpOlderThan", ctx, mock.AnythingOfType("uint64")).
 		Return(int64(0), nil)
 
-	q.MockQLedgers.On("InsertLedger", ctx, ledger.V0.LedgerHeader, 0, 0, 0, 0, CurrentVersion).
-		Return(int64(1), nil).Once()
+	mockBatchInsertBuilder := &history.MockLedgersBatchInsertBuilder{}
+	q.MockQLedgers.On("NewLedgerBatchInsertBuilder").Return(mockBatchInsertBuilder)
+	mockBatchInsertBuilder.On(
+		"Add",
+		ledger.V0.LedgerHeader, 0, 0, 0, 0, CurrentVersion).Return(nil)
+	mockSession := &db.MockSession{}
+	mockBatchInsertBuilder.On(
+		"Exec",
+		ctx,
+		mockSession,
+	).Return(nil)
 
 	runner := ProcessorRunner{
 		ctx:      ctx,
 		config:   config,
 		historyQ: q,
+		session:  mockSession,
 		filters:  &MockFilters{},
 	}
 
@@ -372,13 +383,23 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) {
 	q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize).
 		Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once()
 
-	q.MockQLedgers.On("InsertLedger", ctx, ledger.V0.LedgerHeader, 0, 0, 0, 0, CurrentVersion).
-		Return(int64(1), nil).Once()
+	mockBatchInsertBuilder := &history.MockLedgersBatchInsertBuilder{}
+	q.MockQLedgers.On("NewLedgerBatchInsertBuilder").Return(mockBatchInsertBuilder)
+	mockBatchInsertBuilder.On(
+		"Add",
+		ledger.V0.LedgerHeader, 0, 0, 0, 0, CurrentVersion).Return(nil)
+	mockSession := &db.MockSession{}
+	mockBatchInsertBuilder.On(
+		"Exec",
+		ctx,
+		mockSession,
+	).Return(nil)
 
 	runner := ProcessorRunner{
 		ctx:      ctx,
 		config:   config,
 		historyQ: q,
+		session:  mockSession,
 		filters:  &MockFilters{},
 	}
 
diff --git a/services/horizon/internal/ingest/processors/ledgers_processor.go b/services/horizon/internal/ingest/processors/ledgers_processor.go
index 01c29b43d9..aee2f12709 100644
--- a/services/horizon/internal/ingest/processors/ledgers_processor.go
+++ b/services/horizon/internal/ingest/processors/ledgers_processor.go
@@ -5,11 +5,13 @@ import (
 
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/xdr"
 )
 
 type LedgersProcessor struct {
+	session        db.SessionInterface
 	ledgersQ       history.QLedgers
 	ledger         xdr.LedgerHeaderHistoryEntry
 	ingestVersion  int
@@ -20,11 +22,13 @@ type LedgersProcessor struct {
 }
 
 func NewLedgerProcessor(
+	session db.SessionInterface,
 	ledgerQ history.QLedgers,
 	ledger xdr.LedgerHeaderHistoryEntry,
 	ingestVersion int,
 ) *LedgersProcessor {
 	return &LedgersProcessor{
+		session:       session,
 		ledger:        ledger,
 		ledgersQ:      ledgerQ,
 		ingestVersion: ingestVersion,
@@ -45,29 +49,14 @@ func (p *LedgersProcessor) ProcessTransaction(ctx context.Context, transaction i
 }
 
 func (p *LedgersProcessor) Commit(ctx context.Context) error {
-	rowsAffected, err := p.ledgersQ.InsertLedger(ctx,
-		p.ledger,
-		p.successTxCount,
-		p.failedTxCount,
-		p.opCount,
-		p.txSetOpCount,
-		p.ingestVersion,
-	)
-
+	batch := p.ledgersQ.NewLedgerBatchInsertBuilder()
+	err := batch.Add(p.ledger, p.successTxCount, p.failedTxCount, p.opCount, p.txSetOpCount, p.ingestVersion)
 	if err != nil {
 		return errors.Wrap(err, "Could not insert ledger")
 	}
 
-	sequence := uint32(p.ledger.Header.LedgerSeq)
-
-	if rowsAffected != 1 {
-		log.WithField("rowsAffected", rowsAffected).
-			WithField("sequence", sequence).
-			Error("Invalid number of rows affected when ingesting new ledger")
-		return errors.Errorf(
-			"0 rows affected when ingesting new ledger: %v",
-			sequence,
-		)
+	if err = batch.Exec(ctx, p.session); err != nil {
+		return errors.Wrap(err, "Could not commit ledger")
 	}
 
 	return nil
diff --git a/services/horizon/internal/ingest/processors/ledgers_processor_test.go b/services/horizon/internal/ingest/processors/ledgers_processor_test.go
index 05bd2c3c3b..9cbd2c8643 100644
--- a/services/horizon/internal/ingest/processors/ledgers_processor_test.go
+++ b/services/horizon/internal/ingest/processors/ledgers_processor_test.go
@@ -8,6 +8,7 @@ import (
 
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/xdr"
 	"github.com/stretchr/testify/mock"
@@ -17,6 +18,7 @@ import (
 type LedgersProcessorTestSuiteLedger struct {
 	suite.Suite
 	processor     *LedgersProcessor
+	mockSession   *db.MockSession
 	mockQ         *history.MockQLedgers
 	header        xdr.LedgerHeaderHistoryEntry
 	successCount  int
@@ -85,7 +87,9 @@ func (s *LedgersProcessorTestSuiteLedger) SetupTest() {
 			LedgerSeq: xdr.Uint32(20),
 		},
 	}
+
 	s.processor = NewLedgerProcessor(
+		s.mockSession,
 		s.mockQ,
 		s.header,
 		s.ingestVersion,
@@ -109,16 +113,23 @@ func (s *LedgersProcessorTestSuiteLedger) TearDownTest() {
 
 func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerSucceeds() {
 	ctx := context.Background()
-	s.mockQ.On(
-		"InsertLedger",
-		ctx,
+	mockBatchInsertBuilder := &history.MockLedgersBatchInsertBuilder{}
+	s.mockQ.On("NewLedgerBatchInsertBuilder").Return(mockBatchInsertBuilder)
+
+	mockBatchInsertBuilder.On(
+		"Add",
 		s.header,
 		s.successCount,
 		s.failedCount,
 		s.opCount,
 		s.txSetOpCount,
 		s.ingestVersion,
-	).Return(int64(1), nil)
+	).Return(nil)
+	mockBatchInsertBuilder.On(
+		"Exec",
+		ctx,
+		s.mockSession,
+	).Return(nil)
 
 	for _, tx := range s.txs {
 		err := s.processor.ProcessTransaction(ctx, tx)
@@ -130,37 +141,21 @@ func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerSucceeds() {
 }
 
 func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerReturnsError() {
-	ctx := context.Background()
-	s.mockQ.On(
-		"InsertLedger",
-		ctx,
-		mock.Anything,
-		mock.Anything,
-		mock.Anything,
-		mock.Anything,
-		mock.Anything,
-		mock.Anything,
-	).Return(int64(0), errors.New("transient error"))
-
-	err := s.processor.Commit(ctx)
-	s.Assert().Error(err)
-	s.Assert().EqualError(err, "Could not insert ledger: transient error")
-}
+	mockBatchInsertBuilder := &history.MockLedgersBatchInsertBuilder{}
+	s.mockQ.On("NewLedgerBatchInsertBuilder").Return(mockBatchInsertBuilder)
 
-func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerNoRowsAffected() {
-	ctx := context.Background()
-	s.mockQ.On(
-		"InsertLedger",
-		ctx,
+	mockBatchInsertBuilder.On(
+		"Add",
 		mock.Anything,
 		mock.Anything,
 		mock.Anything,
 		mock.Anything,
 		mock.Anything,
 		mock.Anything,
-	).Return(int64(0), nil)
+	).Return(errors.New("transient error"))
 
+	ctx := context.Background()
 	err := s.processor.Commit(ctx)
 	s.Assert().Error(err)
-	s.Assert().EqualError(err, "0 rows affected when ingesting new ledger: 20")
+	s.Assert().EqualError(err, "Could not insert ledger: transient error")
 }
diff --git a/services/horizon/internal/middleware_test.go b/services/horizon/internal/middleware_test.go
index 08b90465f3..6dd1a46681 100644
--- a/services/horizon/internal/middleware_test.go
+++ b/services/horizon/internal/middleware_test.go
@@ -284,7 +284,10 @@ func TestStateMiddleware(t *testing.T) {
 		t.Run(testCase.name, func(t *testing.T) {
 			stateMiddleware.NoStateVerification = testCase.noStateVerification
 			tt.Assert.NoError(q.UpdateExpStateInvalid(context.Background(), testCase.stateInvalid))
-			_, err = q.InsertLedger(context.Background(), xdr.LedgerHeaderHistoryEntry{
+
+			tt.Assert.NoError(q.Begin())
+			ledgerBatch := q.NewLedgerBatchInsertBuilder()
+			err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 				Hash: xdr.Hash{byte(i)},
 				Header: xdr.LedgerHeader{
 					LedgerSeq:          testCase.latestHistoryLedger,
@@ -292,6 +295,9 @@ func TestStateMiddleware(t *testing.T) {
 				},
 			}, 0, 0, 0, 0, 0)
 			tt.Assert.NoError(err)
+			tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q))
+			tt.Assert.NoError(q.Commit())
+
 			tt.Assert.NoError(q.UpdateLastLedgerIngest(context.Background(), testCase.lastIngestedLedger))
 			tt.Assert.NoError(q.UpdateIngestVersion(context.Background(), testCase.ingestionVersion))
 

From af8161b90e4de1911cb7dbfcef1ecdd08356857d Mon Sep 17 00:00:00 2001
From: tamirms <tamirms@gmail.com>
Date: Tue, 18 Jul 2023 13:20:53 +0100
Subject: [PATCH 03/17] services/horizon/internal/db2/history: Use
 FastBatchInsertBuilder to insert transactions into the history_transactions
 (#4950)

---
 .../internal/db2/history/fee_bump_scenario.go | 20 ++++++----
 .../db2/history/mock_q_transactions.go        |  8 ++--
 .../mock_transactions_batch_insert_builder.go |  9 +++--
 .../operation_batch_insert_builder_test.go    | 10 +++--
 .../internal/db2/history/operation_test.go    | 10 +++--
 .../internal/db2/history/transaction.go       |  4 +-
 .../transaction_batch_insert_builder.go       | 33 +++++++--------
 .../internal/db2/history/transaction_test.go  | 40 +++++++++++--------
 .../internal/ingest/processor_runner.go       |  4 +-
 .../internal/ingest/processor_runner_test.go  | 16 ++++----
 .../processors/transactions_processor.go      | 16 +++++---
 .../processors/transactions_processor_test.go | 21 +++++-----
 12 files changed, 108 insertions(+), 83 deletions(-)

diff --git a/services/horizon/internal/db2/history/fee_bump_scenario.go b/services/horizon/internal/db2/history/fee_bump_scenario.go
index 7263e7aeb2..0c279d737f 100644
--- a/services/horizon/internal/db2/history/fee_bump_scenario.go
+++ b/services/horizon/internal/db2/history/fee_bump_scenario.go
@@ -247,17 +247,19 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 		hash:          "edba3051b2f2d9b713e8a08709d631eccb72c59864ff3c564c68792271bb24a7",
 	})
 	ctx := context.Background()
-	insertBuilder := q.NewTransactionBatchInsertBuilder(2)
-	prefilterInsertBuilder := q.NewTransactionFilteredTmpBatchInsertBuilder(2)
+	tt.Assert.NoError(q.Begin())
+
+	insertBuilder := q.NewTransactionBatchInsertBuilder()
+	prefilterInsertBuilder := q.NewTransactionFilteredTmpBatchInsertBuilder()
 	// include both fee bump and normal transaction in the same batch
 	// to make sure both kinds of transactions can be inserted using a single exec statement
-	tt.Assert.NoError(insertBuilder.Add(ctx, feeBumpTransaction, sequence))
-	tt.Assert.NoError(insertBuilder.Add(ctx, normalTransaction, sequence))
-	tt.Assert.NoError(insertBuilder.Exec(ctx))
+	tt.Assert.NoError(insertBuilder.Add(feeBumpTransaction, sequence))
+	tt.Assert.NoError(insertBuilder.Add(normalTransaction, sequence))
+	tt.Assert.NoError(insertBuilder.Exec(ctx, q))
 
-	tt.Assert.NoError(prefilterInsertBuilder.Add(ctx, feeBumpTransaction, sequence))
-	tt.Assert.NoError(prefilterInsertBuilder.Add(ctx, normalTransaction, sequence))
-	tt.Assert.NoError(prefilterInsertBuilder.Exec(ctx))
+	tt.Assert.NoError(prefilterInsertBuilder.Add(feeBumpTransaction, sequence))
+	tt.Assert.NoError(prefilterInsertBuilder.Add(normalTransaction, sequence))
+	tt.Assert.NoError(prefilterInsertBuilder.Exec(ctx, q))
 
 	account := fixture.Envelope.SourceAccount().ToAccountId()
 	feeBumpAccount := fixture.Envelope.FeeBumpAccount().ToAccountId()
@@ -299,6 +301,8 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 	tt.Assert.NoError(err)
 	tt.Assert.NoError(effectBuilder.Exec(ctx))
 
+	tt.Assert.NoError(q.Commit())
+
 	fixture.Transaction = Transaction{
 		TransactionWithoutLedger: TransactionWithoutLedger{
 			TotalOrderID:         TotalOrderID{528280981504},
diff --git a/services/horizon/internal/db2/history/mock_q_transactions.go b/services/horizon/internal/db2/history/mock_q_transactions.go
index 3bf308128f..064d0e34c4 100644
--- a/services/horizon/internal/db2/history/mock_q_transactions.go
+++ b/services/horizon/internal/db2/history/mock_q_transactions.go
@@ -7,12 +7,12 @@ type MockQTransactions struct {
 	mock.Mock
 }
 
-func (m *MockQTransactions) NewTransactionBatchInsertBuilder(maxBatchSize int) TransactionBatchInsertBuilder {
-	a := m.Called(maxBatchSize)
+func (m *MockQTransactions) NewTransactionBatchInsertBuilder() TransactionBatchInsertBuilder {
+	a := m.Called()
 	return a.Get(0).(TransactionBatchInsertBuilder)
 }
 
-func (m *MockQTransactions) NewTransactionFilteredTmpBatchInsertBuilder(maxBatchSize int) TransactionBatchInsertBuilder {
-	a := m.Called(maxBatchSize)
+func (m *MockQTransactions) NewTransactionFilteredTmpBatchInsertBuilder() TransactionBatchInsertBuilder {
+	a := m.Called()
 	return a.Get(0).(TransactionBatchInsertBuilder)
 }
diff --git a/services/horizon/internal/db2/history/mock_transactions_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_transactions_batch_insert_builder.go
index 8e2608d553..db16097a03 100644
--- a/services/horizon/internal/db2/history/mock_transactions_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/mock_transactions_batch_insert_builder.go
@@ -6,18 +6,19 @@ import (
 	"github.com/stretchr/testify/mock"
 
 	"github.com/stellar/go/ingest"
+	"github.com/stellar/go/support/db"
 )
 
 type MockTransactionsBatchInsertBuilder struct {
 	mock.Mock
 }
 
-func (m *MockTransactionsBatchInsertBuilder) Add(ctx context.Context, transaction ingest.LedgerTransaction, sequence uint32) error {
-	a := m.Called(ctx, transaction, sequence)
+func (m *MockTransactionsBatchInsertBuilder) Add(transaction ingest.LedgerTransaction, sequence uint32) error {
+	a := m.Called(transaction, sequence)
 	return a.Error(0)
 }
 
-func (m *MockTransactionsBatchInsertBuilder) Exec(ctx context.Context) error {
-	a := m.Called(ctx)
+func (m *MockTransactionsBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	a := m.Called(ctx, session)
 	return a.Error(0)
 }
diff --git a/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go b/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go
index 18b4913a0a..4ad624745c 100644
--- a/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go
+++ b/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go
@@ -16,7 +16,9 @@ func TestAddOperation(t *testing.T) {
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
 
-	txBatch := q.NewTransactionBatchInsertBuilder(0)
+	tt.Assert.NoError(q.Begin())
+
+	txBatch := q.NewTransactionBatchInsertBuilder()
 
 	builder := q.NewOperationBatchInsertBuilder(1)
 
@@ -35,8 +37,8 @@ func TestAddOperation(t *testing.T) {
 	)
 
 	sequence := int32(56)
-	tt.Assert.NoError(txBatch.Add(tt.Ctx, transaction, uint32(sequence)))
-	tt.Assert.NoError(txBatch.Exec(tt.Ctx))
+	tt.Assert.NoError(txBatch.Add(transaction, uint32(sequence)))
+	tt.Assert.NoError(txBatch.Exec(tt.Ctx, q))
 
 	details, err := json.Marshal(map[string]string{
 		"to":         "GANFZDRBCNTUXIODCJEYMACPMCSZEVE4WZGZ3CZDZ3P2SXK4KH75IK6Y",
@@ -62,6 +64,8 @@ func TestAddOperation(t *testing.T) {
 	err = builder.Exec(tt.Ctx)
 	tt.Assert.NoError(err)
 
+	tt.Assert.NoError(q.Commit())
+
 	ops := []Operation{}
 	err = q.Select(tt.Ctx, &ops, selectOperation)
 
diff --git a/services/horizon/internal/db2/history/operation_test.go b/services/horizon/internal/db2/history/operation_test.go
index 6fadfde67a..8c28809568 100644
--- a/services/horizon/internal/db2/history/operation_test.go
+++ b/services/horizon/internal/db2/history/operation_test.go
@@ -77,8 +77,10 @@ func TestOperationByLiquidityPool(t *testing.T) {
 	opID1 := toid.New(sequence, txIndex, 1).ToInt64()
 	opID2 := toid.New(sequence, txIndex, 2).ToInt64()
 
+	tt.Assert.NoError(q.Begin())
+
 	// Insert a phony transaction
-	transactionBuilder := q.NewTransactionBatchInsertBuilder(2)
+	transactionBuilder := q.NewTransactionBatchInsertBuilder()
 	firstTransaction := buildLedgerTransaction(tt.T, testTransaction{
 		index:         uint32(txIndex),
 		envelopeXDR:   "AAAAACiSTRmpH6bHC6Ekna5e82oiGY5vKDEEUgkq9CB//t+rAAAAyAEXUhsAADDRAAAAAAAAAAAAAAABAAAAAAAAAAsBF1IbAABX4QAAAAAAAAAA",
@@ -87,9 +89,9 @@ func TestOperationByLiquidityPool(t *testing.T) {
 		metaXDR:       "AAAAAQAAAAAAAAAA",
 		hash:          "19aaa18db88605aedec04659fb45e06f240b022eb2d429e05133e4d53cd945ba",
 	})
-	err := transactionBuilder.Add(tt.Ctx, firstTransaction, uint32(sequence))
+	err := transactionBuilder.Add(firstTransaction, uint32(sequence))
 	tt.Assert.NoError(err)
-	err = transactionBuilder.Exec(tt.Ctx)
+	err = transactionBuilder.Exec(tt.Ctx, q)
 	tt.Assert.NoError(err)
 
 	// Insert a two phony operations
@@ -137,6 +139,8 @@ func TestOperationByLiquidityPool(t *testing.T) {
 	err = lpOperationBuilder.Exec(tt.Ctx)
 	tt.Assert.NoError(err)
 
+	tt.Assert.NoError(q.Commit())
+
 	// Check ascending order
 	pq := db2.PageQuery{
 		Cursor: "",
diff --git a/services/horizon/internal/db2/history/transaction.go b/services/horizon/internal/db2/history/transaction.go
index 0771a626b8..a308ab9ddc 100644
--- a/services/horizon/internal/db2/history/transaction.go
+++ b/services/horizon/internal/db2/history/transaction.go
@@ -232,8 +232,8 @@ func (q *TransactionsQ) Select(ctx context.Context, dest interface{}) error {
 
 // QTransactions defines transaction related queries.
 type QTransactions interface {
-	NewTransactionBatchInsertBuilder(maxBatchSize int) TransactionBatchInsertBuilder
-	NewTransactionFilteredTmpBatchInsertBuilder(maxBatchSize int) TransactionBatchInsertBuilder
+	NewTransactionBatchInsertBuilder() TransactionBatchInsertBuilder
+	NewTransactionFilteredTmpBatchInsertBuilder() TransactionBatchInsertBuilder
 }
 
 func selectTransaction(table string) sq.SelectBuilder {
diff --git a/services/horizon/internal/db2/history/transaction_batch_insert_builder.go b/services/horizon/internal/db2/history/transaction_batch_insert_builder.go
index 2ecb25dbe7..742621cec5 100644
--- a/services/horizon/internal/db2/history/transaction_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/transaction_batch_insert_builder.go
@@ -21,50 +21,47 @@ import (
 // TransactionBatchInsertBuilder is used to insert transactions into the
 // history_transactions table
 type TransactionBatchInsertBuilder interface {
-	Add(ctx context.Context, transaction ingest.LedgerTransaction, sequence uint32) error
-	Exec(ctx context.Context) error
+	Add(transaction ingest.LedgerTransaction, sequence uint32) error
+	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
 // transactionBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder
 type transactionBatchInsertBuilder struct {
 	encodingBuffer *xdr.EncodingBuffer
-	builder        db.BatchInsertBuilder
+	table          string
+	builder        db.FastBatchInsertBuilder
 }
 
 // NewTransactionBatchInsertBuilder constructs a new TransactionBatchInsertBuilder instance
-func (q *Q) NewTransactionBatchInsertBuilder(maxBatchSize int) TransactionBatchInsertBuilder {
+func (q *Q) NewTransactionBatchInsertBuilder() TransactionBatchInsertBuilder {
 	return &transactionBatchInsertBuilder{
 		encodingBuffer: xdr.NewEncodingBuffer(),
-		builder: db.BatchInsertBuilder{
-			Table:        q.GetTable("history_transactions"),
-			MaxBatchSize: maxBatchSize,
-		},
+		table:          "history_transactions",
+		builder:        db.FastBatchInsertBuilder{},
 	}
 }
 
-// NewTransactionBatchInsertBuilder constructs a new TransactionBatchInsertBuilder instance
-func (q *Q) NewTransactionFilteredTmpBatchInsertBuilder(maxBatchSize int) TransactionBatchInsertBuilder {
+// NewTransactionFilteredTmpBatchInsertBuilder constructs a new TransactionBatchInsertBuilder instance
+func (q *Q) NewTransactionFilteredTmpBatchInsertBuilder() TransactionBatchInsertBuilder {
 	return &transactionBatchInsertBuilder{
 		encodingBuffer: xdr.NewEncodingBuffer(),
-		builder: db.BatchInsertBuilder{
-			Table:        q.GetTable("history_transactions_filtered_tmp"),
-			MaxBatchSize: maxBatchSize,
-		},
+		table:          "history_transactions_filtered_tmp",
+		builder:        db.FastBatchInsertBuilder{},
 	}
 }
 
 // Add adds a new transaction to the batch
-func (i *transactionBatchInsertBuilder) Add(ctx context.Context, transaction ingest.LedgerTransaction, sequence uint32) error {
+func (i *transactionBatchInsertBuilder) Add(transaction ingest.LedgerTransaction, sequence uint32) error {
 	row, err := transactionToRow(transaction, sequence, i.encodingBuffer)
 	if err != nil {
 		return err
 	}
 
-	return i.builder.RowStruct(ctx, row)
+	return i.builder.RowStruct(row)
 }
 
-func (i *transactionBatchInsertBuilder) Exec(ctx context.Context) error {
-	return i.builder.Exec(ctx)
+func (i *transactionBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	return i.builder.Exec(ctx, session, i.table)
 }
 
 func signatures(xdrSignatures []xdr.DecoratedSignature) pq.StringArray {
diff --git a/services/horizon/internal/db2/history/transaction_test.go b/services/horizon/internal/db2/history/transaction_test.go
index 576b93ffb9..33ef7b0d3c 100644
--- a/services/horizon/internal/db2/history/transaction_test.go
+++ b/services/horizon/internal/db2/history/transaction_test.go
@@ -55,12 +55,13 @@ func TestTransactionByLiquidityPool(t *testing.T) {
 		},
 	}, 0, 0, 0, 0, 0)
 	tt.Assert.NoError(err)
+
 	tt.Assert.NoError(q.Begin())
+
 	tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q.SessionInterface))
-	tt.Assert.NoError(q.Commit())
 
 	// Insert a phony transaction
-	transactionBuilder := q.NewTransactionBatchInsertBuilder(2)
+	transactionBuilder := q.NewTransactionBatchInsertBuilder()
 	firstTransaction := buildLedgerTransaction(tt.T, testTransaction{
 		index:         uint32(txIndex),
 		envelopeXDR:   "AAAAACiSTRmpH6bHC6Ekna5e82oiGY5vKDEEUgkq9CB//t+rAAAAyAEXUhsAADDRAAAAAAAAAAAAAAABAAAAAAAAAAsBF1IbAABX4QAAAAAAAAAA",
@@ -69,9 +70,9 @@ func TestTransactionByLiquidityPool(t *testing.T) {
 		metaXDR:       "AAAAAQAAAAAAAAAA",
 		hash:          "19aaa18db88605aedec04659fb45e06f240b022eb2d429e05133e4d53cd945ba",
 	})
-	err = transactionBuilder.Add(tt.Ctx, firstTransaction, uint32(sequence))
+	err = transactionBuilder.Add(firstTransaction, uint32(sequence))
 	tt.Assert.NoError(err)
-	err = transactionBuilder.Exec(tt.Ctx)
+	err = transactionBuilder.Exec(tt.Ctx, q)
 	tt.Assert.NoError(err)
 
 	// Insert Liquidity Pool history
@@ -87,6 +88,8 @@ func TestTransactionByLiquidityPool(t *testing.T) {
 	err = lpTransactionBuilder.Exec(tt.Ctx)
 	tt.Assert.NoError(err)
 
+	tt.Assert.NoError(q.Commit())
+
 	var records []Transaction
 	err = q.Transactions().ForLiquidityPool(tt.Ctx, liquidityPoolID).Select(tt.Ctx, &records)
 	tt.Assert.NoError(err)
@@ -210,8 +213,10 @@ func TestInsertTransactionDoesNotAllowDuplicateIndex(t *testing.T) {
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
 
+	tt.Assert.NoError(q.Begin())
+
 	sequence := uint32(123)
-	insertBuilder := q.NewTransactionBatchInsertBuilder(0)
+	insertBuilder := q.NewTransactionBatchInsertBuilder()
 
 	firstTransaction := buildLedgerTransaction(tt.T, testTransaction{
 		index:         1,
@@ -230,16 +235,18 @@ func TestInsertTransactionDoesNotAllowDuplicateIndex(t *testing.T) {
 		hash:          "7e2def20d5a21a56be2a457b648f702ee1af889d3df65790e92a05081e9fabf1",
 	})
 
-	tt.Assert.NoError(insertBuilder.Add(tt.Ctx, firstTransaction, sequence))
-	tt.Assert.NoError(insertBuilder.Exec(tt.Ctx))
+	tt.Assert.NoError(insertBuilder.Add(firstTransaction, sequence))
+	tt.Assert.NoError(insertBuilder.Exec(tt.Ctx, q))
+	tt.Assert.NoError(q.Commit())
 
-	tt.Assert.NoError(insertBuilder.Add(tt.Ctx, secondTransaction, sequence))
+	tt.Assert.NoError(q.Begin())
+	insertBuilder = q.NewTransactionBatchInsertBuilder()
+	tt.Assert.NoError(insertBuilder.Add(secondTransaction, sequence))
 	tt.Assert.EqualError(
-		insertBuilder.Exec(tt.Ctx),
-		"error adding values while inserting to history_transactions: "+
-			"exec failed: pq: duplicate key value violates unique constraint "+
-			"\"hs_transaction_by_id\"",
+		insertBuilder.Exec(tt.Ctx, q),
+		"pq: duplicate key value violates unique constraint \"hs_transaction_by_id\"",
 	)
+	tt.Assert.NoError(q.Rollback())
 
 	ledger := Ledger{
 		Sequence:                   int32(sequence),
@@ -305,8 +312,6 @@ func TestInsertTransaction(t *testing.T) {
 	_, err := q.Exec(tt.Ctx, sq.Insert("history_ledgers").SetMap(ledgerToMap(ledger)))
 	tt.Assert.NoError(err)
 
-	insertBuilder := q.NewTransactionBatchInsertBuilder(0)
-
 	success := true
 
 	emptySignatures := []string{}
@@ -826,8 +831,11 @@ func TestInsertTransaction(t *testing.T) {
 		},
 	} {
 		t.Run(testCase.name, func(t *testing.T) {
-			tt.Assert.NoError(insertBuilder.Add(tt.Ctx, testCase.toInsert, sequence))
-			tt.Assert.NoError(insertBuilder.Exec(tt.Ctx))
+			insertBuilder := q.NewTransactionBatchInsertBuilder()
+			tt.Assert.NoError(q.Begin())
+			tt.Assert.NoError(insertBuilder.Add(testCase.toInsert, sequence))
+			tt.Assert.NoError(insertBuilder.Exec(tt.Ctx, q))
+			tt.Assert.NoError(q.Commit())
 
 			var transactions []Transaction
 			tt.Assert.NoError(q.Transactions().IncludeFailed().Select(tt.Ctx, &transactions))
diff --git a/services/horizon/internal/ingest/processor_runner.go b/services/horizon/internal/ingest/processor_runner.go
index c6fcd75f2c..4dc3d1eda9 100644
--- a/services/horizon/internal/ingest/processor_runner.go
+++ b/services/horizon/internal/ingest/processor_runner.go
@@ -149,7 +149,7 @@ func (s *ProcessorRunner) buildTransactionProcessor(
 		processors.NewOperationProcessor(s.historyQ, sequence),
 		tradeProcessor,
 		processors.NewParticipantsProcessor(s.historyQ, sequence),
-		processors.NewTransactionProcessor(s.historyQ, sequence),
+		processors.NewTransactionProcessor(s.session, s.historyQ, sequence),
 		processors.NewClaimableBalancesTransactionProcessor(s.historyQ, sequence),
 		processors.NewLiquidityPoolsTransactionProcessor(s.historyQ, sequence),
 	})
@@ -168,7 +168,7 @@ func (s *ProcessorRunner) buildFilteredOutProcessor(ledger xdr.LedgerHeaderHisto
 	// when in online mode, the submission result processor must always run (regardless of filtering)
 	var p []horizonTransactionProcessor
 	if s.config.EnableIngestionFiltering {
-		txSubProc := processors.NewTransactionFilteredTmpProcessor(s.historyQ, uint32(ledger.Header.LedgerSeq))
+		txSubProc := processors.NewTransactionFilteredTmpProcessor(s.session, s.historyQ, uint32(ledger.Header.LedgerSeq))
 		p = append(p, txSubProc)
 	}
 
diff --git a/services/horizon/internal/ingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go
index 14c1daf0df..dbbe3a5191 100644
--- a/services/horizon/internal/ingest/processor_runner_test.go
+++ b/services/horizon/internal/ingest/processor_runner_test.go
@@ -241,7 +241,7 @@ func TestProcessorRunnerBuildTransactionProcessor(t *testing.T) {
 
 	q.MockQOperations.On("NewOperationBatchInsertBuilder", maxBatchSize).
 		Return(&history.MockOperationsBatchInsertBuilder{}).Twice() // Twice = with/without failed
-	q.MockQTransactions.On("NewTransactionBatchInsertBuilder", maxBatchSize).
+	q.MockQTransactions.On("NewTransactionBatchInsertBuilder").
 		Return(&history.MockTransactionsBatchInsertBuilder{}).Twice()
 	q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize).
 		Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Twice()
@@ -277,6 +277,7 @@ func TestProcessorRunnerWithFilterEnabled(t *testing.T) {
 	}
 
 	q := &mockDBQ{}
+	mockSession := &db.MockSession{}
 	defer mock.AssertExpectationsForObjects(t, q)
 
 	ledger := xdr.LedgerCloseMeta{
@@ -303,12 +304,12 @@ func TestProcessorRunnerWithFilterEnabled(t *testing.T) {
 
 	mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{}
 	defer mock.AssertExpectationsForObjects(t, mockTransactionsBatchInsertBuilder)
-	mockTransactionsBatchInsertBuilder.On("Exec", ctx).Return(nil).Twice()
+	mockTransactionsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Twice()
 
-	q.MockQTransactions.On("NewTransactionBatchInsertBuilder", maxBatchSize).
+	q.MockQTransactions.On("NewTransactionBatchInsertBuilder").
 		Return(mockTransactionsBatchInsertBuilder)
 
-	q.MockQTransactions.On("NewTransactionFilteredTmpBatchInsertBuilder", maxBatchSize).
+	q.MockQTransactions.On("NewTransactionFilteredTmpBatchInsertBuilder").
 		Return(mockTransactionsBatchInsertBuilder)
 
 	q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize).
@@ -322,7 +323,6 @@ func TestProcessorRunnerWithFilterEnabled(t *testing.T) {
 	mockBatchInsertBuilder.On(
 		"Add",
 		ledger.V0.LedgerHeader, 0, 0, 0, 0, CurrentVersion).Return(nil)
-	mockSession := &db.MockSession{}
 	mockBatchInsertBuilder.On(
 		"Exec",
 		ctx,
@@ -349,6 +349,7 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) {
 		NetworkPassphrase: network.PublicNetworkPassphrase,
 	}
 
+	mockSession := &db.MockSession{}
 	q := &mockDBQ{}
 	defer mock.AssertExpectationsForObjects(t, q)
 
@@ -376,8 +377,8 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) {
 
 	mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{}
 	defer mock.AssertExpectationsForObjects(t, mockTransactionsBatchInsertBuilder)
-	mockTransactionsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once()
-	q.MockQTransactions.On("NewTransactionBatchInsertBuilder", maxBatchSize).
+	mockTransactionsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once()
+	q.MockQTransactions.On("NewTransactionBatchInsertBuilder").
 		Return(mockTransactionsBatchInsertBuilder).Twice()
 
 	q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize).
@@ -388,7 +389,6 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) {
 	mockBatchInsertBuilder.On(
 		"Add",
 		ledger.V0.LedgerHeader, 0, 0, 0, 0, CurrentVersion).Return(nil)
-	mockSession := &db.MockSession{}
 	mockBatchInsertBuilder.On(
 		"Exec",
 		ctx,
diff --git a/services/horizon/internal/ingest/processors/transactions_processor.go b/services/horizon/internal/ingest/processors/transactions_processor.go
index e2a880f296..0e6603f804 100644
--- a/services/horizon/internal/ingest/processors/transactions_processor.go
+++ b/services/horizon/internal/ingest/processors/transactions_processor.go
@@ -5,33 +5,37 @@ import (
 
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 )
 
 type TransactionProcessor struct {
+	session       db.SessionInterface
 	transactionsQ history.QTransactions
 	sequence      uint32
 	batch         history.TransactionBatchInsertBuilder
 }
 
-func NewTransactionFilteredTmpProcessor(transactionsQ history.QTransactions, sequence uint32) *TransactionProcessor {
+func NewTransactionFilteredTmpProcessor(session db.SessionInterface, transactionsQ history.QTransactions, sequence uint32) *TransactionProcessor {
 	return &TransactionProcessor{
+		session:       session,
 		transactionsQ: transactionsQ,
 		sequence:      sequence,
-		batch:         transactionsQ.NewTransactionFilteredTmpBatchInsertBuilder(maxBatchSize),
+		batch:         transactionsQ.NewTransactionFilteredTmpBatchInsertBuilder(),
 	}
 }
 
-func NewTransactionProcessor(transactionsQ history.QTransactions, sequence uint32) *TransactionProcessor {
+func NewTransactionProcessor(session db.SessionInterface, transactionsQ history.QTransactions, sequence uint32) *TransactionProcessor {
 	return &TransactionProcessor{
+		session:       session,
 		transactionsQ: transactionsQ,
 		sequence:      sequence,
-		batch:         transactionsQ.NewTransactionBatchInsertBuilder(maxBatchSize),
+		batch:         transactionsQ.NewTransactionBatchInsertBuilder(),
 	}
 }
 
 func (p *TransactionProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error {
-	if err := p.batch.Add(ctx, transaction, p.sequence); err != nil {
+	if err := p.batch.Add(transaction, p.sequence); err != nil {
 		return errors.Wrap(err, "Error batch inserting transaction rows")
 	}
 
@@ -39,7 +43,7 @@ func (p *TransactionProcessor) ProcessTransaction(ctx context.Context, transacti
 }
 
 func (p *TransactionProcessor) Commit(ctx context.Context) error {
-	if err := p.batch.Exec(ctx); err != nil {
+	if err := p.batch.Exec(ctx, p.session); err != nil {
 		return errors.Wrap(err, "Error flushing transaction batch")
 	}
 
diff --git a/services/horizon/internal/ingest/processors/transactions_processor_test.go b/services/horizon/internal/ingest/processors/transactions_processor_test.go
index ec1cf105e5..dcaf307729 100644
--- a/services/horizon/internal/ingest/processors/transactions_processor_test.go
+++ b/services/horizon/internal/ingest/processors/transactions_processor_test.go
@@ -7,7 +7,9 @@ import (
 	"testing"
 
 	"github.com/stellar/go/services/horizon/internal/db2/history"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
+
 	"github.com/stretchr/testify/suite"
 )
 
@@ -15,6 +17,7 @@ type TransactionsProcessorTestSuiteLedger struct {
 	suite.Suite
 	ctx                    context.Context
 	processor              *TransactionProcessor
+	mockSession            *db.MockSession
 	mockQ                  *history.MockQTransactions
 	mockBatchInsertBuilder *history.MockTransactionsBatchInsertBuilder
 }
@@ -29,10 +32,10 @@ func (s *TransactionsProcessorTestSuiteLedger) SetupTest() {
 	s.mockBatchInsertBuilder = &history.MockTransactionsBatchInsertBuilder{}
 
 	s.mockQ.
-		On("NewTransactionBatchInsertBuilder", maxBatchSize).
+		On("NewTransactionBatchInsertBuilder").
 		Return(s.mockBatchInsertBuilder).Once()
 
-	s.processor = NewTransactionProcessor(s.mockQ, 20)
+	s.processor = NewTransactionProcessor(s.mockSession, s.mockQ, 20)
 }
 
 func (s *TransactionsProcessorTestSuiteLedger) TearDownTest() {
@@ -47,10 +50,10 @@ func (s *TransactionsProcessorTestSuiteLedger) TestAddTransactionsSucceeds() {
 	secondTx := createTransaction(false, 3)
 	thirdTx := createTransaction(true, 4)
 
-	s.mockBatchInsertBuilder.On("Add", s.ctx, firstTx, sequence).Return(nil).Once()
-	s.mockBatchInsertBuilder.On("Add", s.ctx, secondTx, sequence).Return(nil).Once()
-	s.mockBatchInsertBuilder.On("Add", s.ctx, thirdTx, sequence).Return(nil).Once()
-	s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once()
+	s.mockBatchInsertBuilder.On("Add", firstTx, sequence).Return(nil).Once()
+	s.mockBatchInsertBuilder.On("Add", secondTx, sequence).Return(nil).Once()
+	s.mockBatchInsertBuilder.On("Add", thirdTx, sequence).Return(nil).Once()
+	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 	s.Assert().NoError(s.processor.Commit(s.ctx))
 
 	err := s.processor.ProcessTransaction(s.ctx, firstTx)
@@ -66,7 +69,7 @@ func (s *TransactionsProcessorTestSuiteLedger) TestAddTransactionsSucceeds() {
 func (s *TransactionsProcessorTestSuiteLedger) TestAddTransactionsFails() {
 	sequence := uint32(20)
 	firstTx := createTransaction(true, 1)
-	s.mockBatchInsertBuilder.On("Add", s.ctx, firstTx, sequence).
+	s.mockBatchInsertBuilder.On("Add", firstTx, sequence).
 		Return(errors.New("transient error")).Once()
 
 	err := s.processor.ProcessTransaction(s.ctx, firstTx)
@@ -78,8 +81,8 @@ func (s *TransactionsProcessorTestSuiteLedger) TestExecFails() {
 	sequence := uint32(20)
 	firstTx := createTransaction(true, 1)
 
-	s.mockBatchInsertBuilder.On("Add", s.ctx, firstTx, sequence).Return(nil).Once()
-	s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(errors.New("transient error")).Once()
+	s.mockBatchInsertBuilder.On("Add", firstTx, sequence).Return(nil).Once()
+	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(errors.New("transient error")).Once()
 
 	err := s.processor.ProcessTransaction(s.ctx, firstTx)
 	s.Assert().NoError(err)

From e163964055130d67e648d4880f79910933601a03 Mon Sep 17 00:00:00 2001
From: tamirms <tamirms@gmail.com>
Date: Wed, 19 Jul 2023 06:17:48 +0100
Subject: [PATCH 04/17] services/horizon/internal/db2/history: Use
 FastBatchInsertBuilder to insert operations into history_operations (#4952)

---
 .github/workflows/go.yml                      |  2 +-
 .github/workflows/horizon.yml                 |  2 +-
 .../internal/db2/history/fee_bump_scenario.go |  5 ++---
 .../mock_operations_batch_insert_builder.go   |  9 ++++----
 .../internal/db2/history/mock_q_operations.go |  4 ++--
 .../horizon/internal/db2/history/operation.go |  2 +-
 .../history/operation_batch_insert_builder.go | 21 ++++++++-----------
 .../operation_batch_insert_builder_test.go    |  6 +++---
 .../internal/db2/history/operation_test.go    |  8 ++-----
 .../internal/ingest/processor_runner.go       |  2 +-
 .../internal/ingest/processor_runner_test.go  | 10 ++++-----
 .../ingest/processors/operations_processor.go | 11 ++++++----
 .../processors/operations_processor_test.go   | 12 ++++++-----
 13 files changed, 46 insertions(+), 48 deletions(-)

diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 795ff54318..2de770423a 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -57,7 +57,7 @@ jobs:
       matrix:
         os: [ubuntu-20.04]
         go: [1.19.6, 1.20.1]
-        pg: [9.6.5, 10]
+        pg: [10]
     runs-on: ${{ matrix.os }}
     services:
       postgres:
diff --git a/.github/workflows/horizon.yml b/.github/workflows/horizon.yml
index 68fa123de3..933252ce0c 100644
--- a/.github/workflows/horizon.yml
+++ b/.github/workflows/horizon.yml
@@ -13,7 +13,7 @@ jobs:
       matrix:
         os: [ubuntu-20.04]
         go: [1.19.6, 1.20.1]
-        pg: [9.6.5]
+        pg: [10]
         ingestion-backend: [db, captive-core, captive-core-remote-storage]
         protocol-version: [19]
     runs-on: ${{ matrix.os }}
diff --git a/services/horizon/internal/db2/history/fee_bump_scenario.go b/services/horizon/internal/db2/history/fee_bump_scenario.go
index 0c279d737f..46cc423227 100644
--- a/services/horizon/internal/db2/history/fee_bump_scenario.go
+++ b/services/horizon/internal/db2/history/fee_bump_scenario.go
@@ -264,14 +264,13 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 	account := fixture.Envelope.SourceAccount().ToAccountId()
 	feeBumpAccount := fixture.Envelope.FeeBumpAccount().ToAccountId()
 
-	opBuilder := q.NewOperationBatchInsertBuilder(1)
+	opBuilder := q.NewOperationBatchInsertBuilder()
 	details, err := json.Marshal(map[string]string{
 		"bump_to": "98",
 	})
 	tt.Assert.NoError(err)
 
 	tt.Assert.NoError(opBuilder.Add(
-		ctx,
 		toid.New(fixture.Ledger.Sequence, 1, 1).ToInt64(),
 		toid.New(fixture.Ledger.Sequence, 1, 0).ToInt64(),
 		1,
@@ -280,7 +279,7 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 		account.Address(),
 		null.String{},
 	))
-	tt.Assert.NoError(opBuilder.Exec(ctx))
+	tt.Assert.NoError(opBuilder.Exec(ctx, q))
 
 	effectBuilder := q.NewEffectBatchInsertBuilder(2)
 	details, err = json.Marshal(map[string]interface{}{"new_seq": 98})
diff --git a/services/horizon/internal/db2/history/mock_operations_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_operations_batch_insert_builder.go
index e57eb93db9..34cda0d3f8 100644
--- a/services/horizon/internal/db2/history/mock_operations_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/mock_operations_batch_insert_builder.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/guregu/null"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/xdr"
 	"github.com/stretchr/testify/mock"
 )
@@ -14,7 +15,7 @@ type MockOperationsBatchInsertBuilder struct {
 }
 
 // Add mock
-func (m *MockOperationsBatchInsertBuilder) Add(ctx context.Context,
+func (m *MockOperationsBatchInsertBuilder) Add(
 	id int64,
 	transactionID int64,
 	applicationOrder uint32,
@@ -23,7 +24,7 @@ func (m *MockOperationsBatchInsertBuilder) Add(ctx context.Context,
 	sourceAccount string,
 	sourceAccountMuxed null.String,
 ) error {
-	a := m.Called(ctx,
+	a := m.Called(
 		id,
 		transactionID,
 		applicationOrder,
@@ -36,7 +37,7 @@ func (m *MockOperationsBatchInsertBuilder) Add(ctx context.Context,
 }
 
 // Exec mock
-func (m *MockOperationsBatchInsertBuilder) Exec(ctx context.Context) error {
-	a := m.Called(ctx)
+func (m *MockOperationsBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	a := m.Called(ctx, session)
 	return a.Error(0)
 }
diff --git a/services/horizon/internal/db2/history/mock_q_operations.go b/services/horizon/internal/db2/history/mock_q_operations.go
index 08a97c6da9..6c2741ad3e 100644
--- a/services/horizon/internal/db2/history/mock_q_operations.go
+++ b/services/horizon/internal/db2/history/mock_q_operations.go
@@ -8,7 +8,7 @@ type MockQOperations struct {
 }
 
 // NewOperationBatchInsertBuilder mock
-func (m *MockQOperations) NewOperationBatchInsertBuilder(maxBatchSize int) OperationBatchInsertBuilder {
-	a := m.Called(maxBatchSize)
+func (m *MockQOperations) NewOperationBatchInsertBuilder() OperationBatchInsertBuilder {
+	a := m.Called()
 	return a.Get(0).(OperationBatchInsertBuilder)
 }
diff --git a/services/horizon/internal/db2/history/operation.go b/services/horizon/internal/db2/history/operation.go
index af1741b0d0..c1b5d71bb9 100644
--- a/services/horizon/internal/db2/history/operation.go
+++ b/services/horizon/internal/db2/history/operation.go
@@ -379,7 +379,7 @@ func validateTransactionForOperation(transaction Transaction, operation Operatio
 
 // QOperations defines history_operation related queries.
 type QOperations interface {
-	NewOperationBatchInsertBuilder(maxBatchSize int) OperationBatchInsertBuilder
+	NewOperationBatchInsertBuilder() OperationBatchInsertBuilder
 }
 
 var selectOperation = sq.Select(
diff --git a/services/horizon/internal/db2/history/operation_batch_insert_builder.go b/services/horizon/internal/db2/history/operation_batch_insert_builder.go
index a3baee8863..9922ee7d7c 100644
--- a/services/horizon/internal/db2/history/operation_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/operation_batch_insert_builder.go
@@ -12,7 +12,6 @@ import (
 // history_operations table
 type OperationBatchInsertBuilder interface {
 	Add(
-		ctx context.Context,
 		id int64,
 		transactionID int64,
 		applicationOrder uint32,
@@ -21,27 +20,25 @@ type OperationBatchInsertBuilder interface {
 		sourceAccount string,
 		sourceAcccountMuxed null.String,
 	) error
-	Exec(ctx context.Context) error
+	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
 // operationBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder
 type operationBatchInsertBuilder struct {
-	builder db.BatchInsertBuilder
+	builder db.FastBatchInsertBuilder
+	table   string
 }
 
 // NewOperationBatchInsertBuilder constructs a new TransactionBatchInsertBuilder instance
-func (q *Q) NewOperationBatchInsertBuilder(maxBatchSize int) OperationBatchInsertBuilder {
+func (q *Q) NewOperationBatchInsertBuilder() OperationBatchInsertBuilder {
 	return &operationBatchInsertBuilder{
-		builder: db.BatchInsertBuilder{
-			Table:        q.GetTable("history_operations"),
-			MaxBatchSize: maxBatchSize,
-		},
+		table:   "history_operations",
+		builder: db.FastBatchInsertBuilder{},
 	}
 }
 
 // Add adds a transaction's operations to the batch
 func (i *operationBatchInsertBuilder) Add(
-	ctx context.Context,
 	id int64,
 	transactionID int64,
 	applicationOrder uint32,
@@ -50,7 +47,7 @@ func (i *operationBatchInsertBuilder) Add(
 	sourceAccount string,
 	sourceAccountMuxed null.String,
 ) error {
-	return i.builder.Row(ctx, map[string]interface{}{
+	return i.builder.Row(map[string]interface{}{
 		"id":                   id,
 		"transaction_id":       transactionID,
 		"application_order":    applicationOrder,
@@ -62,6 +59,6 @@ func (i *operationBatchInsertBuilder) Add(
 
 }
 
-func (i *operationBatchInsertBuilder) Exec(ctx context.Context) error {
-	return i.builder.Exec(ctx)
+func (i *operationBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	return i.builder.Exec(ctx, session, i.table)
 }
diff --git a/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go b/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go
index 4ad624745c..cf10c19d2b 100644
--- a/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go
+++ b/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go
@@ -20,7 +20,7 @@ func TestAddOperation(t *testing.T) {
 
 	txBatch := q.NewTransactionBatchInsertBuilder()
 
-	builder := q.NewOperationBatchInsertBuilder(1)
+	builder := q.NewOperationBatchInsertBuilder()
 
 	transactionHash := "2a805712c6d10f9e74bb0ccf54ae92a2b4b1e586451fe8133a2433816f6b567c"
 	transactionResult := "AAAAAAAAAGQAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAA="
@@ -50,7 +50,7 @@ func TestAddOperation(t *testing.T) {
 
 	sourceAccount := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY"
 	sourceAccountMuxed := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26"
-	err = builder.Add(tt.Ctx,
+	err = builder.Add(
 		toid.New(sequence, 1, 1).ToInt64(),
 		toid.New(sequence, 1, 0).ToInt64(),
 		1,
@@ -61,7 +61,7 @@ func TestAddOperation(t *testing.T) {
 	)
 	tt.Assert.NoError(err)
 
-	err = builder.Exec(tt.Ctx)
+	err = builder.Exec(tt.Ctx, q)
 	tt.Assert.NoError(err)
 
 	tt.Assert.NoError(q.Commit())
diff --git a/services/horizon/internal/db2/history/operation_test.go b/services/horizon/internal/db2/history/operation_test.go
index 8c28809568..a010dda56f 100644
--- a/services/horizon/internal/db2/history/operation_test.go
+++ b/services/horizon/internal/db2/history/operation_test.go
@@ -95,9 +95,8 @@ func TestOperationByLiquidityPool(t *testing.T) {
 	tt.Assert.NoError(err)
 
 	// Insert a two phony operations
-	operationBuilder := q.NewOperationBatchInsertBuilder(2)
+	operationBuilder := q.NewOperationBatchInsertBuilder()
 	err = operationBuilder.Add(
-		tt.Ctx,
 		opID1,
 		txID,
 		1,
@@ -107,11 +106,8 @@ func TestOperationByLiquidityPool(t *testing.T) {
 		null.String{},
 	)
 	tt.Assert.NoError(err)
-	err = operationBuilder.Exec(tt.Ctx)
-	tt.Assert.NoError(err)
 
 	err = operationBuilder.Add(
-		tt.Ctx,
 		opID2,
 		txID,
 		1,
@@ -121,7 +117,7 @@ func TestOperationByLiquidityPool(t *testing.T) {
 		null.String{},
 	)
 	tt.Assert.NoError(err)
-	err = operationBuilder.Exec(tt.Ctx)
+	err = operationBuilder.Exec(tt.Ctx, q)
 	tt.Assert.NoError(err)
 
 	// Insert Liquidity Pool history
diff --git a/services/horizon/internal/ingest/processor_runner.go b/services/horizon/internal/ingest/processor_runner.go
index 4dc3d1eda9..b64e9db245 100644
--- a/services/horizon/internal/ingest/processor_runner.go
+++ b/services/horizon/internal/ingest/processor_runner.go
@@ -146,7 +146,7 @@ func (s *ProcessorRunner) buildTransactionProcessor(
 		statsLedgerTransactionProcessor,
 		processors.NewEffectProcessor(s.historyQ, sequence),
 		processors.NewLedgerProcessor(s.session, s.historyQ, ledger, CurrentVersion),
-		processors.NewOperationProcessor(s.historyQ, sequence),
+		processors.NewOperationProcessor(s.session, s.historyQ, sequence),
 		tradeProcessor,
 		processors.NewParticipantsProcessor(s.historyQ, sequence),
 		processors.NewTransactionProcessor(s.session, s.historyQ, sequence),
diff --git a/services/horizon/internal/ingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go
index dbbe3a5191..c01ee53730 100644
--- a/services/horizon/internal/ingest/processor_runner_test.go
+++ b/services/horizon/internal/ingest/processor_runner_test.go
@@ -239,7 +239,7 @@ func TestProcessorRunnerBuildTransactionProcessor(t *testing.T) {
 	q := &mockDBQ{}
 	defer mock.AssertExpectationsForObjects(t, q)
 
-	q.MockQOperations.On("NewOperationBatchInsertBuilder", maxBatchSize).
+	q.MockQOperations.On("NewOperationBatchInsertBuilder").
 		Return(&history.MockOperationsBatchInsertBuilder{}).Twice() // Twice = with/without failed
 	q.MockQTransactions.On("NewTransactionBatchInsertBuilder").
 		Return(&history.MockTransactionsBatchInsertBuilder{}).Twice()
@@ -298,8 +298,8 @@ func TestProcessorRunnerWithFilterEnabled(t *testing.T) {
 
 	mockOperationsBatchInsertBuilder := &history.MockOperationsBatchInsertBuilder{}
 	defer mock.AssertExpectationsForObjects(t, mockOperationsBatchInsertBuilder)
-	mockOperationsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once()
-	q.MockQOperations.On("NewOperationBatchInsertBuilder", maxBatchSize).
+	mockOperationsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once()
+	q.MockQOperations.On("NewOperationBatchInsertBuilder").
 		Return(mockOperationsBatchInsertBuilder).Twice()
 
 	mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{}
@@ -371,8 +371,8 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) {
 
 	mockOperationsBatchInsertBuilder := &history.MockOperationsBatchInsertBuilder{}
 	defer mock.AssertExpectationsForObjects(t, mockOperationsBatchInsertBuilder)
-	mockOperationsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once()
-	q.MockQOperations.On("NewOperationBatchInsertBuilder", maxBatchSize).
+	mockOperationsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once()
+	q.MockQOperations.On("NewOperationBatchInsertBuilder").
 		Return(mockOperationsBatchInsertBuilder).Twice()
 
 	mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{}
diff --git a/services/horizon/internal/ingest/processors/operations_processor.go b/services/horizon/internal/ingest/processors/operations_processor.go
index a194d5472c..1d6972f91f 100644
--- a/services/horizon/internal/ingest/processors/operations_processor.go
+++ b/services/horizon/internal/ingest/processors/operations_processor.go
@@ -11,6 +11,7 @@ import (
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/protocols/horizon/base"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
@@ -18,17 +19,19 @@ import (
 
 // OperationProcessor operations processor
 type OperationProcessor struct {
+	session     db.SessionInterface
 	operationsQ history.QOperations
 
 	sequence uint32
 	batch    history.OperationBatchInsertBuilder
 }
 
-func NewOperationProcessor(operationsQ history.QOperations, sequence uint32) *OperationProcessor {
+func NewOperationProcessor(session db.SessionInterface, operationsQ history.QOperations, sequence uint32) *OperationProcessor {
 	return &OperationProcessor{
+		session:     session,
 		operationsQ: operationsQ,
 		sequence:    sequence,
-		batch:       operationsQ.NewOperationBatchInsertBuilder(maxBatchSize),
+		batch:       operationsQ.NewOperationBatchInsertBuilder(),
 	}
 }
 
@@ -57,7 +60,7 @@ func (p *OperationProcessor) ProcessTransaction(ctx context.Context, transaction
 		if source.Type == xdr.CryptoKeyTypeKeyTypeMuxedEd25519 {
 			sourceAccountMuxed = null.StringFrom(source.Address())
 		}
-		if err := p.batch.Add(ctx,
+		if err := p.batch.Add(
 			operation.ID(),
 			operation.TransactionID(),
 			operation.Order(),
@@ -74,7 +77,7 @@ func (p *OperationProcessor) ProcessTransaction(ctx context.Context, transaction
 }
 
 func (p *OperationProcessor) Commit(ctx context.Context) error {
-	return p.batch.Exec(ctx)
+	return p.batch.Exec(ctx, p.session)
 }
 
 // transactionOperationWrapper represents the data for a single operation within a transaction
diff --git a/services/horizon/internal/ingest/processors/operations_processor_test.go b/services/horizon/internal/ingest/processors/operations_processor_test.go
index 79b94b1f7f..af5e585ddd 100644
--- a/services/horizon/internal/ingest/processors/operations_processor_test.go
+++ b/services/horizon/internal/ingest/processors/operations_processor_test.go
@@ -13,6 +13,7 @@ import (
 
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/xdr"
 )
@@ -21,6 +22,7 @@ type OperationsProcessorTestSuiteLedger struct {
 	suite.Suite
 	ctx                    context.Context
 	processor              *OperationProcessor
+	mockSession            *db.MockSession
 	mockQ                  *history.MockQOperations
 	mockBatchInsertBuilder *history.MockOperationsBatchInsertBuilder
 }
@@ -34,10 +36,11 @@ func (s *OperationsProcessorTestSuiteLedger) SetupTest() {
 	s.mockQ = &history.MockQOperations{}
 	s.mockBatchInsertBuilder = &history.MockOperationsBatchInsertBuilder{}
 	s.mockQ.
-		On("NewOperationBatchInsertBuilder", maxBatchSize).
+		On("NewOperationBatchInsertBuilder").
 		Return(s.mockBatchInsertBuilder).Once()
 
 	s.processor = NewOperationProcessor(
+		s.mockSession,
 		s.mockQ,
 		56,
 	)
@@ -74,7 +77,6 @@ func (s *OperationsProcessorTestSuiteLedger) mockBatchInsertAdds(txs []ingest.Le
 			}
 			s.mockBatchInsertBuilder.On(
 				"Add",
-				s.ctx,
 				expected.ID(),
 				expected.TransactionID(),
 				expected.Order(),
@@ -122,7 +124,7 @@ func (s *OperationsProcessorTestSuiteLedger) TestAddOperationSucceeds() {
 
 	err = s.mockBatchInsertAdds(txs, uint32(56))
 	s.Assert().NoError(err)
-	s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once()
+	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 	s.Assert().NoError(s.processor.Commit(s.ctx))
 
 	for _, tx := range txs {
@@ -136,7 +138,7 @@ func (s *OperationsProcessorTestSuiteLedger) TestAddOperationFails() {
 
 	s.mockBatchInsertBuilder.
 		On(
-			"Add", s.ctx,
+			"Add",
 			mock.Anything,
 			mock.Anything,
 			mock.Anything,
@@ -152,7 +154,7 @@ func (s *OperationsProcessorTestSuiteLedger) TestAddOperationFails() {
 }
 
 func (s *OperationsProcessorTestSuiteLedger) TestExecFails() {
-	s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(errors.New("transient error")).Once()
+	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(errors.New("transient error")).Once()
 	err := s.processor.Commit(s.ctx)
 	s.Assert().Error(err)
 	s.Assert().EqualError(err, "transient error")

From 6564385eabf13a021ad35f3429dac2c381ac9833 Mon Sep 17 00:00:00 2001
From: tamirms <tamirms@gmail.com>
Date: Wed, 19 Jul 2023 17:06:40 +0100
Subject: [PATCH 05/17] services/horizon/internal/db2/history: Use
 FastBatchInsertBuilder for claimable balances, liquidity pools, and
 participants tables (#4971)

---
 .../internal/db2/history/effect_test.go       |  9 ++-
 .../db2/history/history_claimable_balances.go | 50 ++++++-------
 .../db2/history/history_liquidity_pools.go    | 50 ++++++-------
 services/horizon/internal/db2/history/main.go |  4 +-
 ...ration_participant_batch_insert_builder.go | 11 ++-
 .../mock_q_history_claimable_balances.go      | 26 ++++---
 .../history/mock_q_history_liquidity_pools.go | 26 ++++---
 .../db2/history/mock_q_participants.go        | 21 +++---
 ...ration_participant_batch_insert_builder.go | 23 +++---
 ...n_participant_batch_insert_builder_test.go |  8 +-
 .../internal/db2/history/operation_test.go    |  8 +-
 .../internal/db2/history/participants.go      | 27 ++++---
 .../internal/db2/history/participants_test.go | 10 ++-
 .../internal/db2/history/transaction_test.go  |  6 +-
 services/horizon/internal/ingest/main_test.go |  8 +-
 .../internal/ingest/processor_runner.go       |  6 +-
 ...laimable_balances_transaction_processor.go | 17 +++--
 ...ble_balances_transaction_processor_test.go | 15 ++--
 .../liquidity_pools_transaction_processor.go  | 17 +++--
 ...uidity_pools_transaction_processor_test.go | 15 ++--
 .../processors/participants_processor.go      | 17 +++--
 .../processors/participants_processor_test.go | 75 ++++++++++---------
 22 files changed, 238 insertions(+), 211 deletions(-)

diff --git a/services/horizon/internal/db2/history/effect_test.go b/services/horizon/internal/db2/history/effect_test.go
index 6430049ab4..21ea63e308 100644
--- a/services/horizon/internal/db2/history/effect_test.go
+++ b/services/horizon/internal/db2/history/effect_test.go
@@ -48,14 +48,17 @@ func TestEffectsForLiquidityPool(t *testing.T) {
 	liquidityPoolID := "abcde"
 	toInternalID, err := q.CreateHistoryLiquidityPools(tt.Ctx, []string{liquidityPoolID}, 2)
 	tt.Assert.NoError(err)
-	operationBuilder := q.NewOperationLiquidityPoolBatchInsertBuilder(2)
+
+	tt.Assert.NoError(q.Begin())
+	operationBuilder := q.NewOperationLiquidityPoolBatchInsertBuilder()
 	tt.Assert.NoError(err)
 	internalID, ok := toInternalID[liquidityPoolID]
 	tt.Assert.True(ok)
-	err = operationBuilder.Add(tt.Ctx, opID, internalID)
+	err = operationBuilder.Add(opID, internalID)
 	tt.Assert.NoError(err)
-	err = operationBuilder.Exec(tt.Ctx)
+	err = operationBuilder.Exec(tt.Ctx, q)
 	tt.Assert.NoError(err)
+	tt.Assert.NoError(q.Commit())
 
 	var result []Effect
 	err = q.Effects().ForLiquidityPool(tt.Ctx, db2.PageQuery{
diff --git a/services/horizon/internal/db2/history/history_claimable_balances.go b/services/horizon/internal/db2/history/history_claimable_balances.go
index b67294f4a6..162c31a70b 100644
--- a/services/horizon/internal/db2/history/history_claimable_balances.go
+++ b/services/horizon/internal/db2/history/history_claimable_balances.go
@@ -13,8 +13,8 @@ import (
 // QHistoryClaimableBalances defines account related queries.
 type QHistoryClaimableBalances interface {
 	CreateHistoryClaimableBalances(ctx context.Context, ids []string, batchSize int) (map[string]int64, error)
-	NewOperationClaimableBalanceBatchInsertBuilder(maxBatchSize int) OperationClaimableBalanceBatchInsertBuilder
-	NewTransactionClaimableBalanceBatchInsertBuilder(maxBatchSize int) TransactionClaimableBalanceBatchInsertBuilder
+	NewOperationClaimableBalanceBatchInsertBuilder() OperationClaimableBalanceBatchInsertBuilder
+	NewTransactionClaimableBalanceBatchInsertBuilder() TransactionClaimableBalanceBatchInsertBuilder
 }
 
 // CreateHistoryClaimableBalances creates rows in the history_claimable_balances table for a given list of ids.
@@ -92,63 +92,61 @@ func (q *Q) ClaimableBalanceByID(ctx context.Context, id string) (dest HistoryCl
 }
 
 type OperationClaimableBalanceBatchInsertBuilder interface {
-	Add(ctx context.Context, operationID, internalID int64) error
-	Exec(ctx context.Context) error
+	Add(operationID, internalID int64) error
+	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
 type operationClaimableBalanceBatchInsertBuilder struct {
-	builder db.BatchInsertBuilder
+	table   string
+	builder db.FastBatchInsertBuilder
 }
 
-func (q *Q) NewOperationClaimableBalanceBatchInsertBuilder(maxBatchSize int) OperationClaimableBalanceBatchInsertBuilder {
+func (q *Q) NewOperationClaimableBalanceBatchInsertBuilder() OperationClaimableBalanceBatchInsertBuilder {
 	return &operationClaimableBalanceBatchInsertBuilder{
-		builder: db.BatchInsertBuilder{
-			Table:        q.GetTable("history_operation_claimable_balances"),
-			MaxBatchSize: maxBatchSize,
-		},
+		table:   "history_operation_claimable_balances",
+		builder: db.FastBatchInsertBuilder{},
 	}
 }
 
 // Add adds a new operation claimable balance to the batch
-func (i *operationClaimableBalanceBatchInsertBuilder) Add(ctx context.Context, operationID, internalID int64) error {
-	return i.builder.Row(ctx, map[string]interface{}{
+func (i *operationClaimableBalanceBatchInsertBuilder) Add(operationID, internalID int64) error {
+	return i.builder.Row(map[string]interface{}{
 		"history_operation_id":         operationID,
 		"history_claimable_balance_id": internalID,
 	})
 }
 
 // Exec flushes all pending operation claimable balances to the db
-func (i *operationClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context) error {
-	return i.builder.Exec(ctx)
+func (i *operationClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	return i.builder.Exec(ctx, session, i.table)
 }
 
 type TransactionClaimableBalanceBatchInsertBuilder interface {
-	Add(ctx context.Context, transactionID, internalID int64) error
-	Exec(ctx context.Context) error
+	Add(transactionID, internalID int64) error
+	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
 type transactionClaimableBalanceBatchInsertBuilder struct {
-	builder db.BatchInsertBuilder
+	table   string
+	builder db.FastBatchInsertBuilder
 }
 
-func (q *Q) NewTransactionClaimableBalanceBatchInsertBuilder(maxBatchSize int) TransactionClaimableBalanceBatchInsertBuilder {
+func (q *Q) NewTransactionClaimableBalanceBatchInsertBuilder() TransactionClaimableBalanceBatchInsertBuilder {
 	return &transactionClaimableBalanceBatchInsertBuilder{
-		builder: db.BatchInsertBuilder{
-			Table:        q.GetTable("history_transaction_claimable_balances"),
-			MaxBatchSize: maxBatchSize,
-		},
+		table:   "history_transaction_claimable_balances",
+		builder: db.FastBatchInsertBuilder{},
 	}
 }
 
 // Add adds a new transaction claimable balance to the batch
-func (i *transactionClaimableBalanceBatchInsertBuilder) Add(ctx context.Context, transactionID, internalID int64) error {
-	return i.builder.Row(ctx, map[string]interface{}{
+func (i *transactionClaimableBalanceBatchInsertBuilder) Add(transactionID, internalID int64) error {
+	return i.builder.Row(map[string]interface{}{
 		"history_transaction_id":       transactionID,
 		"history_claimable_balance_id": internalID,
 	})
 }
 
 // Exec flushes all pending transaction claimable balances to the db
-func (i *transactionClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context) error {
-	return i.builder.Exec(ctx)
+func (i *transactionClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	return i.builder.Exec(ctx, session, i.table)
 }
diff --git a/services/horizon/internal/db2/history/history_liquidity_pools.go b/services/horizon/internal/db2/history/history_liquidity_pools.go
index 0601d91be7..3953280aa9 100644
--- a/services/horizon/internal/db2/history/history_liquidity_pools.go
+++ b/services/horizon/internal/db2/history/history_liquidity_pools.go
@@ -12,8 +12,8 @@ import (
 // QHistoryLiquidityPools defines account related queries.
 type QHistoryLiquidityPools interface {
 	CreateHistoryLiquidityPools(ctx context.Context, poolIDs []string, batchSize int) (map[string]int64, error)
-	NewOperationLiquidityPoolBatchInsertBuilder(maxBatchSize int) OperationLiquidityPoolBatchInsertBuilder
-	NewTransactionLiquidityPoolBatchInsertBuilder(maxBatchSize int) TransactionLiquidityPoolBatchInsertBuilder
+	NewOperationLiquidityPoolBatchInsertBuilder() OperationLiquidityPoolBatchInsertBuilder
+	NewTransactionLiquidityPoolBatchInsertBuilder() TransactionLiquidityPoolBatchInsertBuilder
 }
 
 // CreateHistoryLiquidityPools creates rows in the history_liquidity_pools table for a given list of ids.
@@ -101,63 +101,61 @@ func (q *Q) LiquidityPoolByID(ctx context.Context, poolID string) (dest HistoryL
 }
 
 type OperationLiquidityPoolBatchInsertBuilder interface {
-	Add(ctx context.Context, operationID, internalID int64) error
-	Exec(ctx context.Context) error
+	Add(operationID, internalID int64) error
+	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
 type operationLiquidityPoolBatchInsertBuilder struct {
-	builder db.BatchInsertBuilder
+	table   string
+	builder db.FastBatchInsertBuilder
 }
 
-func (q *Q) NewOperationLiquidityPoolBatchInsertBuilder(maxBatchSize int) OperationLiquidityPoolBatchInsertBuilder {
+func (q *Q) NewOperationLiquidityPoolBatchInsertBuilder() OperationLiquidityPoolBatchInsertBuilder {
 	return &operationLiquidityPoolBatchInsertBuilder{
-		builder: db.BatchInsertBuilder{
-			Table:        q.GetTable("history_operation_liquidity_pools"),
-			MaxBatchSize: maxBatchSize,
-		},
+		table:   "history_operation_liquidity_pools",
+		builder: db.FastBatchInsertBuilder{},
 	}
 }
 
 // Add adds a new operation claimable balance to the batch
-func (i *operationLiquidityPoolBatchInsertBuilder) Add(ctx context.Context, operationID, internalID int64) error {
-	return i.builder.Row(ctx, map[string]interface{}{
+func (i *operationLiquidityPoolBatchInsertBuilder) Add(operationID, internalID int64) error {
+	return i.builder.Row(map[string]interface{}{
 		"history_operation_id":      operationID,
 		"history_liquidity_pool_id": internalID,
 	})
 }
 
 // Exec flushes all pending operation claimable balances to the db
-func (i *operationLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context) error {
-	return i.builder.Exec(ctx)
+func (i *operationLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	return i.builder.Exec(ctx, session, i.table)
 }
 
 type TransactionLiquidityPoolBatchInsertBuilder interface {
-	Add(ctx context.Context, transactionID, internalID int64) error
-	Exec(ctx context.Context) error
+	Add(transactionID, internalID int64) error
+	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
 type transactionLiquidityPoolBatchInsertBuilder struct {
-	builder db.BatchInsertBuilder
+	table   string
+	builder db.FastBatchInsertBuilder
 }
 
-func (q *Q) NewTransactionLiquidityPoolBatchInsertBuilder(maxBatchSize int) TransactionLiquidityPoolBatchInsertBuilder {
+func (q *Q) NewTransactionLiquidityPoolBatchInsertBuilder() TransactionLiquidityPoolBatchInsertBuilder {
 	return &transactionLiquidityPoolBatchInsertBuilder{
-		builder: db.BatchInsertBuilder{
-			Table:        q.GetTable("history_transaction_liquidity_pools"),
-			MaxBatchSize: maxBatchSize,
-		},
+		table:   "history_transaction_liquidity_pools",
+		builder: db.FastBatchInsertBuilder{},
 	}
 }
 
 // Add adds a new transaction claimable balance to the batch
-func (i *transactionLiquidityPoolBatchInsertBuilder) Add(ctx context.Context, transactionID, internalID int64) error {
-	return i.builder.Row(ctx, map[string]interface{}{
+func (i *transactionLiquidityPoolBatchInsertBuilder) Add(transactionID, internalID int64) error {
+	return i.builder.Row(map[string]interface{}{
 		"history_transaction_id":    transactionID,
 		"history_liquidity_pool_id": internalID,
 	})
 }
 
 // Exec flushes all pending transaction claimable balances to the db
-func (i *transactionLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context) error {
-	return i.builder.Exec(ctx)
+func (i *transactionLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	return i.builder.Exec(ctx, session, i.table)
 }
diff --git a/services/horizon/internal/db2/history/main.go b/services/horizon/internal/db2/history/main.go
index 2fc9b249c2..bbf8bad912 100644
--- a/services/horizon/internal/db2/history/main.go
+++ b/services/horizon/internal/db2/history/main.go
@@ -255,8 +255,8 @@ type IngestionQ interface {
 	// QParticipants
 	// Copy the small interfaces with shared methods directly, otherwise error:
 	// duplicate method CreateAccounts
-	NewTransactionParticipantsBatchInsertBuilder(maxBatchSize int) TransactionParticipantsBatchInsertBuilder
-	NewOperationParticipantBatchInsertBuilder(maxBatchSize int) OperationParticipantBatchInsertBuilder
+	NewTransactionParticipantsBatchInsertBuilder() TransactionParticipantsBatchInsertBuilder
+	NewOperationParticipantBatchInsertBuilder() OperationParticipantBatchInsertBuilder
 	QSigners
 	//QTrades
 	NewTradeBatchInsertBuilder(maxBatchSize int) TradeBatchInsertBuilder
diff --git a/services/horizon/internal/db2/history/mock_operation_participant_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_operation_participant_batch_insert_builder.go
index 014763f989..7c98ad2729 100644
--- a/services/horizon/internal/db2/history/mock_operation_participant_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/mock_operation_participant_batch_insert_builder.go
@@ -2,6 +2,9 @@ package history
 
 import (
 	"context"
+
+	"github.com/stellar/go/support/db"
+
 	"github.com/stretchr/testify/mock"
 )
 
@@ -11,13 +14,13 @@ type MockOperationParticipantBatchInsertBuilder struct {
 }
 
 // Add mock
-func (m *MockOperationParticipantBatchInsertBuilder) Add(ctx context.Context, operationID int64, accountID int64) error {
-	a := m.Called(ctx, operationID, accountID)
+func (m *MockOperationParticipantBatchInsertBuilder) Add(operationID int64, accountID int64) error {
+	a := m.Called(operationID, accountID)
 	return a.Error(0)
 }
 
 // Exec mock
-func (m *MockOperationParticipantBatchInsertBuilder) Exec(ctx context.Context) error {
-	a := m.Called(ctx)
+func (m *MockOperationParticipantBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	a := m.Called(ctx, session)
 	return a.Error(0)
 }
diff --git a/services/horizon/internal/db2/history/mock_q_history_claimable_balances.go b/services/horizon/internal/db2/history/mock_q_history_claimable_balances.go
index 9b7fe1d3b6..6ce3926c91 100644
--- a/services/horizon/internal/db2/history/mock_q_history_claimable_balances.go
+++ b/services/horizon/internal/db2/history/mock_q_history_claimable_balances.go
@@ -3,6 +3,8 @@ package history
 import (
 	"context"
 
+	"github.com/stellar/go/support/db"
+
 	"github.com/stretchr/testify/mock"
 )
 
@@ -16,8 +18,8 @@ func (m *MockQHistoryClaimableBalances) CreateHistoryClaimableBalances(ctx conte
 	return a.Get(0).(map[string]int64), a.Error(1)
 }
 
-func (m *MockQHistoryClaimableBalances) NewTransactionClaimableBalanceBatchInsertBuilder(maxBatchSize int) TransactionClaimableBalanceBatchInsertBuilder {
-	a := m.Called(maxBatchSize)
+func (m *MockQHistoryClaimableBalances) NewTransactionClaimableBalanceBatchInsertBuilder() TransactionClaimableBalanceBatchInsertBuilder {
+	a := m.Called()
 	return a.Get(0).(TransactionClaimableBalanceBatchInsertBuilder)
 }
 
@@ -27,19 +29,19 @@ type MockTransactionClaimableBalanceBatchInsertBuilder struct {
 	mock.Mock
 }
 
-func (m *MockTransactionClaimableBalanceBatchInsertBuilder) Add(ctx context.Context, transactionID, accountID int64) error {
-	a := m.Called(ctx, transactionID, accountID)
+func (m *MockTransactionClaimableBalanceBatchInsertBuilder) Add(transactionID, accountID int64) error {
+	a := m.Called(transactionID, accountID)
 	return a.Error(0)
 }
 
-func (m *MockTransactionClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context) error {
-	a := m.Called(ctx)
+func (m *MockTransactionClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	a := m.Called(ctx, session)
 	return a.Error(0)
 }
 
 // NewOperationClaimableBalanceBatchInsertBuilder mock
-func (m *MockQHistoryClaimableBalances) NewOperationClaimableBalanceBatchInsertBuilder(maxBatchSize int) OperationClaimableBalanceBatchInsertBuilder {
-	a := m.Called(maxBatchSize)
+func (m *MockQHistoryClaimableBalances) NewOperationClaimableBalanceBatchInsertBuilder() OperationClaimableBalanceBatchInsertBuilder {
+	a := m.Called()
 	return a.Get(0).(OperationClaimableBalanceBatchInsertBuilder)
 }
 
@@ -49,12 +51,12 @@ type MockOperationClaimableBalanceBatchInsertBuilder struct {
 	mock.Mock
 }
 
-func (m *MockOperationClaimableBalanceBatchInsertBuilder) Add(ctx context.Context, transactionID, accountID int64) error {
-	a := m.Called(ctx, transactionID, accountID)
+func (m *MockOperationClaimableBalanceBatchInsertBuilder) Add(transactionID, accountID int64) error {
+	a := m.Called(transactionID, accountID)
 	return a.Error(0)
 }
 
-func (m *MockOperationClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context) error {
-	a := m.Called(ctx)
+func (m *MockOperationClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	a := m.Called(ctx, session)
 	return a.Error(0)
 }
diff --git a/services/horizon/internal/db2/history/mock_q_history_liquidity_pools.go b/services/horizon/internal/db2/history/mock_q_history_liquidity_pools.go
index 08f4920de2..33f5c8a46b 100644
--- a/services/horizon/internal/db2/history/mock_q_history_liquidity_pools.go
+++ b/services/horizon/internal/db2/history/mock_q_history_liquidity_pools.go
@@ -3,6 +3,8 @@ package history
 import (
 	"context"
 
+	"github.com/stellar/go/support/db"
+
 	"github.com/stretchr/testify/mock"
 )
 
@@ -16,8 +18,8 @@ func (m *MockQHistoryLiquidityPools) CreateHistoryLiquidityPools(ctx context.Con
 	return a.Get(0).(map[string]int64), a.Error(1)
 }
 
-func (m *MockQHistoryLiquidityPools) NewTransactionLiquidityPoolBatchInsertBuilder(maxBatchSize int) TransactionLiquidityPoolBatchInsertBuilder {
-	a := m.Called(maxBatchSize)
+func (m *MockQHistoryLiquidityPools) NewTransactionLiquidityPoolBatchInsertBuilder() TransactionLiquidityPoolBatchInsertBuilder {
+	a := m.Called()
 	return a.Get(0).(TransactionLiquidityPoolBatchInsertBuilder)
 }
 
@@ -27,19 +29,19 @@ type MockTransactionLiquidityPoolBatchInsertBuilder struct {
 	mock.Mock
 }
 
-func (m *MockTransactionLiquidityPoolBatchInsertBuilder) Add(ctx context.Context, transactionID, accountID int64) error {
-	a := m.Called(ctx, transactionID, accountID)
+func (m *MockTransactionLiquidityPoolBatchInsertBuilder) Add(transactionID, accountID int64) error {
+	a := m.Called(transactionID, accountID)
 	return a.Error(0)
 }
 
-func (m *MockTransactionLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context) error {
-	a := m.Called(ctx)
+func (m *MockTransactionLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	a := m.Called(ctx, session)
 	return a.Error(0)
 }
 
 // NewOperationLiquidityPoolBatchInsertBuilder mock
-func (m *MockQHistoryLiquidityPools) NewOperationLiquidityPoolBatchInsertBuilder(maxBatchSize int) OperationLiquidityPoolBatchInsertBuilder {
-	a := m.Called(maxBatchSize)
+func (m *MockQHistoryLiquidityPools) NewOperationLiquidityPoolBatchInsertBuilder() OperationLiquidityPoolBatchInsertBuilder {
+	a := m.Called()
 	return a.Get(0).(OperationLiquidityPoolBatchInsertBuilder)
 }
 
@@ -49,12 +51,12 @@ type MockOperationLiquidityPoolBatchInsertBuilder struct {
 	mock.Mock
 }
 
-func (m *MockOperationLiquidityPoolBatchInsertBuilder) Add(ctx context.Context, transactionID, accountID int64) error {
-	a := m.Called(ctx, transactionID, accountID)
+func (m *MockOperationLiquidityPoolBatchInsertBuilder) Add(transactionID, accountID int64) error {
+	a := m.Called(transactionID, accountID)
 	return a.Error(0)
 }
 
-func (m *MockOperationLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context) error {
-	a := m.Called(ctx)
+func (m *MockOperationLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	a := m.Called(ctx, session)
 	return a.Error(0)
 }
diff --git a/services/horizon/internal/db2/history/mock_q_participants.go b/services/horizon/internal/db2/history/mock_q_participants.go
index 9365e06db3..caeb6d23d7 100644
--- a/services/horizon/internal/db2/history/mock_q_participants.go
+++ b/services/horizon/internal/db2/history/mock_q_participants.go
@@ -2,6 +2,8 @@ package history
 
 import (
 	"context"
+
+	"github.com/stellar/go/support/db"
 	"github.com/stretchr/testify/mock"
 )
 
@@ -15,8 +17,8 @@ func (m *MockQParticipants) CreateAccounts(ctx context.Context, addresses []stri
 	return a.Get(0).(map[string]int64), a.Error(1)
 }
 
-func (m *MockQParticipants) NewTransactionParticipantsBatchInsertBuilder(maxBatchSize int) TransactionParticipantsBatchInsertBuilder {
-	a := m.Called(maxBatchSize)
+func (m *MockQParticipants) NewTransactionParticipantsBatchInsertBuilder() TransactionParticipantsBatchInsertBuilder {
+	a := m.Called()
 	return a.Get(0).(TransactionParticipantsBatchInsertBuilder)
 }
 
@@ -26,18 +28,19 @@ type MockTransactionParticipantsBatchInsertBuilder struct {
 	mock.Mock
 }
 
-func (m *MockTransactionParticipantsBatchInsertBuilder) Add(ctx context.Context, transactionID, accountID int64) error {
-	a := m.Called(ctx, transactionID, accountID)
+func (m *MockTransactionParticipantsBatchInsertBuilder) Add(transactionID, accountID int64) error {
+	a := m.Called(transactionID, accountID)
 	return a.Error(0)
 }
 
-func (m *MockTransactionParticipantsBatchInsertBuilder) Exec(ctx context.Context) error {
-	a := m.Called(ctx)
+func (m *MockTransactionParticipantsBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	a := m.Called(ctx, session)
 	return a.Error(0)
 }
 
 // NewOperationParticipantBatchInsertBuilder mock
-func (m *MockQParticipants) NewOperationParticipantBatchInsertBuilder(maxBatchSize int) OperationParticipantBatchInsertBuilder {
-	a := m.Called(maxBatchSize)
-	return a.Get(0).(OperationParticipantBatchInsertBuilder)
+func (m *MockQParticipants) NewOperationParticipantBatchInsertBuilder() OperationParticipantBatchInsertBuilder {
+	a := m.Called()
+	v := a.Get(0)
+	return v.(OperationParticipantBatchInsertBuilder)
 }
diff --git a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder.go b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder.go
index 6b2b3afb56..78ce63a0c1 100644
--- a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder.go
@@ -10,40 +10,37 @@ import (
 // history_operations table
 type OperationParticipantBatchInsertBuilder interface {
 	Add(
-		ctx context.Context,
 		operationID int64,
 		accountID int64,
 	) error
-	Exec(ctx context.Context) error
+	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
 // operationParticipantBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder
 type operationParticipantBatchInsertBuilder struct {
-	builder db.BatchInsertBuilder
+	table   string
+	builder db.FastBatchInsertBuilder
 }
 
-// NewOperationParticipantBatchInsertBuilder constructs a new TransactionBatchInsertBuilder instance
-func (q *Q) NewOperationParticipantBatchInsertBuilder(maxBatchSize int) OperationParticipantBatchInsertBuilder {
+// NewOperationParticipantBatchInsertBuilder constructs a new OperationParticipantBatchInsertBuilder instance
+func (q *Q) NewOperationParticipantBatchInsertBuilder() OperationParticipantBatchInsertBuilder {
 	return &operationParticipantBatchInsertBuilder{
-		builder: db.BatchInsertBuilder{
-			Table:        q.GetTable("history_operation_participants"),
-			MaxBatchSize: maxBatchSize,
-		},
+		table:   "history_operation_participants",
+		builder: db.FastBatchInsertBuilder{},
 	}
 }
 
 // Add adds an operation participant to the batch
 func (i *operationParticipantBatchInsertBuilder) Add(
-	ctx context.Context,
 	operationID int64,
 	accountID int64,
 ) error {
-	return i.builder.Row(ctx, map[string]interface{}{
+	return i.builder.Row(map[string]interface{}{
 		"history_operation_id": operationID,
 		"history_account_id":   accountID,
 	})
 }
 
-func (i *operationParticipantBatchInsertBuilder) Exec(ctx context.Context) error {
-	return i.builder.Exec(ctx)
+func (i *operationParticipantBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	return i.builder.Exec(ctx, session, i.table)
 }
diff --git a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
index 51a0c4800d..84e114d0b0 100644
--- a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
+++ b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
@@ -13,12 +13,14 @@ func TestAddOperationParticipants(t *testing.T) {
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
 
-	builder := q.NewOperationParticipantBatchInsertBuilder(1)
-	err := builder.Add(tt.Ctx, 240518172673, 1)
+	tt.Assert.NoError(q.Begin())
+	builder := q.NewOperationParticipantBatchInsertBuilder()
+	err := builder.Add(240518172673, 1)
 	tt.Assert.NoError(err)
 
-	err = builder.Exec(tt.Ctx)
+	err = builder.Exec(tt.Ctx, q)
 	tt.Assert.NoError(err)
+	tt.Assert.NoError(q.Commit())
 
 	type hop struct {
 		OperationID int64 `db:"history_operation_id"`
diff --git a/services/horizon/internal/db2/history/operation_test.go b/services/horizon/internal/db2/history/operation_test.go
index a010dda56f..a996ac49b7 100644
--- a/services/horizon/internal/db2/history/operation_test.go
+++ b/services/horizon/internal/db2/history/operation_test.go
@@ -124,15 +124,15 @@ func TestOperationByLiquidityPool(t *testing.T) {
 	liquidityPoolID := "a2f38836a839de008cf1d782c81f45e1253cc5d3dad9110b872965484fec0a49"
 	toInternalID, err := q.CreateHistoryLiquidityPools(tt.Ctx, []string{liquidityPoolID}, 2)
 	tt.Assert.NoError(err)
-	lpOperationBuilder := q.NewOperationLiquidityPoolBatchInsertBuilder(3)
+	lpOperationBuilder := q.NewOperationLiquidityPoolBatchInsertBuilder()
 	tt.Assert.NoError(err)
 	internalID, ok := toInternalID[liquidityPoolID]
 	tt.Assert.True(ok)
-	err = lpOperationBuilder.Add(tt.Ctx, opID1, internalID)
+	err = lpOperationBuilder.Add(opID1, internalID)
 	tt.Assert.NoError(err)
-	err = lpOperationBuilder.Add(tt.Ctx, opID2, internalID)
+	err = lpOperationBuilder.Add(opID2, internalID)
 	tt.Assert.NoError(err)
-	err = lpOperationBuilder.Exec(tt.Ctx)
+	err = lpOperationBuilder.Exec(tt.Ctx, q)
 	tt.Assert.NoError(err)
 
 	tt.Assert.NoError(q.Commit())
diff --git a/services/horizon/internal/db2/history/participants.go b/services/horizon/internal/db2/history/participants.go
index 658e877f78..58b0a80892 100644
--- a/services/horizon/internal/db2/history/participants.go
+++ b/services/horizon/internal/db2/history/participants.go
@@ -9,40 +9,39 @@ import (
 // QParticipants defines ingestion participant related queries.
 type QParticipants interface {
 	QCreateAccountsHistory
-	NewTransactionParticipantsBatchInsertBuilder(maxBatchSize int) TransactionParticipantsBatchInsertBuilder
-	NewOperationParticipantBatchInsertBuilder(maxBatchSize int) OperationParticipantBatchInsertBuilder
+	NewTransactionParticipantsBatchInsertBuilder() TransactionParticipantsBatchInsertBuilder
+	NewOperationParticipantBatchInsertBuilder() OperationParticipantBatchInsertBuilder
 }
 
 // TransactionParticipantsBatchInsertBuilder is used to insert transaction participants into the
 // history_transaction_participants table
 type TransactionParticipantsBatchInsertBuilder interface {
-	Add(ctx context.Context, transactionID, accountID int64) error
-	Exec(ctx context.Context) error
+	Add(transactionID, accountID int64) error
+	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
 type transactionParticipantsBatchInsertBuilder struct {
-	builder db.BatchInsertBuilder
+	tableName string
+	builder   db.FastBatchInsertBuilder
 }
 
 // NewTransactionParticipantsBatchInsertBuilder constructs a new TransactionParticipantsBatchInsertBuilder instance
-func (q *Q) NewTransactionParticipantsBatchInsertBuilder(maxBatchSize int) TransactionParticipantsBatchInsertBuilder {
+func (q *Q) NewTransactionParticipantsBatchInsertBuilder() TransactionParticipantsBatchInsertBuilder {
 	return &transactionParticipantsBatchInsertBuilder{
-		builder: db.BatchInsertBuilder{
-			Table:        q.GetTable("history_transaction_participants"),
-			MaxBatchSize: maxBatchSize,
-		},
+		tableName: "history_transaction_participants",
+		builder:   db.FastBatchInsertBuilder{},
 	}
 }
 
 // Add adds a new transaction participant to the batch
-func (i *transactionParticipantsBatchInsertBuilder) Add(ctx context.Context, transactionID, accountID int64) error {
-	return i.builder.Row(ctx, map[string]interface{}{
+func (i *transactionParticipantsBatchInsertBuilder) Add(transactionID, accountID int64) error {
+	return i.builder.Row(map[string]interface{}{
 		"history_transaction_id": transactionID,
 		"history_account_id":     accountID,
 	})
 }
 
 // Exec flushes all pending transaction participants to the db
-func (i *transactionParticipantsBatchInsertBuilder) Exec(ctx context.Context) error {
-	return i.builder.Exec(ctx)
+func (i *transactionParticipantsBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	return i.builder.Exec(ctx, session, i.tableName)
 }
diff --git a/services/horizon/internal/db2/history/participants_test.go b/services/horizon/internal/db2/history/participants_test.go
index 4aaf70b151..37f50706e6 100644
--- a/services/horizon/internal/db2/history/participants_test.go
+++ b/services/horizon/internal/db2/history/participants_test.go
@@ -32,18 +32,20 @@ func TestTransactionParticipantsBatch(t *testing.T) {
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
 
-	batch := q.NewTransactionParticipantsBatchInsertBuilder(0)
+	tt.Assert.NoError(q.Begin())
+	batch := q.NewTransactionParticipantsBatchInsertBuilder()
 
 	transactionID := int64(1)
 	otherTransactionID := int64(2)
 	accountID := int64(100)
 
 	for i := int64(0); i < 3; i++ {
-		tt.Assert.NoError(batch.Add(tt.Ctx, transactionID, accountID+i))
+		tt.Assert.NoError(batch.Add(transactionID, accountID+i))
 	}
 
-	tt.Assert.NoError(batch.Add(tt.Ctx, otherTransactionID, accountID))
-	tt.Assert.NoError(batch.Exec(tt.Ctx))
+	tt.Assert.NoError(batch.Add(otherTransactionID, accountID))
+	tt.Assert.NoError(batch.Exec(tt.Ctx, q))
+	tt.Assert.NoError(q.Commit())
 
 	participants := getTransactionParticipants(tt, q)
 	tt.Assert.Equal(
diff --git a/services/horizon/internal/db2/history/transaction_test.go b/services/horizon/internal/db2/history/transaction_test.go
index 33ef7b0d3c..5bc423006e 100644
--- a/services/horizon/internal/db2/history/transaction_test.go
+++ b/services/horizon/internal/db2/history/transaction_test.go
@@ -79,13 +79,13 @@ func TestTransactionByLiquidityPool(t *testing.T) {
 	liquidityPoolID := "a2f38836a839de008cf1d782c81f45e1253cc5d3dad9110b872965484fec0a49"
 	toInternalID, err := q.CreateHistoryLiquidityPools(tt.Ctx, []string{liquidityPoolID}, 2)
 	tt.Assert.NoError(err)
-	lpTransactionBuilder := q.NewTransactionLiquidityPoolBatchInsertBuilder(2)
+	lpTransactionBuilder := q.NewTransactionLiquidityPoolBatchInsertBuilder()
 	tt.Assert.NoError(err)
 	internalID, ok := toInternalID[liquidityPoolID]
 	tt.Assert.True(ok)
-	err = lpTransactionBuilder.Add(tt.Ctx, txID, internalID)
+	err = lpTransactionBuilder.Add(txID, internalID)
 	tt.Assert.NoError(err)
-	err = lpTransactionBuilder.Exec(tt.Ctx)
+	err = lpTransactionBuilder.Exec(tt.Ctx, q)
 	tt.Assert.NoError(err)
 
 	tt.Assert.NoError(q.Commit())
diff --git a/services/horizon/internal/ingest/main_test.go b/services/horizon/internal/ingest/main_test.go
index 9dd902287c..f807448590 100644
--- a/services/horizon/internal/ingest/main_test.go
+++ b/services/horizon/internal/ingest/main_test.go
@@ -413,13 +413,13 @@ func (m *mockDBQ) DeleteRangeAll(ctx context.Context, start, end int64) error {
 
 // Methods from interfaces duplicating methods:
 
-func (m *mockDBQ) NewTransactionParticipantsBatchInsertBuilder(maxBatchSize int) history.TransactionParticipantsBatchInsertBuilder {
-	args := m.Called(maxBatchSize)
+func (m *mockDBQ) NewTransactionParticipantsBatchInsertBuilder() history.TransactionParticipantsBatchInsertBuilder {
+	args := m.Called()
 	return args.Get(0).(history.TransactionParticipantsBatchInsertBuilder)
 }
 
-func (m *mockDBQ) NewOperationParticipantBatchInsertBuilder(maxBatchSize int) history.OperationParticipantBatchInsertBuilder {
-	args := m.Called(maxBatchSize)
+func (m *mockDBQ) NewOperationParticipantBatchInsertBuilder() history.OperationParticipantBatchInsertBuilder {
+	args := m.Called()
 	return args.Get(0).(history.TransactionParticipantsBatchInsertBuilder)
 }
 
diff --git a/services/horizon/internal/ingest/processor_runner.go b/services/horizon/internal/ingest/processor_runner.go
index b64e9db245..99c0816da8 100644
--- a/services/horizon/internal/ingest/processor_runner.go
+++ b/services/horizon/internal/ingest/processor_runner.go
@@ -148,10 +148,10 @@ func (s *ProcessorRunner) buildTransactionProcessor(
 		processors.NewLedgerProcessor(s.session, s.historyQ, ledger, CurrentVersion),
 		processors.NewOperationProcessor(s.session, s.historyQ, sequence),
 		tradeProcessor,
-		processors.NewParticipantsProcessor(s.historyQ, sequence),
+		processors.NewParticipantsProcessor(s.session, s.historyQ, sequence),
 		processors.NewTransactionProcessor(s.session, s.historyQ, sequence),
-		processors.NewClaimableBalancesTransactionProcessor(s.historyQ, sequence),
-		processors.NewLiquidityPoolsTransactionProcessor(s.historyQ, sequence),
+		processors.NewClaimableBalancesTransactionProcessor(s.session, s.historyQ, sequence),
+		processors.NewLiquidityPoolsTransactionProcessor(s.session, s.historyQ, sequence),
 	})
 }
 
diff --git a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor.go b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor.go
index 33c8bfd9c6..aecb25f88d 100644
--- a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor.go
+++ b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor.go
@@ -6,6 +6,7 @@ import (
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	set "github.com/stellar/go/support/collections/set"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
@@ -32,13 +33,15 @@ func (b *claimableBalance) addOperationID(id int64) {
 }
 
 type ClaimableBalancesTransactionProcessor struct {
+	session             db.SessionInterface
 	sequence            uint32
 	claimableBalanceSet map[string]claimableBalance
 	qClaimableBalances  history.QHistoryClaimableBalances
 }
 
-func NewClaimableBalancesTransactionProcessor(Q history.QHistoryClaimableBalances, sequence uint32) *ClaimableBalancesTransactionProcessor {
+func NewClaimableBalancesTransactionProcessor(session db.SessionInterface, Q history.QHistoryClaimableBalances, sequence uint32) *ClaimableBalancesTransactionProcessor {
 	return &ClaimableBalancesTransactionProcessor{
+		session:             session,
 		qClaimableBalances:  Q,
 		sequence:            sequence,
 		claimableBalanceSet: map[string]claimableBalance{},
@@ -223,34 +226,34 @@ func (p *ClaimableBalancesTransactionProcessor) loadClaimableBalanceIDs(ctx cont
 }
 
 func (p ClaimableBalancesTransactionProcessor) insertDBTransactionClaimableBalances(ctx context.Context, claimableBalanceSet map[string]claimableBalance) error {
-	batch := p.qClaimableBalances.NewTransactionClaimableBalanceBatchInsertBuilder(maxBatchSize)
+	batch := p.qClaimableBalances.NewTransactionClaimableBalanceBatchInsertBuilder()
 
 	for _, entry := range claimableBalanceSet {
 		for transactionID := range entry.transactionSet {
-			if err := batch.Add(ctx, transactionID, entry.internalID); err != nil {
+			if err := batch.Add(transactionID, entry.internalID); err != nil {
 				return errors.Wrap(err, "could not insert transaction claimable balance in db")
 			}
 		}
 	}
 
-	if err := batch.Exec(ctx); err != nil {
+	if err := batch.Exec(ctx, p.session); err != nil {
 		return errors.Wrap(err, "could not flush transaction claimable balances to db")
 	}
 	return nil
 }
 
 func (p ClaimableBalancesTransactionProcessor) insertDBOperationsClaimableBalances(ctx context.Context, claimableBalanceSet map[string]claimableBalance) error {
-	batch := p.qClaimableBalances.NewOperationClaimableBalanceBatchInsertBuilder(maxBatchSize)
+	batch := p.qClaimableBalances.NewOperationClaimableBalanceBatchInsertBuilder()
 
 	for _, entry := range claimableBalanceSet {
 		for operationID := range entry.operationSet {
-			if err := batch.Add(ctx, operationID, entry.internalID); err != nil {
+			if err := batch.Add(operationID, entry.internalID); err != nil {
 				return errors.Wrap(err, "could not insert operation claimable balance in db")
 			}
 		}
 	}
 
-	if err := batch.Exec(ctx); err != nil {
+	if err := batch.Exec(ctx, p.session); err != nil {
 		return errors.Wrap(err, "could not flush operation claimable balances to db")
 	}
 	return nil
diff --git a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go
index abdaf5b070..9c771c7e8a 100644
--- a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go
+++ b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go
@@ -10,6 +10,7 @@ import (
 	"github.com/stretchr/testify/suite"
 
 	"github.com/stellar/go/services/horizon/internal/db2/history"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
 )
@@ -18,6 +19,7 @@ type ClaimableBalancesTransactionProcessorTestSuiteLedger struct {
 	suite.Suite
 	ctx                               context.Context
 	processor                         *ClaimableBalancesTransactionProcessor
+	mockSession                       *db.MockSession
 	mockQ                             *history.MockQHistoryClaimableBalances
 	mockTransactionBatchInsertBuilder *history.MockTransactionClaimableBalanceBatchInsertBuilder
 	mockOperationBatchInsertBuilder   *history.MockOperationClaimableBalanceBatchInsertBuilder
@@ -37,6 +39,7 @@ func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) SetupTest() {
 	s.sequence = 20
 
 	s.processor = NewClaimableBalancesTransactionProcessor(
+		s.mockSession,
 		s.mockQ,
 		s.sequence,
 	)
@@ -49,11 +52,11 @@ func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) TearDownTest() {
 }
 
 func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) mockTransactionBatchAdd(transactionID, internalID int64, err error) {
-	s.mockTransactionBatchInsertBuilder.On("Add", s.ctx, transactionID, internalID).Return(err).Once()
+	s.mockTransactionBatchInsertBuilder.On("Add", transactionID, internalID).Return(err).Once()
 }
 
 func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) mockOperationBatchAdd(operationID, internalID int64, err error) {
-	s.mockOperationBatchInsertBuilder.On("Add", s.ctx, operationID, internalID).Return(err).Once()
+	s.mockOperationBatchInsertBuilder.On("Add", operationID, internalID).Return(err).Once()
 }
 
 func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) TestEmptyClaimableBalances() {
@@ -126,16 +129,16 @@ func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) testOperationInse
 	}, nil).Once()
 
 	// Prepare to process transactions successfully
-	s.mockQ.On("NewTransactionClaimableBalanceBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewTransactionClaimableBalanceBatchInsertBuilder").
 		Return(s.mockTransactionBatchInsertBuilder).Once()
 	s.mockTransactionBatchAdd(txnID, internalID, nil)
-	s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once()
+	s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	// Prepare to process operations successfully
-	s.mockQ.On("NewOperationClaimableBalanceBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewOperationClaimableBalanceBatchInsertBuilder").
 		Return(s.mockOperationBatchInsertBuilder).Once()
 	s.mockOperationBatchAdd(opID, internalID, nil)
-	s.mockOperationBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once()
+	s.mockOperationBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	// Process the transaction
 	err := s.processor.ProcessTransaction(s.ctx, txn)
diff --git a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor.go b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor.go
index 38010ddb51..a55cb95934 100644
--- a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor.go
+++ b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor.go
@@ -6,6 +6,7 @@ import (
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	set "github.com/stellar/go/support/collections/set"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
@@ -32,13 +33,15 @@ func (b *liquidityPool) addOperationID(id int64) {
 }
 
 type LiquidityPoolsTransactionProcessor struct {
+	session          db.SessionInterface
 	sequence         uint32
 	liquidityPoolSet map[string]liquidityPool
 	qLiquidityPools  history.QHistoryLiquidityPools
 }
 
-func NewLiquidityPoolsTransactionProcessor(Q history.QHistoryLiquidityPools, sequence uint32) *LiquidityPoolsTransactionProcessor {
+func NewLiquidityPoolsTransactionProcessor(session db.SessionInterface, Q history.QHistoryLiquidityPools, sequence uint32) *LiquidityPoolsTransactionProcessor {
 	return &LiquidityPoolsTransactionProcessor{
+		session:          session,
 		qLiquidityPools:  Q,
 		sequence:         sequence,
 		liquidityPoolSet: map[string]liquidityPool{},
@@ -218,34 +221,34 @@ func (p *LiquidityPoolsTransactionProcessor) loadLiquidityPoolIDs(ctx context.Co
 }
 
 func (p LiquidityPoolsTransactionProcessor) insertDBTransactionLiquidityPools(ctx context.Context, liquidityPoolSet map[string]liquidityPool) error {
-	batch := p.qLiquidityPools.NewTransactionLiquidityPoolBatchInsertBuilder(maxBatchSize)
+	batch := p.qLiquidityPools.NewTransactionLiquidityPoolBatchInsertBuilder()
 
 	for _, entry := range liquidityPoolSet {
 		for transactionID := range entry.transactionSet {
-			if err := batch.Add(ctx, transactionID, entry.internalID); err != nil {
+			if err := batch.Add(transactionID, entry.internalID); err != nil {
 				return errors.Wrap(err, "could not insert transaction liquidity pool in db")
 			}
 		}
 	}
 
-	if err := batch.Exec(ctx); err != nil {
+	if err := batch.Exec(ctx, p.session); err != nil {
 		return errors.Wrap(err, "could not flush transaction liquidity pools to db")
 	}
 	return nil
 }
 
 func (p LiquidityPoolsTransactionProcessor) insertDBOperationsLiquidityPools(ctx context.Context, liquidityPoolSet map[string]liquidityPool) error {
-	batch := p.qLiquidityPools.NewOperationLiquidityPoolBatchInsertBuilder(maxBatchSize)
+	batch := p.qLiquidityPools.NewOperationLiquidityPoolBatchInsertBuilder()
 
 	for _, entry := range liquidityPoolSet {
 		for operationID := range entry.operationSet {
-			if err := batch.Add(ctx, operationID, entry.internalID); err != nil {
+			if err := batch.Add(operationID, entry.internalID); err != nil {
 				return errors.Wrap(err, "could not insert operation liquidity pool in db")
 			}
 		}
 	}
 
-	if err := batch.Exec(ctx); err != nil {
+	if err := batch.Exec(ctx, p.session); err != nil {
 		return errors.Wrap(err, "could not flush operation liquidity pools to db")
 	}
 	return nil
diff --git a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go
index bd80ca09cb..f86845e931 100644
--- a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go
+++ b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go
@@ -10,6 +10,7 @@ import (
 	"github.com/stretchr/testify/suite"
 
 	"github.com/stellar/go/services/horizon/internal/db2/history"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
 )
@@ -18,6 +19,7 @@ type LiquidityPoolsTransactionProcessorTestSuiteLedger struct {
 	suite.Suite
 	ctx                               context.Context
 	processor                         *LiquidityPoolsTransactionProcessor
+	mockSession                       *db.MockSession
 	mockQ                             *history.MockQHistoryLiquidityPools
 	mockTransactionBatchInsertBuilder *history.MockTransactionLiquidityPoolBatchInsertBuilder
 	mockOperationBatchInsertBuilder   *history.MockOperationLiquidityPoolBatchInsertBuilder
@@ -37,6 +39,7 @@ func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) SetupTest() {
 	s.sequence = 20
 
 	s.processor = NewLiquidityPoolsTransactionProcessor(
+		s.mockSession,
 		s.mockQ,
 		s.sequence,
 	)
@@ -49,11 +52,11 @@ func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) TearDownTest() {
 }
 
 func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) mockTransactionBatchAdd(transactionID, internalID int64, err error) {
-	s.mockTransactionBatchInsertBuilder.On("Add", s.ctx, transactionID, internalID).Return(err).Once()
+	s.mockTransactionBatchInsertBuilder.On("Add", transactionID, internalID).Return(err).Once()
 }
 
 func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) mockOperationBatchAdd(operationID, internalID int64, err error) {
-	s.mockOperationBatchInsertBuilder.On("Add", s.ctx, operationID, internalID).Return(err).Once()
+	s.mockOperationBatchInsertBuilder.On("Add", operationID, internalID).Return(err).Once()
 }
 
 func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) TestEmptyLiquidityPools() {
@@ -123,16 +126,16 @@ func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) testOperationInserts
 	}, nil).Once()
 
 	// Prepare to process transactions successfully
-	s.mockQ.On("NewTransactionLiquidityPoolBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewTransactionLiquidityPoolBatchInsertBuilder").
 		Return(s.mockTransactionBatchInsertBuilder).Once()
 	s.mockTransactionBatchAdd(txnID, internalID, nil)
-	s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once()
+	s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	// Prepare to process operations successfully
-	s.mockQ.On("NewOperationLiquidityPoolBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewOperationLiquidityPoolBatchInsertBuilder").
 		Return(s.mockOperationBatchInsertBuilder).Once()
 	s.mockOperationBatchAdd(opID, internalID, nil)
-	s.mockOperationBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once()
+	s.mockOperationBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	// Process the transaction
 	err := s.processor.ProcessTransaction(s.ctx, txn)
diff --git a/services/horizon/internal/ingest/processors/participants_processor.go b/services/horizon/internal/ingest/processors/participants_processor.go
index d908f9ac69..debb812154 100644
--- a/services/horizon/internal/ingest/processors/participants_processor.go
+++ b/services/horizon/internal/ingest/processors/participants_processor.go
@@ -8,6 +8,7 @@ import (
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	set "github.com/stellar/go/support/collections/set"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
@@ -16,13 +17,15 @@ import (
 // ParticipantsProcessor is a processor which ingests various participants
 // from different sources (transactions, operations, etc)
 type ParticipantsProcessor struct {
+	session        db.SessionInterface
 	participantsQ  history.QParticipants
 	sequence       uint32
 	participantSet map[string]participant
 }
 
-func NewParticipantsProcessor(participantsQ history.QParticipants, sequence uint32) *ParticipantsProcessor {
+func NewParticipantsProcessor(session db.SessionInterface, participantsQ history.QParticipants, sequence uint32) *ParticipantsProcessor {
 	return &ParticipantsProcessor{
+		session:        session,
 		participantsQ:  participantsQ,
 		sequence:       sequence,
 		participantSet: map[string]participant{},
@@ -187,34 +190,34 @@ func (p *ParticipantsProcessor) addOperationsParticipants(
 }
 
 func (p *ParticipantsProcessor) insertDBTransactionParticipants(ctx context.Context, participantSet map[string]participant) error {
-	batch := p.participantsQ.NewTransactionParticipantsBatchInsertBuilder(maxBatchSize)
+	batch := p.participantsQ.NewTransactionParticipantsBatchInsertBuilder()
 
 	for _, entry := range participantSet {
 		for transactionID := range entry.transactionSet {
-			if err := batch.Add(ctx, transactionID, entry.accountID); err != nil {
+			if err := batch.Add(transactionID, entry.accountID); err != nil {
 				return errors.Wrap(err, "Could not insert transaction participant in db")
 			}
 		}
 	}
 
-	if err := batch.Exec(ctx); err != nil {
+	if err := batch.Exec(ctx, p.session); err != nil {
 		return errors.Wrap(err, "Could not flush transaction participants to db")
 	}
 	return nil
 }
 
 func (p *ParticipantsProcessor) insertDBOperationsParticipants(ctx context.Context, participantSet map[string]participant) error {
-	batch := p.participantsQ.NewOperationParticipantBatchInsertBuilder(maxBatchSize)
+	batch := p.participantsQ.NewOperationParticipantBatchInsertBuilder()
 
 	for _, entry := range participantSet {
 		for operationID := range entry.operationSet {
-			if err := batch.Add(ctx, operationID, entry.accountID); err != nil {
+			if err := batch.Add(operationID, entry.accountID); err != nil {
 				return errors.Wrap(err, "could not insert operation participant in db")
 			}
 		}
 	}
 
-	if err := batch.Exec(ctx); err != nil {
+	if err := batch.Exec(ctx, p.session); err != nil {
 		return errors.Wrap(err, "could not flush operation participants to db")
 	}
 	return nil
diff --git a/services/horizon/internal/ingest/processors/participants_processor_test.go b/services/horizon/internal/ingest/processors/participants_processor_test.go
index 4780c2709c..e6a8fa7b81 100644
--- a/services/horizon/internal/ingest/processors/participants_processor_test.go
+++ b/services/horizon/internal/ingest/processors/participants_processor_test.go
@@ -11,6 +11,7 @@ import (
 
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
@@ -20,6 +21,7 @@ type ParticipantsProcessorTestSuiteLedger struct {
 	suite.Suite
 	ctx                              context.Context
 	processor                        *ParticipantsProcessor
+	mockSession                      *db.MockSession
 	mockQ                            *history.MockQParticipants
 	mockBatchInsertBuilder           *history.MockTransactionParticipantsBatchInsertBuilder
 	mockOperationsBatchInsertBuilder *history.MockOperationParticipantBatchInsertBuilder
@@ -83,6 +85,7 @@ func (s *ParticipantsProcessorTestSuiteLedger) SetupTest() {
 	}
 
 	s.processor = NewParticipantsProcessor(
+		s.mockSession,
 		s.mockQ,
 		sequence,
 	)
@@ -102,33 +105,33 @@ func (s *ParticipantsProcessorTestSuiteLedger) TearDownTest() {
 
 func (s *ParticipantsProcessorTestSuiteLedger) mockSuccessfulTransactionBatchAdds() {
 	s.mockBatchInsertBuilder.On(
-		"Add", s.ctx, s.firstTxID, s.addressToID[s.addresses[0]],
+		"Add", s.firstTxID, s.addressToID[s.addresses[0]],
 	).Return(nil).Once()
 
 	s.mockBatchInsertBuilder.On(
-		"Add", s.ctx, s.secondTxID, s.addressToID[s.addresses[1]],
+		"Add", s.secondTxID, s.addressToID[s.addresses[1]],
 	).Return(nil).Once()
 	s.mockBatchInsertBuilder.On(
-		"Add", s.ctx, s.secondTxID, s.addressToID[s.addresses[2]],
+		"Add", s.secondTxID, s.addressToID[s.addresses[2]],
 	).Return(nil).Once()
 
 	s.mockBatchInsertBuilder.On(
-		"Add", s.ctx, s.thirdTxID, s.addressToID[s.addresses[0]],
+		"Add", s.thirdTxID, s.addressToID[s.addresses[0]],
 	).Return(nil).Once()
 }
 
 func (s *ParticipantsProcessorTestSuiteLedger) mockSuccessfulOperationBatchAdds() {
 	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.ctx, s.firstTxID+1, s.addressToID[s.addresses[0]],
+		"Add", s.firstTxID+1, s.addressToID[s.addresses[0]],
 	).Return(nil).Once()
 	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.ctx, s.secondTxID+1, s.addressToID[s.addresses[1]],
+		"Add", s.secondTxID+1, s.addressToID[s.addresses[1]],
 	).Return(nil).Once()
 	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.ctx, s.secondTxID+1, s.addressToID[s.addresses[2]],
+		"Add", s.secondTxID+1, s.addressToID[s.addresses[2]],
 	).Return(nil).Once()
 	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.ctx, s.thirdTxID+1, s.addressToID[s.addresses[0]],
+		"Add", s.thirdTxID+1, s.addressToID[s.addresses[0]],
 	).Return(nil).Once()
 }
 func (s *ParticipantsProcessorTestSuiteLedger) TestEmptyParticipants() {
@@ -179,20 +182,20 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestFeeBumptransaction() {
 				arg,
 			)
 		}).Return(addressToID, nil).Once()
-	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder").
 		Return(s.mockBatchInsertBuilder).Once()
-	s.mockQ.On("NewOperationParticipantBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewOperationParticipantBatchInsertBuilder").
 		Return(s.mockOperationsBatchInsertBuilder).Once()
 
 	s.mockBatchInsertBuilder.On(
-		"Add", s.ctx, feeBumpTxID, addressToID[addresses[0]],
+		"Add", feeBumpTxID, addressToID[addresses[0]],
 	).Return(nil).Once()
 	s.mockBatchInsertBuilder.On(
-		"Add", s.ctx, feeBumpTxID, addressToID[addresses[1]],
+		"Add", feeBumpTxID, addressToID[addresses[1]],
 	).Return(nil).Once()
 
-	s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once()
-	s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once()
+	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
+	s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	s.Assert().NoError(s.processor.ProcessTransaction(s.ctx, feeBumpTx))
 	s.Assert().NoError(s.processor.Commit(s.ctx))
@@ -207,16 +210,16 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestIngestParticipantsSucceeds()
 				arg,
 			)
 		}).Return(s.addressToID, nil).Once()
-	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder").
 		Return(s.mockBatchInsertBuilder).Once()
-	s.mockQ.On("NewOperationParticipantBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewOperationParticipantBatchInsertBuilder").
 		Return(s.mockOperationsBatchInsertBuilder).Once()
 
 	s.mockSuccessfulTransactionBatchAdds()
 	s.mockSuccessfulOperationBatchAdds()
 
-	s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once()
-	s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once()
+	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
+	s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	for _, tx := range s.txs {
 		err := s.processor.ProcessTransaction(s.ctx, tx)
@@ -246,22 +249,22 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestBatchAddFails() {
 				arg,
 			)
 		}).Return(s.addressToID, nil).Once()
-	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder").
 		Return(s.mockBatchInsertBuilder).Once()
 
 	s.mockBatchInsertBuilder.On(
-		"Add", s.ctx, s.firstTxID, s.addressToID[s.addresses[0]],
+		"Add", s.firstTxID, s.addressToID[s.addresses[0]],
 	).Return(errors.New("transient error")).Once()
 
 	s.mockBatchInsertBuilder.On(
-		"Add", s.ctx, s.secondTxID, s.addressToID[s.addresses[1]],
+		"Add", s.secondTxID, s.addressToID[s.addresses[1]],
 	).Return(nil).Maybe()
 	s.mockBatchInsertBuilder.On(
-		"Add", s.ctx, s.secondTxID, s.addressToID[s.addresses[2]],
+		"Add", s.secondTxID, s.addressToID[s.addresses[2]],
 	).Return(nil).Maybe()
 
 	s.mockBatchInsertBuilder.On(
-		"Add", s.ctx, s.thirdTxID, s.addressToID[s.addresses[0]],
+		"Add", s.thirdTxID, s.addressToID[s.addresses[0]],
 	).Return(nil).Maybe()
 	for _, tx := range s.txs {
 		err := s.processor.ProcessTransaction(s.ctx, tx)
@@ -280,27 +283,27 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestOperationParticipantsBatchAdd
 				arg,
 			)
 		}).Return(s.addressToID, nil).Once()
-	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder").
 		Return(s.mockBatchInsertBuilder).Once()
-	s.mockQ.On("NewOperationParticipantBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewOperationParticipantBatchInsertBuilder").
 		Return(s.mockOperationsBatchInsertBuilder).Once()
 
 	s.mockSuccessfulTransactionBatchAdds()
 
 	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.ctx, s.firstTxID+1, s.addressToID[s.addresses[0]],
+		"Add", s.firstTxID+1, s.addressToID[s.addresses[0]],
 	).Return(errors.New("transient error")).Once()
 	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.ctx, s.secondTxID+1, s.addressToID[s.addresses[1]],
+		"Add", s.secondTxID+1, s.addressToID[s.addresses[1]],
 	).Return(nil).Maybe()
 	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.ctx, s.secondTxID+1, s.addressToID[s.addresses[2]],
+		"Add", s.secondTxID+1, s.addressToID[s.addresses[2]],
 	).Return(nil).Maybe()
 	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.ctx, s.thirdTxID+1, s.addressToID[s.addresses[0]],
+		"Add", s.thirdTxID+1, s.addressToID[s.addresses[0]],
 	).Return(nil).Maybe()
 
-	s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once()
+	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	for _, tx := range s.txs {
 		err := s.processor.ProcessTransaction(s.ctx, tx)
@@ -319,12 +322,12 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestBatchAddExecFails() {
 				arg,
 			)
 		}).Return(s.addressToID, nil).Once()
-	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder").
 		Return(s.mockBatchInsertBuilder).Once()
 
 	s.mockSuccessfulTransactionBatchAdds()
 
-	s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(errors.New("transient error")).Once()
+	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(errors.New("transient error")).Once()
 
 	for _, tx := range s.txs {
 		err := s.processor.ProcessTransaction(s.ctx, tx)
@@ -343,16 +346,16 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestOpeartionBatchAddExecFails()
 				arg,
 			)
 		}).Return(s.addressToID, nil).Once()
-	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder").
 		Return(s.mockBatchInsertBuilder).Once()
-	s.mockQ.On("NewOperationParticipantBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewOperationParticipantBatchInsertBuilder").
 		Return(s.mockOperationsBatchInsertBuilder).Once()
 
 	s.mockSuccessfulTransactionBatchAdds()
 	s.mockSuccessfulOperationBatchAdds()
 
-	s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once()
-	s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx).Return(errors.New("transient error")).Once()
+	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
+	s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(errors.New("transient error")).Once()
 
 	for _, tx := range s.txs {
 		err := s.processor.ProcessTransaction(s.ctx, tx)

From c645724d42ba1ecea9911fd4d21ce6fbad9f3c94 Mon Sep 17 00:00:00 2001
From: tamirms <tamirms@gmail.com>
Date: Mon, 24 Jul 2023 23:01:04 +0100
Subject: [PATCH 06/17] services/horizon/internal/db2/history: Use
 FastBatchInsertBuilder for trades and effects (#4975)

---
 .../horizon/internal/actions_trade_test.go    | 16 ++++++++---
 .../horizon/internal/db2/history/effect.go    |  2 +-
 .../history/effect_batch_insert_builder.go    | 23 +++++++---------
 .../effect_batch_insert_builder_test.go       |  9 ++++---
 .../internal/db2/history/effect_test.go       | 17 +++++++-----
 .../internal/db2/history/fee_bump_scenario.go |  5 ++--
 services/horizon/internal/db2/history/main.go |  2 +-
 .../mock_effect_batch_insert_builder.go       | 10 ++++---
 .../internal/db2/history/mock_q_effects.go    |  4 +--
 .../internal/db2/history/mock_q_trades.go     | 13 ++++-----
 .../horizon/internal/db2/history/trade.go     |  2 +-
 .../db2/history/trade_batch_insert_builder.go | 27 +++++++++----------
 .../internal/db2/history/trade_scenario.go    |  8 +++---
 services/horizon/internal/ingest/main_test.go |  4 +--
 .../internal/ingest/processor_runner.go       |  4 +--
 .../ingest/processors/effects_processor.go    | 11 +++++---
 .../processors/effects_processor_test.go      | 17 +++++-------
 .../ingest/processors/trades_processor.go     | 11 +++++---
 .../processors/trades_processor_test.go       | 19 +++++++------
 .../integration/trade_aggregations_test.go    |  6 ++---
 20 files changed, 114 insertions(+), 96 deletions(-)

diff --git a/services/horizon/internal/actions_trade_test.go b/services/horizon/internal/actions_trade_test.go
index 1aa7b157fb..d46e22fc9d 100644
--- a/services/horizon/internal/actions_trade_test.go
+++ b/services/horizon/internal/actions_trade_test.go
@@ -820,8 +820,11 @@ func IngestTestTrade(
 		return err
 	}
 
-	batch := q.NewTradeBatchInsertBuilder(0)
-	batch.Add(ctx, history.InsertTrade{
+	if err = q.Begin(); err != nil {
+		return err
+	}
+	batch := q.NewTradeBatchInsertBuilder()
+	err = batch.Add(history.InsertTrade{
 		HistoryOperationID: opCounter,
 		Order:              0,
 		CounterAssetID:     assets[assetBought.String()].ID,
@@ -839,7 +842,10 @@ func IngestTestTrade(
 
 		Type: history.OrderbookTradeType,
 	})
-	err = batch.Exec(ctx)
+	if err != nil {
+		return err
+	}
+	err = batch.Exec(ctx, q)
 	if err != nil {
 		return err
 	}
@@ -849,6 +855,10 @@ func IngestTestTrade(
 		return err
 	}
 
+	if err := q.Commit(); err != nil {
+		return err
+	}
+
 	return nil
 }
 
diff --git a/services/horizon/internal/db2/history/effect.go b/services/horizon/internal/db2/history/effect.go
index c905479c6c..13a9c52519 100644
--- a/services/horizon/internal/db2/history/effect.go
+++ b/services/horizon/internal/db2/history/effect.go
@@ -246,7 +246,7 @@ func (q *EffectsQ) Select(ctx context.Context, dest interface{}) error {
 // QEffects defines history_effects related queries.
 type QEffects interface {
 	QCreateAccountsHistory
-	NewEffectBatchInsertBuilder(maxBatchSize int) EffectBatchInsertBuilder
+	NewEffectBatchInsertBuilder() EffectBatchInsertBuilder
 }
 
 var selectEffect = sq.Select("heff.*, hacc.address").
diff --git a/services/horizon/internal/db2/history/effect_batch_insert_builder.go b/services/horizon/internal/db2/history/effect_batch_insert_builder.go
index 8b2522cf9e..e3e5896e7f 100644
--- a/services/horizon/internal/db2/history/effect_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/effect_batch_insert_builder.go
@@ -11,7 +11,6 @@ import (
 // history_effects table
 type EffectBatchInsertBuilder interface {
 	Add(
-		ctx context.Context,
 		accountID int64,
 		muxedAccount null.String,
 		operationID int64,
@@ -19,27 +18,25 @@ type EffectBatchInsertBuilder interface {
 		effectType EffectType,
 		details []byte,
 	) error
-	Exec(ctx context.Context) error
+	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
 // effectBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder
 type effectBatchInsertBuilder struct {
-	builder db.BatchInsertBuilder
+	table   string
+	builder db.FastBatchInsertBuilder
 }
 
 // NewEffectBatchInsertBuilder constructs a new EffectBatchInsertBuilder instance
-func (q *Q) NewEffectBatchInsertBuilder(maxBatchSize int) EffectBatchInsertBuilder {
+func (q *Q) NewEffectBatchInsertBuilder() EffectBatchInsertBuilder {
 	return &effectBatchInsertBuilder{
-		builder: db.BatchInsertBuilder{
-			Table:        q.GetTable("history_effects"),
-			MaxBatchSize: maxBatchSize,
-		},
+		table:   "history_effects",
+		builder: db.FastBatchInsertBuilder{},
 	}
 }
 
 // Add adds a effect to the batch
 func (i *effectBatchInsertBuilder) Add(
-	ctx context.Context,
 	accountID int64,
 	muxedAccount null.String,
 	operationID int64,
@@ -47,16 +44,16 @@ func (i *effectBatchInsertBuilder) Add(
 	effectType EffectType,
 	details []byte,
 ) error {
-	return i.builder.Row(ctx, map[string]interface{}{
+	return i.builder.Row(map[string]interface{}{
 		"history_account_id":   accountID,
 		"address_muxed":        muxedAccount,
 		"history_operation_id": operationID,
-		"\"order\"":            order,
+		"order":                order,
 		"type":                 effectType,
 		"details":              details,
 	})
 }
 
-func (i *effectBatchInsertBuilder) Exec(ctx context.Context) error {
-	return i.builder.Exec(ctx)
+func (i *effectBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	return i.builder.Exec(ctx, session, i.table)
 }
diff --git a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go
index bd57eb4414..78988db2b4 100644
--- a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go
+++ b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go
@@ -14,20 +14,21 @@ func TestAddEffect(t *testing.T) {
 	defer tt.Finish()
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
+	tt.Assert.NoError(q.Begin())
 
 	address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY"
 	muxedAddres := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26"
 	accounIDs, err := q.CreateAccounts(tt.Ctx, []string{address}, 1)
 	tt.Assert.NoError(err)
 
-	builder := q.NewEffectBatchInsertBuilder(2)
+	builder := q.NewEffectBatchInsertBuilder()
 	sequence := int32(56)
 	details, err := json.Marshal(map[string]string{
 		"amount":     "1000.0000000",
 		"asset_type": "native",
 	})
 
-	err = builder.Add(tt.Ctx,
+	err = builder.Add(
 		accounIDs[address],
 		null.StringFrom(muxedAddres),
 		toid.New(sequence, 1, 1).ToInt64(),
@@ -37,8 +38,8 @@ func TestAddEffect(t *testing.T) {
 	)
 	tt.Assert.NoError(err)
 
-	err = builder.Exec(tt.Ctx)
-	tt.Assert.NoError(err)
+	tt.Assert.NoError(builder.Exec(tt.Ctx, q))
+	tt.Assert.NoError(q.Commit())
 
 	effects := []Effect{}
 	tt.Assert.NoError(q.Effects().Select(tt.Ctx, &effects))
diff --git a/services/horizon/internal/db2/history/effect_test.go b/services/horizon/internal/db2/history/effect_test.go
index 21ea63e308..06cfc2adcb 100644
--- a/services/horizon/internal/db2/history/effect_test.go
+++ b/services/horizon/internal/db2/history/effect_test.go
@@ -17,6 +17,7 @@ func TestEffectsForLiquidityPool(t *testing.T) {
 	defer tt.Finish()
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
+	tt.Assert.NoError(q.Begin())
 
 	// Insert Effect
 	address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY"
@@ -24,14 +25,14 @@ func TestEffectsForLiquidityPool(t *testing.T) {
 	accountIDs, err := q.CreateAccounts(tt.Ctx, []string{address}, 1)
 	tt.Assert.NoError(err)
 
-	builder := q.NewEffectBatchInsertBuilder(2)
+	builder := q.NewEffectBatchInsertBuilder()
 	sequence := int32(56)
 	details, err := json.Marshal(map[string]string{
 		"amount":     "1000.0000000",
 		"asset_type": "native",
 	})
 	opID := toid.New(sequence, 1, 1).ToInt64()
-	err = builder.Add(tt.Ctx,
+	err = builder.Add(
 		accountIDs[address],
 		null.StringFrom(muxedAddres),
 		opID,
@@ -41,7 +42,7 @@ func TestEffectsForLiquidityPool(t *testing.T) {
 	)
 	tt.Assert.NoError(err)
 
-	err = builder.Exec(tt.Ctx)
+	err = builder.Exec(tt.Ctx, q)
 	tt.Assert.NoError(err)
 
 	// Insert Liquidity Pool history
@@ -49,7 +50,6 @@ func TestEffectsForLiquidityPool(t *testing.T) {
 	toInternalID, err := q.CreateHistoryLiquidityPools(tt.Ctx, []string{liquidityPoolID}, 2)
 	tt.Assert.NoError(err)
 
-	tt.Assert.NoError(q.Begin())
 	operationBuilder := q.NewOperationLiquidityPoolBatchInsertBuilder()
 	tt.Assert.NoError(err)
 	internalID, ok := toInternalID[liquidityPoolID]
@@ -58,6 +58,7 @@ func TestEffectsForLiquidityPool(t *testing.T) {
 	tt.Assert.NoError(err)
 	err = operationBuilder.Exec(tt.Ctx, q)
 	tt.Assert.NoError(err)
+
 	tt.Assert.NoError(q.Commit())
 
 	var result []Effect
@@ -78,13 +79,14 @@ func TestEffectsForTrustlinesSponsorshipEmptyAssetType(t *testing.T) {
 	defer tt.Finish()
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
+	tt.Assert.NoError(q.Begin())
 
 	address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY"
 	muxedAddres := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26"
 	accountIDs, err := q.CreateAccounts(tt.Ctx, []string{address}, 1)
 	tt.Assert.NoError(err)
 
-	builder := q.NewEffectBatchInsertBuilder(1)
+	builder := q.NewEffectBatchInsertBuilder()
 	sequence := int32(56)
 	tests := []struct {
 		effectType        EffectType
@@ -147,7 +149,7 @@ func TestEffectsForTrustlinesSponsorshipEmptyAssetType(t *testing.T) {
 		bytes, err = json.Marshal(test.details)
 		tt.Require.NoError(err)
 
-		err = builder.Add(tt.Ctx,
+		err = builder.Add(
 			accountIDs[address],
 			null.StringFrom(muxedAddres),
 			opID,
@@ -158,8 +160,9 @@ func TestEffectsForTrustlinesSponsorshipEmptyAssetType(t *testing.T) {
 		tt.Require.NoError(err)
 	}
 
-	err = builder.Exec(tt.Ctx)
+	err = builder.Exec(tt.Ctx, q)
 	tt.Require.NoError(err)
+	tt.Assert.NoError(q.Commit())
 
 	var results []Effect
 	err = q.Effects().Select(tt.Ctx, &results)
diff --git a/services/horizon/internal/db2/history/fee_bump_scenario.go b/services/horizon/internal/db2/history/fee_bump_scenario.go
index 46cc423227..95fc7ac1b4 100644
--- a/services/horizon/internal/db2/history/fee_bump_scenario.go
+++ b/services/horizon/internal/db2/history/fee_bump_scenario.go
@@ -281,7 +281,7 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 	))
 	tt.Assert.NoError(opBuilder.Exec(ctx, q))
 
-	effectBuilder := q.NewEffectBatchInsertBuilder(2)
+	effectBuilder := q.NewEffectBatchInsertBuilder()
 	details, err = json.Marshal(map[string]interface{}{"new_seq": 98})
 	tt.Assert.NoError(err)
 
@@ -289,7 +289,6 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 	tt.Assert.NoError(err)
 
 	err = effectBuilder.Add(
-		ctx,
 		accounIDs[account.Address()],
 		null.String{},
 		toid.New(fixture.Ledger.Sequence, 1, 1).ToInt64(),
@@ -298,7 +297,7 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 		details,
 	)
 	tt.Assert.NoError(err)
-	tt.Assert.NoError(effectBuilder.Exec(ctx))
+	tt.Assert.NoError(effectBuilder.Exec(ctx, q))
 
 	tt.Assert.NoError(q.Commit())
 
diff --git a/services/horizon/internal/db2/history/main.go b/services/horizon/internal/db2/history/main.go
index bbf8bad912..060800ce80 100644
--- a/services/horizon/internal/db2/history/main.go
+++ b/services/horizon/internal/db2/history/main.go
@@ -259,7 +259,7 @@ type IngestionQ interface {
 	NewOperationParticipantBatchInsertBuilder() OperationParticipantBatchInsertBuilder
 	QSigners
 	//QTrades
-	NewTradeBatchInsertBuilder(maxBatchSize int) TradeBatchInsertBuilder
+	NewTradeBatchInsertBuilder() TradeBatchInsertBuilder
 	RebuildTradeAggregationTimes(ctx context.Context, from, to strtime.Millis, roundingSlippageFilter int) error
 	RebuildTradeAggregationBuckets(ctx context.Context, fromLedger, toLedger uint32, roundingSlippageFilter int) error
 	ReapLookupTables(ctx context.Context, offsets map[string]int64) (map[string]int64, map[string]int64, error)
diff --git a/services/horizon/internal/db2/history/mock_effect_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_effect_batch_insert_builder.go
index 48ee96e306..f97e4f5a0d 100644
--- a/services/horizon/internal/db2/history/mock_effect_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/mock_effect_batch_insert_builder.go
@@ -3,6 +3,8 @@ package history
 import (
 	"context"
 
+	"github.com/stellar/go/support/db"
+
 	"github.com/guregu/null"
 	"github.com/stretchr/testify/mock"
 )
@@ -13,7 +15,7 @@ type MockEffectBatchInsertBuilder struct {
 }
 
 // Add mock
-func (m *MockEffectBatchInsertBuilder) Add(ctx context.Context,
+func (m *MockEffectBatchInsertBuilder) Add(
 	accountID int64,
 	muxedAccount null.String,
 	operationID int64,
@@ -21,7 +23,7 @@ func (m *MockEffectBatchInsertBuilder) Add(ctx context.Context,
 	effectType EffectType,
 	details []byte,
 ) error {
-	a := m.Called(ctx,
+	a := m.Called(
 		accountID,
 		muxedAccount,
 		operationID,
@@ -33,7 +35,7 @@ func (m *MockEffectBatchInsertBuilder) Add(ctx context.Context,
 }
 
 // Exec mock
-func (m *MockEffectBatchInsertBuilder) Exec(ctx context.Context) error {
-	a := m.Called(ctx)
+func (m *MockEffectBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	a := m.Called(ctx, session)
 	return a.Error(0)
 }
diff --git a/services/horizon/internal/db2/history/mock_q_effects.go b/services/horizon/internal/db2/history/mock_q_effects.go
index d8bdd97765..615e4699fa 100644
--- a/services/horizon/internal/db2/history/mock_q_effects.go
+++ b/services/horizon/internal/db2/history/mock_q_effects.go
@@ -10,8 +10,8 @@ type MockQEffects struct {
 	mock.Mock
 }
 
-func (m *MockQEffects) NewEffectBatchInsertBuilder(maxBatchSize int) EffectBatchInsertBuilder {
-	a := m.Called(maxBatchSize)
+func (m *MockQEffects) NewEffectBatchInsertBuilder() EffectBatchInsertBuilder {
+	a := m.Called()
 	return a.Get(0).(EffectBatchInsertBuilder)
 }
 
diff --git a/services/horizon/internal/db2/history/mock_q_trades.go b/services/horizon/internal/db2/history/mock_q_trades.go
index d05e0e6a3d..2080f14a8d 100644
--- a/services/horizon/internal/db2/history/mock_q_trades.go
+++ b/services/horizon/internal/db2/history/mock_q_trades.go
@@ -3,6 +3,7 @@ package history
 import (
 	"context"
 
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/xdr"
 
 	"github.com/stretchr/testify/mock"
@@ -27,8 +28,8 @@ func (m *MockQTrades) CreateHistoryLiquidityPools(ctx context.Context, poolIDs [
 	return a.Get(0).(map[string]int64), a.Error(1)
 }
 
-func (m *MockQTrades) NewTradeBatchInsertBuilder(maxBatchSize int) TradeBatchInsertBuilder {
-	a := m.Called(maxBatchSize)
+func (m *MockQTrades) NewTradeBatchInsertBuilder() TradeBatchInsertBuilder {
+	a := m.Called()
 	return a.Get(0).(TradeBatchInsertBuilder)
 }
 
@@ -41,12 +42,12 @@ type MockTradeBatchInsertBuilder struct {
 	mock.Mock
 }
 
-func (m *MockTradeBatchInsertBuilder) Add(ctx context.Context, entries ...InsertTrade) error {
-	a := m.Called(ctx, entries)
+func (m *MockTradeBatchInsertBuilder) Add(entries ...InsertTrade) error {
+	a := m.Called(entries)
 	return a.Error(0)
 }
 
-func (m *MockTradeBatchInsertBuilder) Exec(ctx context.Context) error {
-	a := m.Called(ctx)
+func (m *MockTradeBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	a := m.Called(ctx, session)
 	return a.Error(0)
 }
diff --git a/services/horizon/internal/db2/history/trade.go b/services/horizon/internal/db2/history/trade.go
index 65b6a1ce98..6d8a7fca56 100644
--- a/services/horizon/internal/db2/history/trade.go
+++ b/services/horizon/internal/db2/history/trade.go
@@ -348,7 +348,7 @@ func getCanonicalAssetOrder(
 
 type QTrades interface {
 	QCreateAccountsHistory
-	NewTradeBatchInsertBuilder(maxBatchSize int) TradeBatchInsertBuilder
+	NewTradeBatchInsertBuilder() TradeBatchInsertBuilder
 	RebuildTradeAggregationBuckets(ctx context.Context, fromledger, toLedger uint32, roundingSlippageFilter int) error
 	CreateAssets(ctx context.Context, assets []xdr.Asset, maxBatchSize int) (map[string]Asset, error)
 	CreateHistoryLiquidityPools(ctx context.Context, poolIDs []string, batchSize int) (map[string]int64, error)
diff --git a/services/horizon/internal/db2/history/trade_batch_insert_builder.go b/services/horizon/internal/db2/history/trade_batch_insert_builder.go
index 1f2d614424..8420fabd36 100644
--- a/services/horizon/internal/db2/history/trade_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/trade_batch_insert_builder.go
@@ -24,7 +24,7 @@ const (
 // rows into the history_trades table
 type InsertTrade struct {
 	HistoryOperationID int64     `db:"history_operation_id"`
-	Order              int32     `db:"\"order\""`
+	Order              int32     `db:"order"`
 	LedgerCloseTime    time.Time `db:"ledger_closed_at"`
 
 	CounterAssetID         int64    `db:"counter_asset_id"`
@@ -55,36 +55,33 @@ type InsertTrade struct {
 // TradeBatchInsertBuilder is used to insert trades into the
 // history_trades table
 type TradeBatchInsertBuilder interface {
-	Add(ctx context.Context, entries ...InsertTrade) error
-	Exec(ctx context.Context) error
+	Add(entries ...InsertTrade) error
+	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
 // tradeBatchInsertBuilder is a simple wrapper around db.BatchInsertBuilder
 type tradeBatchInsertBuilder struct {
-	builder db.BatchInsertBuilder
-	q       *Q
+	builder db.FastBatchInsertBuilder
+	table   string
 }
 
 // NewTradeBatchInsertBuilder constructs a new TradeBatchInsertBuilder instance
-func (q *Q) NewTradeBatchInsertBuilder(maxBatchSize int) TradeBatchInsertBuilder {
+func (q *Q) NewTradeBatchInsertBuilder() TradeBatchInsertBuilder {
 	return &tradeBatchInsertBuilder{
-		builder: db.BatchInsertBuilder{
-			Table:        q.GetTable("history_trades"),
-			MaxBatchSize: maxBatchSize,
-		},
-		q: q,
+		table:   "history_trades",
+		builder: db.FastBatchInsertBuilder{},
 	}
 }
 
 // Exec flushes all outstanding trades to the database
-func (i *tradeBatchInsertBuilder) Exec(ctx context.Context) error {
-	return i.builder.Exec(ctx)
+func (i *tradeBatchInsertBuilder) Exec(ctx context.Context, session db.SessionInterface) error {
+	return i.builder.Exec(ctx, session, i.table)
 }
 
 // Add adds a new trade to the batch
-func (i *tradeBatchInsertBuilder) Add(ctx context.Context, entries ...InsertTrade) error {
+func (i *tradeBatchInsertBuilder) Add(entries ...InsertTrade) error {
 	for _, entry := range entries {
-		err := i.builder.RowStruct(ctx, entry)
+		err := i.builder.RowStruct(entry)
 		if err != nil {
 			return errors.Wrap(err, "failed to add trade")
 		}
diff --git a/services/horizon/internal/db2/history/trade_scenario.go b/services/horizon/internal/db2/history/trade_scenario.go
index 22e1830277..d993b3912c 100644
--- a/services/horizon/internal/db2/history/trade_scenario.go
+++ b/services/horizon/internal/db2/history/trade_scenario.go
@@ -199,7 +199,7 @@ func FilterTradesByType(trades []Trade, tradeType string) []Trade {
 
 // TradeScenario inserts trade rows into the Horizon DB
 func TradeScenario(tt *test.T, q *Q) TradeFixtures {
-	builder := q.NewTradeBatchInsertBuilder(0)
+	builder := q.NewTradeBatchInsertBuilder()
 
 	addresses := []string{
 		"GB2QIYT2IAUFMRXKLSLLPRECC6OCOGJMADSPTRK7TGNT2SFR2YGWDARD",
@@ -229,10 +229,12 @@ func TradeScenario(tt *test.T, q *Q) TradeFixtures {
 
 	inserts := createInsertTrades(accountIDs, assetIDs, poolIDs, 3)
 
+	tt.Assert.NoError(q.Begin())
 	tt.Assert.NoError(
-		builder.Add(tt.Ctx, inserts...),
+		builder.Add(inserts...),
 	)
-	tt.Assert.NoError(builder.Exec(tt.Ctx))
+	tt.Assert.NoError(builder.Exec(tt.Ctx, q))
+	tt.Assert.NoError(q.Commit())
 
 	idToAccount := buildIDtoAccountMapping(addresses, accountIDs)
 	idToAsset := buildIDtoAssetMapping(assets, assetIDs)
diff --git a/services/horizon/internal/ingest/main_test.go b/services/horizon/internal/ingest/main_test.go
index f807448590..6f0138ffe3 100644
--- a/services/horizon/internal/ingest/main_test.go
+++ b/services/horizon/internal/ingest/main_test.go
@@ -423,8 +423,8 @@ func (m *mockDBQ) NewOperationParticipantBatchInsertBuilder() history.OperationP
 	return args.Get(0).(history.TransactionParticipantsBatchInsertBuilder)
 }
 
-func (m *mockDBQ) NewTradeBatchInsertBuilder(maxBatchSize int) history.TradeBatchInsertBuilder {
-	args := m.Called(maxBatchSize)
+func (m *mockDBQ) NewTradeBatchInsertBuilder() history.TradeBatchInsertBuilder {
+	args := m.Called()
 	return args.Get(0).(history.TradeBatchInsertBuilder)
 }
 
diff --git a/services/horizon/internal/ingest/processor_runner.go b/services/horizon/internal/ingest/processor_runner.go
index 99c0816da8..481a4e7d52 100644
--- a/services/horizon/internal/ingest/processor_runner.go
+++ b/services/horizon/internal/ingest/processor_runner.go
@@ -140,11 +140,11 @@ func (s *ProcessorRunner) buildTransactionProcessor(
 	statsLedgerTransactionProcessor := &statsLedgerTransactionProcessor{
 		StatsLedgerTransactionProcessor: ledgerTransactionStats,
 	}
-	*tradeProcessor = *processors.NewTradeProcessor(s.historyQ, ledger)
+	*tradeProcessor = *processors.NewTradeProcessor(s.session, s.historyQ, ledger)
 	sequence := uint32(ledger.Header.LedgerSeq)
 	return newGroupTransactionProcessors([]horizonTransactionProcessor{
 		statsLedgerTransactionProcessor,
-		processors.NewEffectProcessor(s.historyQ, sequence),
+		processors.NewEffectProcessor(s.session, s.historyQ, sequence),
 		processors.NewLedgerProcessor(s.session, s.historyQ, ledger, CurrentVersion),
 		processors.NewOperationProcessor(s.session, s.historyQ, sequence),
 		tradeProcessor,
diff --git a/services/horizon/internal/ingest/processors/effects_processor.go b/services/horizon/internal/ingest/processors/effects_processor.go
index 0bce813343..f7703016e3 100644
--- a/services/horizon/internal/ingest/processors/effects_processor.go
+++ b/services/horizon/internal/ingest/processors/effects_processor.go
@@ -15,6 +15,7 @@ import (
 	"github.com/stellar/go/keypair"
 	"github.com/stellar/go/protocols/horizon/base"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/xdr"
 )
@@ -22,12 +23,14 @@ import (
 // EffectProcessor process effects
 type EffectProcessor struct {
 	effects  []effect
+	session  db.SessionInterface
 	effectsQ history.QEffects
 	sequence uint32
 }
 
-func NewEffectProcessor(effectsQ history.QEffects, sequence uint32) *EffectProcessor {
+func NewEffectProcessor(session db.SessionInterface, effectsQ history.QEffects, sequence uint32) *EffectProcessor {
 	return &EffectProcessor{
+		session:  session,
 		effectsQ: effectsQ,
 		sequence: sequence,
 	}
@@ -78,7 +81,7 @@ func operationsEffects(transaction ingest.LedgerTransaction, sequence uint32) ([
 }
 
 func (p *EffectProcessor) insertDBOperationsEffects(ctx context.Context, effects []effect, accountSet map[string]int64) error {
-	batch := p.effectsQ.NewEffectBatchInsertBuilder(maxBatchSize)
+	batch := p.effectsQ.NewEffectBatchInsertBuilder()
 
 	for _, effect := range effects {
 		accountID, found := accountSet[effect.address]
@@ -94,7 +97,7 @@ func (p *EffectProcessor) insertDBOperationsEffects(ctx context.Context, effects
 			return errors.Wrapf(err, "Error marshaling details for operation effect %v", effect.operationID)
 		}
 
-		if err := batch.Add(ctx,
+		if err := batch.Add(
 			accountID,
 			effect.addressMuxed,
 			effect.operationID,
@@ -106,7 +109,7 @@ func (p *EffectProcessor) insertDBOperationsEffects(ctx context.Context, effects
 		}
 	}
 
-	if err := batch.Exec(ctx); err != nil {
+	if err := batch.Exec(ctx, p.session); err != nil {
 		return errors.Wrap(err, "could not flush operation effects to db")
 	}
 	return nil
diff --git a/services/horizon/internal/ingest/processors/effects_processor_test.go b/services/horizon/internal/ingest/processors/effects_processor_test.go
index cb0d0dd321..4293fb5b3b 100644
--- a/services/horizon/internal/ingest/processors/effects_processor_test.go
+++ b/services/horizon/internal/ingest/processors/effects_processor_test.go
@@ -16,6 +16,7 @@ import (
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	. "github.com/stellar/go/services/horizon/internal/test/transactions"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
@@ -25,6 +26,7 @@ type EffectsProcessorTestSuiteLedger struct {
 	suite.Suite
 	ctx                    context.Context
 	processor              *EffectProcessor
+	mockSession            *db.MockSession
 	mockQ                  *history.MockQEffects
 	mockBatchInsertBuilder *history.MockEffectBatchInsertBuilder
 
@@ -119,6 +121,7 @@ func (s *EffectsProcessorTestSuiteLedger) SetupTest() {
 	}
 
 	s.processor = NewEffectProcessor(
+		s.mockSession,
 		s.mockQ,
 		20,
 	)
@@ -137,7 +140,6 @@ func (s *EffectsProcessorTestSuiteLedger) TearDownTest() {
 func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulEffectBatchAdds() {
 	s.mockBatchInsertBuilder.On(
 		"Add",
-		s.ctx,
 		s.addressToID[s.addresses[2]],
 		null.String{},
 		toid.New(int32(s.sequence), 1, 1).ToInt64(),
@@ -147,7 +149,6 @@ func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulEffectBatchAdds() {
 	).Return(nil).Once()
 	s.mockBatchInsertBuilder.On(
 		"Add",
-		s.ctx,
 		s.addressToID[s.addresses[2]],
 		null.String{},
 		toid.New(int32(s.sequence), 2, 1).ToInt64(),
@@ -157,7 +158,6 @@ func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulEffectBatchAdds() {
 	).Return(nil).Once()
 	s.mockBatchInsertBuilder.On(
 		"Add",
-		s.ctx,
 		s.addressToID[s.addresses[1]],
 		null.String{},
 		toid.New(int32(s.sequence), 2, 1).ToInt64(),
@@ -167,7 +167,6 @@ func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulEffectBatchAdds() {
 	).Return(nil).Once()
 	s.mockBatchInsertBuilder.On(
 		"Add",
-		s.ctx,
 		s.addressToID[s.addresses[2]],
 		null.String{},
 		toid.New(int32(s.sequence), 2, 1).ToInt64(),
@@ -178,7 +177,6 @@ func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulEffectBatchAdds() {
 
 	s.mockBatchInsertBuilder.On(
 		"Add",
-		s.ctx,
 		s.addressToID[s.addresses[0]],
 		null.String{},
 		toid.New(int32(s.sequence), 3, 1).ToInt64(),
@@ -189,7 +187,6 @@ func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulEffectBatchAdds() {
 
 	s.mockBatchInsertBuilder.On(
 		"Add",
-		s.ctx,
 		s.addressToID[s.addresses[0]],
 		null.String{},
 		toid.New(int32(s.sequence), 3, 1).ToInt64(),
@@ -218,12 +215,12 @@ func (s *EffectsProcessorTestSuiteLedger) TestEmptyEffects() {
 
 func (s *EffectsProcessorTestSuiteLedger) TestIngestEffectsSucceeds() {
 	s.mockSuccessfulCreateAccounts()
-	s.mockQ.On("NewEffectBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewEffectBatchInsertBuilder").
 		Return(s.mockBatchInsertBuilder).Once()
 
 	s.mockSuccessfulEffectBatchAdds()
 
-	s.mockBatchInsertBuilder.On("Exec", s.ctx).Return(nil).Once()
+	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	for _, tx := range s.txs {
 		err := s.processor.ProcessTransaction(s.ctx, tx)
@@ -247,11 +244,11 @@ func (s *EffectsProcessorTestSuiteLedger) TestCreateAccountsFails() {
 
 func (s *EffectsProcessorTestSuiteLedger) TestBatchAddFails() {
 	s.mockSuccessfulCreateAccounts()
-	s.mockQ.On("NewEffectBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewEffectBatchInsertBuilder").
 		Return(s.mockBatchInsertBuilder).Once()
 
 	s.mockBatchInsertBuilder.On(
-		"Add", s.ctx,
+		"Add",
 		s.addressToID[s.addresses[2]],
 		null.String{},
 		toid.New(int32(s.sequence), 1, 1).ToInt64(),
diff --git a/services/horizon/internal/ingest/processors/trades_processor.go b/services/horizon/internal/ingest/processors/trades_processor.go
index 2a95a994c7..10aed5d9ea 100644
--- a/services/horizon/internal/ingest/processors/trades_processor.go
+++ b/services/horizon/internal/ingest/processors/trades_processor.go
@@ -11,6 +11,7 @@ import (
 	"github.com/stellar/go/exp/orderbook"
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
@@ -18,14 +19,16 @@ import (
 
 // TradeProcessor operations processor
 type TradeProcessor struct {
+	session db.SessionInterface
 	tradesQ history.QTrades
 	ledger  xdr.LedgerHeaderHistoryEntry
 	trades  []ingestTrade
 	stats   TradeStats
 }
 
-func NewTradeProcessor(tradesQ history.QTrades, ledger xdr.LedgerHeaderHistoryEntry) *TradeProcessor {
+func NewTradeProcessor(session db.SessionInterface, tradesQ history.QTrades, ledger xdr.LedgerHeaderHistoryEntry) *TradeProcessor {
 	return &TradeProcessor{
+		session: session,
 		tradesQ: tradesQ,
 		ledger:  ledger,
 	}
@@ -65,7 +68,7 @@ func (p *TradeProcessor) Commit(ctx context.Context) error {
 		return nil
 	}
 
-	batch := p.tradesQ.NewTradeBatchInsertBuilder(maxBatchSize)
+	batch := p.tradesQ.NewTradeBatchInsertBuilder()
 	var poolIDs, accounts []string
 	var assets []xdr.Asset
 	for _, trade := range p.trades {
@@ -133,12 +136,12 @@ func (p *TradeProcessor) Commit(ctx context.Context) error {
 			}
 		}
 
-		if err = batch.Add(ctx, row); err != nil {
+		if err = batch.Add(row); err != nil {
 			return errors.Wrap(err, "Error adding trade to batch")
 		}
 	}
 
-	if err = batch.Exec(ctx); err != nil {
+	if err = batch.Exec(ctx, p.session); err != nil {
 		return errors.Wrap(err, "Error flushing operation batch")
 	}
 	return nil
diff --git a/services/horizon/internal/ingest/processors/trades_processor_test.go b/services/horizon/internal/ingest/processors/trades_processor_test.go
index e11fdf370f..d0cf299d30 100644
--- a/services/horizon/internal/ingest/processors/trades_processor_test.go
+++ b/services/horizon/internal/ingest/processors/trades_processor_test.go
@@ -12,6 +12,7 @@ import (
 
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
 	"github.com/stretchr/testify/mock"
@@ -21,6 +22,7 @@ import (
 type TradeProcessorTestSuiteLedger struct {
 	suite.Suite
 	processor              *TradeProcessor
+	mockSession            *db.MockSession
 	mockQ                  *history.MockQTrades
 	mockBatchInsertBuilder *history.MockTradeBatchInsertBuilder
 
@@ -198,6 +200,7 @@ func (s *TradeProcessorTestSuiteLedger) SetupTest() {
 	}
 
 	s.processor = NewTradeProcessor(
+		s.mockSession,
 		s.mockQ,
 		xdr.LedgerHeaderHistoryEntry{
 			Header: xdr.LedgerHeader{
@@ -709,7 +712,7 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions(
 		tx,
 	}
 
-	s.mockQ.On("NewTradeBatchInsertBuilder", maxBatchSize).
+	s.mockQ.On("NewTradeBatchInsertBuilder").
 		Return(s.mockBatchInsertBuilder).Once()
 
 	return inserts
@@ -747,12 +750,12 @@ func (s *TradeProcessorTestSuiteLedger) TestIngestTradesSucceeds() {
 	s.mockCreateHistoryLiquidityPools(ctx)
 
 	for _, insert := range inserts {
-		s.mockBatchInsertBuilder.On("Add", ctx, []history.InsertTrade{
+		s.mockBatchInsertBuilder.On("Add", []history.InsertTrade{
 			insert,
 		}).Return(nil).Once()
 	}
 
-	s.mockBatchInsertBuilder.On("Exec", ctx).Return(nil).Once()
+	s.mockBatchInsertBuilder.On("Exec", ctx, s.mockSession).Return(nil).Once()
 
 	for _, tx := range s.txs {
 		err := s.processor.ProcessTransaction(ctx, tx)
@@ -892,7 +895,7 @@ func (s *TradeProcessorTestSuiteLedger) TestBatchAddError() {
 
 	s.mockCreateHistoryLiquidityPools(ctx)
 
-	s.mockBatchInsertBuilder.On("Add", ctx, mock.AnythingOfType("[]history.InsertTrade")).
+	s.mockBatchInsertBuilder.On("Add", mock.AnythingOfType("[]history.InsertTrade")).
 		Return(fmt.Errorf("batch add error")).Once()
 
 	for _, tx := range s.txs {
@@ -914,9 +917,9 @@ func (s *TradeProcessorTestSuiteLedger) TestBatchExecError() {
 
 	s.mockCreateHistoryLiquidityPools(ctx)
 
-	s.mockBatchInsertBuilder.On("Add", ctx, mock.AnythingOfType("[]history.InsertTrade")).
+	s.mockBatchInsertBuilder.On("Add", mock.AnythingOfType("[]history.InsertTrade")).
 		Return(nil).Times(len(insert))
-	s.mockBatchInsertBuilder.On("Exec", ctx).Return(fmt.Errorf("exec error")).Once()
+	s.mockBatchInsertBuilder.On("Exec", ctx, s.mockSession).Return(fmt.Errorf("exec error")).Once()
 	for _, tx := range s.txs {
 		err := s.processor.ProcessTransaction(ctx, tx)
 		s.Assert().NoError(err)
@@ -935,9 +938,9 @@ func (s *TradeProcessorTestSuiteLedger) TestIgnoreCheckIfSmallLedger() {
 	s.mockCreateAssets(ctx)
 
 	s.mockCreateHistoryLiquidityPools(ctx)
-	s.mockBatchInsertBuilder.On("Add", ctx, mock.AnythingOfType("[]history.InsertTrade")).
+	s.mockBatchInsertBuilder.On("Add", mock.AnythingOfType("[]history.InsertTrade")).
 		Return(nil).Times(len(insert))
-	s.mockBatchInsertBuilder.On("Exec", ctx).Return(nil).Once()
+	s.mockBatchInsertBuilder.On("Exec", ctx, s.mockSession).Return(nil).Once()
 
 	for _, tx := range s.txs {
 		err := s.processor.ProcessTransaction(ctx, tx)
diff --git a/services/horizon/internal/integration/trade_aggregations_test.go b/services/horizon/internal/integration/trade_aggregations_test.go
index ae9c675875..29ed42370c 100644
--- a/services/horizon/internal/integration/trade_aggregations_test.go
+++ b/services/horizon/internal/integration/trade_aggregations_test.go
@@ -277,9 +277,9 @@ func TestTradeAggregations(t *testing.T) {
 				assert.NoError(t, historyQ.Rollback())
 			}()
 
-			batch := historyQ.NewTradeBatchInsertBuilder(1000)
-			batch.Add(ctx, scenario.trades...)
-			assert.NoError(t, batch.Exec(ctx))
+			batch := historyQ.NewTradeBatchInsertBuilder()
+			assert.NoError(t, batch.Add(scenario.trades...))
+			assert.NoError(t, batch.Exec(ctx, historyQ))
 
 			// Rebuild the aggregates.
 			for _, trade := range scenario.trades {

From 88f19e44b0069bcd87cdffeebf4c363775a2f23a Mon Sep 17 00:00:00 2001
From: tamirms <tamirms@gmail.com>
Date: Fri, 11 Aug 2023 09:46:42 +0200
Subject: [PATCH 07/17] services/horizon/internal/ingest/processors: Refactor
 ledgers, transactions, and operations processors to support new ingestion
 data flow (#5004)

---
 .../ingest/processors/ledgers_processor.go    |  74 ++++++----
 .../processors/ledgers_processor_test.go      | 128 +++++++++++++-----
 .../ingest/processors/operations_processor.go |  22 ++-
 .../processors/operations_processor_test.go   |  66 +++++++--
 .../processors/transactions_processor.go      |  32 ++---
 .../processors/transactions_processor_test.go |  63 +++++----
 xdr/ledger_close_meta.go                      |  13 ++
 7 files changed, 266 insertions(+), 132 deletions(-)

diff --git a/services/horizon/internal/ingest/processors/ledgers_processor.go b/services/horizon/internal/ingest/processors/ledgers_processor.go
index aee2f12709..1f14cc5518 100644
--- a/services/horizon/internal/ingest/processors/ledgers_processor.go
+++ b/services/horizon/internal/ingest/processors/ledgers_processor.go
@@ -10,53 +10,75 @@ import (
 	"github.com/stellar/go/xdr"
 )
 
-type LedgersProcessor struct {
-	session        db.SessionInterface
-	ledgersQ       history.QLedgers
-	ledger         xdr.LedgerHeaderHistoryEntry
-	ingestVersion  int
+type ledgerInfo struct {
+	header         xdr.LedgerHeaderHistoryEntry
 	successTxCount int
 	failedTxCount  int
 	opCount        int
 	txSetOpCount   int
 }
 
-func NewLedgerProcessor(
-	session db.SessionInterface,
-	ledgerQ history.QLedgers,
-	ledger xdr.LedgerHeaderHistoryEntry,
-	ingestVersion int,
-) *LedgersProcessor {
+type LedgersProcessor struct {
+	batch         history.LedgerBatchInsertBuilder
+	ledgers       map[uint32]*ledgerInfo
+	ingestVersion int
+}
+
+func NewLedgerProcessor(batch history.LedgerBatchInsertBuilder, ingestVersion int) *LedgersProcessor {
 	return &LedgersProcessor{
-		session:       session,
-		ledger:        ledger,
-		ledgersQ:      ledgerQ,
+		batch:         batch,
+		ledgers:       map[uint32]*ledgerInfo{},
 		ingestVersion: ingestVersion,
 	}
 }
 
-func (p *LedgersProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) (err error) {
+func (p *LedgersProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error {
+	sequence := lcm.LedgerSequence()
+	entry, ok := p.ledgers[sequence]
+	if !ok {
+		entry = &ledgerInfo{header: lcm.LedgerHeaderHistoryEntry()}
+		p.ledgers[sequence] = entry
+	}
+
 	opCount := len(transaction.Envelope.Operations())
-	p.txSetOpCount += opCount
+	entry.txSetOpCount += opCount
 	if transaction.Result.Successful() {
-		p.successTxCount++
-		p.opCount += opCount
+		entry.successTxCount++
+		entry.opCount += opCount
 	} else {
-		p.failedTxCount++
+		entry.failedTxCount++
 	}
 
 	return nil
 }
 
-func (p *LedgersProcessor) Commit(ctx context.Context) error {
-	batch := p.ledgersQ.NewLedgerBatchInsertBuilder()
-	err := batch.Add(p.ledger, p.successTxCount, p.failedTxCount, p.opCount, p.txSetOpCount, p.ingestVersion)
-	if err != nil {
-		return errors.Wrap(err, "Could not insert ledger")
+func (p *LedgersProcessor) Flush(ctx context.Context, session db.SessionInterface) error {
+	if len(p.ledgers) == 0 {
+		return nil
+	}
+	var min, max uint32
+	for ledger, entry := range p.ledgers {
+		err := p.batch.Add(
+			entry.header,
+			entry.successTxCount,
+			entry.failedTxCount,
+			entry.opCount,
+			entry.txSetOpCount,
+			p.ingestVersion,
+		)
+		if err != nil {
+			return errors.Wrapf(err, "error adding ledger %d to batch", ledger)
+		}
+		if min == 0 || ledger < min {
+			min = ledger
+		}
+		if max == 0 || ledger > max {
+			max = ledger
+		}
 	}
 
-	if err = batch.Exec(ctx, p.session); err != nil {
-		return errors.Wrap(err, "Could not commit ledger")
+	if err := p.batch.Exec(ctx, session); err != nil {
+		return errors.Wrapf(err, "error committing ledgers %d - %d", min, max)
 	}
 
 	return nil
diff --git a/services/horizon/internal/ingest/processors/ledgers_processor_test.go b/services/horizon/internal/ingest/processors/ledgers_processor_test.go
index 9cbd2c8643..308bb6995d 100644
--- a/services/horizon/internal/ingest/processors/ledgers_processor_test.go
+++ b/services/horizon/internal/ingest/processors/ledgers_processor_test.go
@@ -17,16 +17,16 @@ import (
 
 type LedgersProcessorTestSuiteLedger struct {
 	suite.Suite
-	processor     *LedgersProcessor
-	mockSession   *db.MockSession
-	mockQ         *history.MockQLedgers
-	header        xdr.LedgerHeaderHistoryEntry
-	successCount  int
-	failedCount   int
-	opCount       int
-	ingestVersion int
-	txs           []ingest.LedgerTransaction
-	txSetOpCount  int
+	processor              *LedgersProcessor
+	mockSession            *db.MockSession
+	mockBatchInsertBuilder *history.MockLedgersBatchInsertBuilder
+	header                 xdr.LedgerHeaderHistoryEntry
+	successCount           int
+	failedCount            int
+	opCount                int
+	ingestVersion          int
+	txs                    []ingest.LedgerTransaction
+	txSetOpCount           int
 }
 
 func TestLedgersProcessorTestSuiteLedger(t *testing.T) {
@@ -80,7 +80,7 @@ func createTransaction(successful bool, numOps int) ingest.LedgerTransaction {
 }
 
 func (s *LedgersProcessorTestSuiteLedger) SetupTest() {
-	s.mockQ = &history.MockQLedgers{}
+	s.mockBatchInsertBuilder = &history.MockLedgersBatchInsertBuilder{}
 	s.ingestVersion = 100
 	s.header = xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
@@ -89,9 +89,7 @@ func (s *LedgersProcessorTestSuiteLedger) SetupTest() {
 	}
 
 	s.processor = NewLedgerProcessor(
-		s.mockSession,
-		s.mockQ,
-		s.header,
+		s.mockBatchInsertBuilder,
 		s.ingestVersion,
 	)
 
@@ -108,15 +106,40 @@ func (s *LedgersProcessorTestSuiteLedger) SetupTest() {
 }
 
 func (s *LedgersProcessorTestSuiteLedger) TearDownTest() {
-	s.mockQ.AssertExpectations(s.T())
+	s.mockBatchInsertBuilder.AssertExpectations(s.T())
 }
 
 func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerSucceeds() {
 	ctx := context.Background()
-	mockBatchInsertBuilder := &history.MockLedgersBatchInsertBuilder{}
-	s.mockQ.On("NewLedgerBatchInsertBuilder").Return(mockBatchInsertBuilder)
 
-	mockBatchInsertBuilder.On(
+	for _, tx := range s.txs {
+		err := s.processor.ProcessTransaction(xdr.LedgerCloseMeta{
+			V0: &xdr.LedgerCloseMetaV0{
+				LedgerHeader: s.header,
+			},
+		}, tx)
+		s.Assert().NoError(err)
+	}
+
+	nextHeader := xdr.LedgerHeaderHistoryEntry{
+		Header: xdr.LedgerHeader{
+			LedgerSeq: xdr.Uint32(21),
+		},
+	}
+	nextTransactions := []ingest.LedgerTransaction{
+		createTransaction(true, 1),
+		createTransaction(false, 2),
+	}
+	for _, tx := range nextTransactions {
+		err := s.processor.ProcessTransaction(xdr.LedgerCloseMeta{
+			V0: &xdr.LedgerCloseMetaV0{
+				LedgerHeader: nextHeader,
+			},
+		}, tx)
+		s.Assert().NoError(err)
+	}
+
+	s.mockBatchInsertBuilder.On(
 		"Add",
 		s.header,
 		s.successCount,
@@ -125,26 +148,29 @@ func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerSucceeds() {
 		s.txSetOpCount,
 		s.ingestVersion,
 	).Return(nil)
-	mockBatchInsertBuilder.On(
+
+	s.mockBatchInsertBuilder.On(
+		"Add",
+		nextHeader,
+		1,
+		1,
+		1,
+		3,
+		s.ingestVersion,
+	).Return(nil)
+
+	s.mockBatchInsertBuilder.On(
 		"Exec",
 		ctx,
 		s.mockSession,
 	).Return(nil)
 
-	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(ctx, tx)
-		s.Assert().NoError(err)
-	}
-
-	err := s.processor.Commit(ctx)
+	err := s.processor.Flush(ctx, s.mockSession)
 	s.Assert().NoError(err)
 }
 
 func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerReturnsError() {
-	mockBatchInsertBuilder := &history.MockLedgersBatchInsertBuilder{}
-	s.mockQ.On("NewLedgerBatchInsertBuilder").Return(mockBatchInsertBuilder)
-
-	mockBatchInsertBuilder.On(
+	s.mockBatchInsertBuilder.On(
 		"Add",
 		mock.Anything,
 		mock.Anything,
@@ -154,8 +180,46 @@ func (s *LedgersProcessorTestSuiteLedger) TestInsertLedgerReturnsError() {
 		mock.Anything,
 	).Return(errors.New("transient error"))
 
+	err := s.processor.ProcessTransaction(xdr.LedgerCloseMeta{
+		V0: &xdr.LedgerCloseMetaV0{
+			LedgerHeader: s.header,
+		},
+	}, s.txs[0])
+	s.Assert().NoError(err)
+
+	s.Assert().EqualError(s.processor.Flush(
+		context.Background(), s.mockSession),
+		"error adding ledger 20 to batch: transient error",
+	)
+}
+
+func (s *LedgersProcessorTestSuiteLedger) TestExecFails() {
 	ctx := context.Background()
-	err := s.processor.Commit(ctx)
-	s.Assert().Error(err)
-	s.Assert().EqualError(err, "Could not insert ledger: transient error")
+	s.mockBatchInsertBuilder.On(
+		"Add",
+		mock.Anything,
+		mock.Anything,
+		mock.Anything,
+		mock.Anything,
+		mock.Anything,
+		mock.Anything,
+	).Return(nil)
+
+	s.mockBatchInsertBuilder.On(
+		"Exec",
+		ctx,
+		s.mockSession,
+	).Return(errors.New("transient exec error"))
+
+	err := s.processor.ProcessTransaction(xdr.LedgerCloseMeta{
+		V0: &xdr.LedgerCloseMetaV0{
+			LedgerHeader: s.header,
+		},
+	}, s.txs[0])
+	s.Assert().NoError(err)
+
+	s.Assert().EqualError(s.processor.Flush(
+		context.Background(), s.mockSession),
+		"error committing ledgers 20 - 20: transient exec error",
+	)
 }
diff --git a/services/horizon/internal/ingest/processors/operations_processor.go b/services/horizon/internal/ingest/processors/operations_processor.go
index 1d6972f91f..d22ba2cd6d 100644
--- a/services/horizon/internal/ingest/processors/operations_processor.go
+++ b/services/horizon/internal/ingest/processors/operations_processor.go
@@ -7,6 +7,7 @@ import (
 	"fmt"
 
 	"github.com/guregu/null"
+
 	"github.com/stellar/go/amount"
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/protocols/horizon/base"
@@ -19,30 +20,23 @@ import (
 
 // OperationProcessor operations processor
 type OperationProcessor struct {
-	session     db.SessionInterface
-	operationsQ history.QOperations
-
-	sequence uint32
-	batch    history.OperationBatchInsertBuilder
+	batch history.OperationBatchInsertBuilder
 }
 
-func NewOperationProcessor(session db.SessionInterface, operationsQ history.QOperations, sequence uint32) *OperationProcessor {
+func NewOperationProcessor(batch history.OperationBatchInsertBuilder) *OperationProcessor {
 	return &OperationProcessor{
-		session:     session,
-		operationsQ: operationsQ,
-		sequence:    sequence,
-		batch:       operationsQ.NewOperationBatchInsertBuilder(),
+		batch: batch,
 	}
 }
 
 // ProcessTransaction process the given transaction
-func (p *OperationProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error {
+func (p *OperationProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error {
 	for i, op := range transaction.Envelope.Operations() {
 		operation := transactionOperationWrapper{
 			index:          uint32(i),
 			transaction:    transaction,
 			operation:      op,
-			ledgerSequence: p.sequence,
+			ledgerSequence: lcm.LedgerSequence(),
 		}
 		details, err := operation.Details()
 		if err != nil {
@@ -76,8 +70,8 @@ func (p *OperationProcessor) ProcessTransaction(ctx context.Context, transaction
 	return nil
 }
 
-func (p *OperationProcessor) Commit(ctx context.Context) error {
-	return p.batch.Exec(ctx, p.session)
+func (p *OperationProcessor) Flush(ctx context.Context, session db.SessionInterface) error {
+	return p.batch.Exec(ctx, session)
 }
 
 // transactionOperationWrapper represents the data for a single operation within a transaction
diff --git a/services/horizon/internal/ingest/processors/operations_processor_test.go b/services/horizon/internal/ingest/processors/operations_processor_test.go
index af5e585ddd..27692faa90 100644
--- a/services/horizon/internal/ingest/processors/operations_processor_test.go
+++ b/services/horizon/internal/ingest/processors/operations_processor_test.go
@@ -23,7 +23,6 @@ type OperationsProcessorTestSuiteLedger struct {
 	ctx                    context.Context
 	processor              *OperationProcessor
 	mockSession            *db.MockSession
-	mockQ                  *history.MockQOperations
 	mockBatchInsertBuilder *history.MockOperationsBatchInsertBuilder
 }
 
@@ -33,21 +32,14 @@ func TestOperationProcessorTestSuiteLedger(t *testing.T) {
 
 func (s *OperationsProcessorTestSuiteLedger) SetupTest() {
 	s.ctx = context.Background()
-	s.mockQ = &history.MockQOperations{}
 	s.mockBatchInsertBuilder = &history.MockOperationsBatchInsertBuilder{}
-	s.mockQ.
-		On("NewOperationBatchInsertBuilder").
-		Return(s.mockBatchInsertBuilder).Once()
 
 	s.processor = NewOperationProcessor(
-		s.mockSession,
-		s.mockQ,
-		56,
+		s.mockBatchInsertBuilder,
 	)
 }
 
 func (s *OperationsProcessorTestSuiteLedger) TearDownTest() {
-	s.mockQ.AssertExpectations(s.T())
 	s.mockBatchInsertBuilder.AssertExpectations(s.T())
 }
 
@@ -92,6 +84,17 @@ func (s *OperationsProcessorTestSuiteLedger) mockBatchInsertAdds(txs []ingest.Le
 }
 
 func (s *OperationsProcessorTestSuiteLedger) TestAddOperationSucceeds() {
+	sequence := uint32(56)
+	lcm := xdr.LedgerCloseMeta{
+		V0: &xdr.LedgerCloseMetaV0{
+			LedgerHeader: xdr.LedgerHeaderHistoryEntry{
+				Header: xdr.LedgerHeader{
+					LedgerSeq: xdr.Uint32(sequence),
+				},
+			},
+		},
+	}
+
 	unmuxed := xdr.MustAddress("GA5WBPYA5Y4WAEHXWR2UKO2UO4BUGHUQ74EUPKON2QHV4WRHOIRNKKH2")
 	muxed := xdr.MuxedAccount{
 		Type: xdr.CryptoKeyTypeKeyTypeMuxedEd25519,
@@ -122,18 +125,28 @@ func (s *OperationsProcessorTestSuiteLedger) TestAddOperationSucceeds() {
 
 	var err error
 
-	err = s.mockBatchInsertAdds(txs, uint32(56))
+	err = s.mockBatchInsertAdds(txs, sequence)
 	s.Assert().NoError(err)
 	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
-	s.Assert().NoError(s.processor.Commit(s.ctx))
 
 	for _, tx := range txs {
-		err = s.processor.ProcessTransaction(s.ctx, tx)
+		err = s.processor.ProcessTransaction(lcm, tx)
 		s.Assert().NoError(err)
 	}
+	s.Assert().NoError(s.processor.Flush(s.ctx, s.mockSession))
 }
 
 func (s *OperationsProcessorTestSuiteLedger) TestAddOperationFails() {
+	sequence := uint32(56)
+	lcm := xdr.LedgerCloseMeta{
+		V0: &xdr.LedgerCloseMetaV0{
+			LedgerHeader: xdr.LedgerHeaderHistoryEntry{
+				Header: xdr.LedgerHeader{
+					LedgerSeq: xdr.Uint32(sequence),
+				},
+			},
+		},
+	}
 	tx := createTransaction(true, 1)
 
 	s.mockBatchInsertBuilder.
@@ -148,14 +161,39 @@ func (s *OperationsProcessorTestSuiteLedger) TestAddOperationFails() {
 			mock.Anything,
 		).Return(errors.New("transient error")).Once()
 
-	err := s.processor.ProcessTransaction(s.ctx, tx)
+	err := s.processor.ProcessTransaction(lcm, tx)
 	s.Assert().Error(err)
 	s.Assert().EqualError(err, "Error batch inserting operation rows: transient error")
 }
 
 func (s *OperationsProcessorTestSuiteLedger) TestExecFails() {
+	sequence := uint32(56)
+	lcm := xdr.LedgerCloseMeta{
+		V0: &xdr.LedgerCloseMetaV0{
+			LedgerHeader: xdr.LedgerHeaderHistoryEntry{
+				Header: xdr.LedgerHeader{
+					LedgerSeq: xdr.Uint32(sequence),
+				},
+			},
+		},
+	}
+	tx := createTransaction(true, 1)
+
+	s.mockBatchInsertBuilder.
+		On(
+			"Add",
+			mock.Anything,
+			mock.Anything,
+			mock.Anything,
+			mock.Anything,
+			mock.Anything,
+			mock.Anything,
+			mock.Anything,
+		).Return(nil).Once()
+	s.Assert().NoError(s.processor.ProcessTransaction(lcm, tx))
+
 	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(errors.New("transient error")).Once()
-	err := s.processor.Commit(s.ctx)
+	err := s.processor.Flush(s.ctx, s.mockSession)
 	s.Assert().Error(err)
 	s.Assert().EqualError(err, "transient error")
 }
diff --git a/services/horizon/internal/ingest/processors/transactions_processor.go b/services/horizon/internal/ingest/processors/transactions_processor.go
index 0e6603f804..871c72624a 100644
--- a/services/horizon/internal/ingest/processors/transactions_processor.go
+++ b/services/horizon/internal/ingest/processors/transactions_processor.go
@@ -7,45 +7,33 @@ import (
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
+	"github.com/stellar/go/xdr"
 )
 
 type TransactionProcessor struct {
-	session       db.SessionInterface
-	transactionsQ history.QTransactions
-	sequence      uint32
-	batch         history.TransactionBatchInsertBuilder
+	batch history.TransactionBatchInsertBuilder
 }
 
-func NewTransactionFilteredTmpProcessor(session db.SessionInterface, transactionsQ history.QTransactions, sequence uint32) *TransactionProcessor {
+func NewTransactionFilteredTmpProcessor(batch history.TransactionBatchInsertBuilder) *TransactionProcessor {
 	return &TransactionProcessor{
-		session:       session,
-		transactionsQ: transactionsQ,
-		sequence:      sequence,
-		batch:         transactionsQ.NewTransactionFilteredTmpBatchInsertBuilder(),
+		batch: batch,
 	}
 }
 
-func NewTransactionProcessor(session db.SessionInterface, transactionsQ history.QTransactions, sequence uint32) *TransactionProcessor {
+func NewTransactionProcessor(batch history.TransactionBatchInsertBuilder) *TransactionProcessor {
 	return &TransactionProcessor{
-		session:       session,
-		transactionsQ: transactionsQ,
-		sequence:      sequence,
-		batch:         transactionsQ.NewTransactionBatchInsertBuilder(),
+		batch: batch,
 	}
 }
 
-func (p *TransactionProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error {
-	if err := p.batch.Add(transaction, p.sequence); err != nil {
+func (p *TransactionProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error {
+	if err := p.batch.Add(transaction, lcm.LedgerSequence()); err != nil {
 		return errors.Wrap(err, "Error batch inserting transaction rows")
 	}
 
 	return nil
 }
 
-func (p *TransactionProcessor) Commit(ctx context.Context) error {
-	if err := p.batch.Exec(ctx, p.session); err != nil {
-		return errors.Wrap(err, "Error flushing transaction batch")
-	}
-
-	return nil
+func (p *TransactionProcessor) Flush(ctx context.Context, session db.SessionInterface) error {
+	return p.batch.Exec(ctx, session)
 }
diff --git a/services/horizon/internal/ingest/processors/transactions_processor_test.go b/services/horizon/internal/ingest/processors/transactions_processor_test.go
index dcaf307729..987e8ce6f9 100644
--- a/services/horizon/internal/ingest/processors/transactions_processor_test.go
+++ b/services/horizon/internal/ingest/processors/transactions_processor_test.go
@@ -9,6 +9,7 @@ import (
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
+	"github.com/stellar/go/xdr"
 
 	"github.com/stretchr/testify/suite"
 )
@@ -18,7 +19,6 @@ type TransactionsProcessorTestSuiteLedger struct {
 	ctx                    context.Context
 	processor              *TransactionProcessor
 	mockSession            *db.MockSession
-	mockQ                  *history.MockQTransactions
 	mockBatchInsertBuilder *history.MockTransactionsBatchInsertBuilder
 }
 
@@ -28,66 +28,81 @@ func TestTransactionsProcessorTestSuiteLedger(t *testing.T) {
 
 func (s *TransactionsProcessorTestSuiteLedger) SetupTest() {
 	s.ctx = context.Background()
-	s.mockQ = &history.MockQTransactions{}
 	s.mockBatchInsertBuilder = &history.MockTransactionsBatchInsertBuilder{}
-
-	s.mockQ.
-		On("NewTransactionBatchInsertBuilder").
-		Return(s.mockBatchInsertBuilder).Once()
-
-	s.processor = NewTransactionProcessor(s.mockSession, s.mockQ, 20)
+	s.processor = NewTransactionProcessor(s.mockBatchInsertBuilder)
 }
 
 func (s *TransactionsProcessorTestSuiteLedger) TearDownTest() {
-	s.mockQ.AssertExpectations(s.T())
 	s.mockBatchInsertBuilder.AssertExpectations(s.T())
 }
 
 func (s *TransactionsProcessorTestSuiteLedger) TestAddTransactionsSucceeds() {
 	sequence := uint32(20)
-
+	lcm := xdr.LedgerCloseMeta{
+		V0: &xdr.LedgerCloseMetaV0{
+			LedgerHeader: xdr.LedgerHeaderHistoryEntry{
+				Header: xdr.LedgerHeader{
+					LedgerSeq: xdr.Uint32(sequence),
+				},
+			},
+		},
+	}
 	firstTx := createTransaction(true, 1)
 	secondTx := createTransaction(false, 3)
 	thirdTx := createTransaction(true, 4)
 
 	s.mockBatchInsertBuilder.On("Add", firstTx, sequence).Return(nil).Once()
 	s.mockBatchInsertBuilder.On("Add", secondTx, sequence).Return(nil).Once()
-	s.mockBatchInsertBuilder.On("Add", thirdTx, sequence).Return(nil).Once()
+	s.mockBatchInsertBuilder.On("Add", thirdTx, sequence+1).Return(nil).Once()
 	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
-	s.Assert().NoError(s.processor.Commit(s.ctx))
-
-	err := s.processor.ProcessTransaction(s.ctx, firstTx)
-	s.Assert().NoError(err)
 
-	err = s.processor.ProcessTransaction(s.ctx, secondTx)
-	s.Assert().NoError(err)
+	s.Assert().NoError(s.processor.ProcessTransaction(lcm, firstTx))
+	s.Assert().NoError(s.processor.ProcessTransaction(lcm, secondTx))
+	lcm.V0.LedgerHeader.Header.LedgerSeq++
+	s.Assert().NoError(s.processor.ProcessTransaction(lcm, thirdTx))
 
-	err = s.processor.ProcessTransaction(s.ctx, thirdTx)
-	s.Assert().NoError(err)
+	s.Assert().NoError(s.processor.Flush(s.ctx, s.mockSession))
 }
 
 func (s *TransactionsProcessorTestSuiteLedger) TestAddTransactionsFails() {
 	sequence := uint32(20)
+	lcm := xdr.LedgerCloseMeta{
+		V0: &xdr.LedgerCloseMetaV0{
+			LedgerHeader: xdr.LedgerHeaderHistoryEntry{
+				Header: xdr.LedgerHeader{
+					LedgerSeq: xdr.Uint32(sequence),
+				},
+			},
+		},
+	}
 	firstTx := createTransaction(true, 1)
 	s.mockBatchInsertBuilder.On("Add", firstTx, sequence).
 		Return(errors.New("transient error")).Once()
 
-	err := s.processor.ProcessTransaction(s.ctx, firstTx)
+	err := s.processor.ProcessTransaction(lcm, firstTx)
 	s.Assert().Error(err)
 	s.Assert().EqualError(err, "Error batch inserting transaction rows: transient error")
 }
 
 func (s *TransactionsProcessorTestSuiteLedger) TestExecFails() {
 	sequence := uint32(20)
+	lcm := xdr.LedgerCloseMeta{
+		V0: &xdr.LedgerCloseMetaV0{
+			LedgerHeader: xdr.LedgerHeaderHistoryEntry{
+				Header: xdr.LedgerHeader{
+					LedgerSeq: xdr.Uint32(sequence),
+				},
+			},
+		},
+	}
 	firstTx := createTransaction(true, 1)
 
 	s.mockBatchInsertBuilder.On("Add", firstTx, sequence).Return(nil).Once()
 	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(errors.New("transient error")).Once()
 
-	err := s.processor.ProcessTransaction(s.ctx, firstTx)
-	s.Assert().NoError(err)
+	s.Assert().NoError(s.processor.ProcessTransaction(lcm, firstTx))
 
-	err = s.processor.Commit(s.ctx)
+	err := s.processor.Flush(s.ctx, s.mockSession)
 	s.Assert().Error(err)
-	s.Assert().EqualError(err, "Error flushing transaction batch: transient error")
+	s.Assert().EqualError(err, "transient error")
 }
diff --git a/xdr/ledger_close_meta.go b/xdr/ledger_close_meta.go
index 4d3248f394..4a4871cd77 100644
--- a/xdr/ledger_close_meta.go
+++ b/xdr/ledger_close_meta.go
@@ -1,5 +1,18 @@
 package xdr
 
+import "fmt"
+
+func (l LedgerCloseMeta) LedgerHeaderHistoryEntry() LedgerHeaderHistoryEntry {
+	switch l.V {
+	case 0:
+		return l.MustV0().LedgerHeader
+	case 1:
+		return l.MustV1().LedgerHeader
+	default:
+		panic(fmt.Sprintf("Unsupported LedgerCloseMeta.V: %d", l.V))
+	}
+}
+
 func (l LedgerCloseMeta) LedgerSequence() uint32 {
 	return uint32(l.MustV0().LedgerHeader.Header.LedgerSeq)
 }

From a630fcb46d296a8c0ea30c3fa1676214b9727285 Mon Sep 17 00:00:00 2001
From: tamirms <tamirms@gmail.com>
Date: Mon, 14 Aug 2023 16:57:48 +0200
Subject: [PATCH 08/17] services/horizon/internal/db2/history: Implement
 account loader and future account ids (#5015)

---
 .../internal/db2/history/account_loader.go    | 193 ++++++++++++++++++
 .../db2/history/account_loader_test.go        |  54 +++++
 2 files changed, 247 insertions(+)
 create mode 100644 services/horizon/internal/db2/history/account_loader.go
 create mode 100644 services/horizon/internal/db2/history/account_loader_test.go

diff --git a/services/horizon/internal/db2/history/account_loader.go b/services/horizon/internal/db2/history/account_loader.go
new file mode 100644
index 0000000000..22481a2c8f
--- /dev/null
+++ b/services/horizon/internal/db2/history/account_loader.go
@@ -0,0 +1,193 @@
+package history
+
+import (
+	"context"
+	"database/sql/driver"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/lib/pq"
+
+	"github.com/stellar/go/support/db"
+	"github.com/stellar/go/support/errors"
+)
+
+// FutureAccountID represents a future history account.
+// A FutureAccountID is created by an AccountLoader and
+// the account id is available after calling Exec() on
+// the AccountLoader.
+type FutureAccountID struct {
+	address string
+	loader  *AccountLoader
+}
+
+const loaderLookupBatchSize = 50000
+
+// Value implements the database/sql/driver Valuer interface.
+func (a FutureAccountID) Value() (driver.Value, error) {
+	return a.loader.GetNow(a.address), nil
+}
+
+// AccountLoader will map account addresses to their history
+// account ids. If there is no existing mapping for a given address,
+// the AccountLoader will insert into the history_accounts table to
+// establish a mapping.
+type AccountLoader struct {
+	sealed bool
+	set    map[string]interface{}
+	ids    map[string]int64
+}
+
+var errSealed = errors.New("cannot register more entries to loader after calling Exec()")
+
+// NewAccountLoader will construct a new AccountLoader instance.
+func NewAccountLoader() *AccountLoader {
+	return &AccountLoader{
+		sealed: false,
+		set:    map[string]interface{}{},
+		ids:    map[string]int64{},
+	}
+}
+
+// GetFuture registers the given account address into the loader and
+// returns a FutureAccountID which will hold the history account id for
+// the address after Exec() is called.
+func (a *AccountLoader) GetFuture(address string) FutureAccountID {
+	if a.sealed {
+		panic(errSealed)
+	}
+
+	a.set[address] = nil
+	return FutureAccountID{
+		address: address,
+		loader:  a,
+	}
+}
+
+// GetNow returns the history account id for the given address.
+// GetNow should only be called on values which were registered by
+// GetFuture() calls. Also, Exec() must be called before any GetNow
+// call can succeed.
+func (a *AccountLoader) GetNow(address string) int64 {
+	if id, ok := a.ids[address]; !ok {
+		panic(fmt.Errorf("address %v not present", address))
+	} else {
+		return id
+	}
+}
+
+func (a *AccountLoader) lookupKeys(ctx context.Context, q *Q, addresses []string) error {
+	for i := 0; i < len(addresses); i += loaderLookupBatchSize {
+		end := i + loaderLookupBatchSize
+		if end > len(addresses) {
+			end = len(addresses)
+		}
+
+		var accounts []Account
+		if err := q.AccountsByAddresses(ctx, &accounts, addresses[i:end]); err != nil {
+			return errors.Wrap(err, "could not select accounts")
+		}
+
+		for _, account := range accounts {
+			a.ids[account.Address] = account.ID
+		}
+	}
+	return nil
+}
+
+// Exec will look up all the history account ids for the addresses registered in the loader.
+// If there are no history account ids for a given set of addresses, Exec will insert rows
+// into the history_accounts table to establish a mapping between address and history account id.
+func (a *AccountLoader) Exec(ctx context.Context, session db.SessionInterface) error {
+	a.sealed = true
+	if len(a.set) == 0 {
+		return nil
+	}
+	q := &Q{session}
+	addresses := make([]string, 0, len(a.set))
+	for address := range a.set {
+		addresses = append(addresses, address)
+	}
+	// sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock
+	// https://github.com/stellar/go/issues/2370
+	sort.Strings(addresses)
+
+	if err := a.lookupKeys(ctx, q, addresses); err != nil {
+		return err
+	}
+
+	insert := 0
+	for _, address := range addresses {
+		if _, ok := a.ids[address]; ok {
+			continue
+		}
+		addresses[insert] = address
+		insert++
+	}
+	if insert == 0 {
+		return nil
+	}
+	addresses = addresses[:insert]
+
+	err := bulkInsert(
+		ctx,
+		q,
+		"history_accounts",
+		[]string{"address"},
+		[]bulkInsertField{
+			{
+				name:    "address",
+				dbType:  "character varying(64)",
+				objects: addresses,
+			},
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	return a.lookupKeys(ctx, q, addresses)
+}
+
+type bulkInsertField struct {
+	name    string
+	dbType  string
+	objects []string
+}
+
+func bulkInsert(ctx context.Context, q *Q, table string, conflictFields []string, fields []bulkInsertField) error {
+	unnestPart := make([]string, 0, len(fields))
+	insertFieldsPart := make([]string, 0, len(fields))
+	pqArrays := make([]interface{}, 0, len(fields))
+
+	for _, field := range fields {
+		unnestPart = append(
+			unnestPart,
+			fmt.Sprintf("unnest(?::%s[]) /* %s */", field.dbType, field.name),
+		)
+		insertFieldsPart = append(
+			insertFieldsPart,
+			field.name,
+		)
+		pqArrays = append(
+			pqArrays,
+			pq.Array(field.objects),
+		)
+	}
+
+	sql := `
+	WITH r AS
+		(SELECT ` + strings.Join(unnestPart, ",") + `)
+	INSERT INTO ` + table + `
+		(` + strings.Join(insertFieldsPart, ",") + `)
+	SELECT * from r
+	ON CONFLICT (` + strings.Join(conflictFields, ",") + `) DO NOTHING`
+
+	_, err := q.ExecRaw(
+		context.WithValue(ctx, &db.QueryTypeContextKey, db.UpsertQueryType),
+		sql,
+		pqArrays...,
+	)
+	return err
+}
diff --git a/services/horizon/internal/db2/history/account_loader_test.go b/services/horizon/internal/db2/history/account_loader_test.go
new file mode 100644
index 0000000000..785a68f118
--- /dev/null
+++ b/services/horizon/internal/db2/history/account_loader_test.go
@@ -0,0 +1,54 @@
+package history
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/stellar/go/keypair"
+	"github.com/stellar/go/services/horizon/internal/test"
+)
+
+func TestAccountLoader(t *testing.T) {
+	tt := test.Start(t)
+	defer tt.Finish()
+	test.ResetHorizonDB(t, tt.HorizonDB)
+	session := tt.HorizonSession()
+
+	var addresses []string
+	for i := 0; i < 100; i++ {
+		addresses = append(addresses, keypair.MustRandom().Address())
+	}
+
+	loader := NewAccountLoader()
+	var futures []FutureAccountID
+	for _, address := range addresses {
+		future := loader.GetFuture(address)
+		futures = append(futures, future)
+		assert.Panics(t, func() {
+			loader.GetNow(address)
+		})
+		assert.Panics(t, func() {
+			future.Value()
+		})
+	}
+
+	assert.NoError(t, loader.Exec(context.Background(), session))
+	assert.Panics(t, func() {
+		loader.GetFuture(keypair.MustRandom().Address())
+	})
+
+	q := &Q{session}
+	for i, address := range addresses {
+		future := futures[i]
+		id := loader.GetNow(address)
+		val, err := future.Value()
+		assert.NoError(t, err)
+		assert.Equal(t, id, val)
+		var account Account
+		assert.NoError(t, q.AccountByAddress(context.Background(), &account, address))
+		assert.Equal(t, account.ID, id)
+		assert.Equal(t, account.Address, address)
+	}
+}

From a4db2a966e3a95534d3a1701ab2bad07cdb48f07 Mon Sep 17 00:00:00 2001
From: tamirms <tamirms@gmail.com>
Date: Mon, 21 Aug 2023 21:01:43 +0100
Subject: [PATCH 09/17] services/horizon/internal/db2/history: Implement
 loaders for assets, claimable balances, and liquidity pools (#5019)

---
 .../internal/db2/history/account_loader.go    |  29 ++-
 .../db2/history/account_loader_test.go        |   4 +-
 .../internal/db2/history/asset_loader.go      | 188 ++++++++++++++++++
 .../internal/db2/history/asset_loader_test.go |  73 +++++++
 .../db2/history/claimable_balance_loader.go   | 143 +++++++++++++
 .../history/claimable_balance_loader_test.go  |  60 ++++++
 .../db2/history/liquidity_pool_loader.go      | 143 +++++++++++++
 .../db2/history/liquidity_pool_loader_test.go |  57 ++++++
 8 files changed, 680 insertions(+), 17 deletions(-)
 create mode 100644 services/horizon/internal/db2/history/asset_loader.go
 create mode 100644 services/horizon/internal/db2/history/asset_loader_test.go
 create mode 100644 services/horizon/internal/db2/history/claimable_balance_loader.go
 create mode 100644 services/horizon/internal/db2/history/claimable_balance_loader_test.go
 create mode 100644 services/horizon/internal/db2/history/liquidity_pool_loader.go
 create mode 100644 services/horizon/internal/db2/history/liquidity_pool_loader_test.go

diff --git a/services/horizon/internal/db2/history/account_loader.go b/services/horizon/internal/db2/history/account_loader.go
index 22481a2c8f..14bcfc5243 100644
--- a/services/horizon/internal/db2/history/account_loader.go
+++ b/services/horizon/internal/db2/history/account_loader.go
@@ -9,8 +9,10 @@ import (
 
 	"github.com/lib/pq"
 
+	"github.com/stellar/go/support/collections/set"
 	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
+	"github.com/stellar/go/support/ordered"
 )
 
 // FutureAccountID represents a future history account.
@@ -26,7 +28,7 @@ const loaderLookupBatchSize = 50000
 
 // Value implements the database/sql/driver Valuer interface.
 func (a FutureAccountID) Value() (driver.Value, error) {
-	return a.loader.GetNow(a.address), nil
+	return a.loader.getNow(a.address), nil
 }
 
 // AccountLoader will map account addresses to their history
@@ -35,7 +37,7 @@ func (a FutureAccountID) Value() (driver.Value, error) {
 // establish a mapping.
 type AccountLoader struct {
 	sealed bool
-	set    map[string]interface{}
+	set    set.Set[string]
 	ids    map[string]int64
 }
 
@@ -45,7 +47,7 @@ var errSealed = errors.New("cannot register more entries to loader after calling
 func NewAccountLoader() *AccountLoader {
 	return &AccountLoader{
 		sealed: false,
-		set:    map[string]interface{}{},
+		set:    set.Set[string]{},
 		ids:    map[string]int64{},
 	}
 }
@@ -58,18 +60,18 @@ func (a *AccountLoader) GetFuture(address string) FutureAccountID {
 		panic(errSealed)
 	}
 
-	a.set[address] = nil
+	a.set.Add(address)
 	return FutureAccountID{
 		address: address,
 		loader:  a,
 	}
 }
 
-// GetNow returns the history account id for the given address.
-// GetNow should only be called on values which were registered by
-// GetFuture() calls. Also, Exec() must be called before any GetNow
+// getNow returns the history account id for the given address.
+// getNow should only be called on values which were registered by
+// GetFuture() calls. Also, Exec() must be called before any getNow
 // call can succeed.
-func (a *AccountLoader) GetNow(address string) int64 {
+func (a *AccountLoader) getNow(address string) int64 {
 	if id, ok := a.ids[address]; !ok {
 		panic(fmt.Errorf("address %v not present", address))
 	} else {
@@ -79,10 +81,7 @@ func (a *AccountLoader) GetNow(address string) int64 {
 
 func (a *AccountLoader) lookupKeys(ctx context.Context, q *Q, addresses []string) error {
 	for i := 0; i < len(addresses); i += loaderLookupBatchSize {
-		end := i + loaderLookupBatchSize
-		if end > len(addresses) {
-			end = len(addresses)
-		}
+		end := ordered.Min(len(addresses), i+loaderLookupBatchSize)
 
 		var accounts []Account
 		if err := q.AccountsByAddresses(ctx, &accounts, addresses[i:end]); err != nil {
@@ -109,9 +108,6 @@ func (a *AccountLoader) Exec(ctx context.Context, session db.SessionInterface) e
 	for address := range a.set {
 		addresses = append(addresses, address)
 	}
-	// sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock
-	// https://github.com/stellar/go/issues/2370
-	sort.Strings(addresses)
 
 	if err := a.lookupKeys(ctx, q, addresses); err != nil {
 		return err
@@ -129,6 +125,9 @@ func (a *AccountLoader) Exec(ctx context.Context, session db.SessionInterface) e
 		return nil
 	}
 	addresses = addresses[:insert]
+	// sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock
+	// https://github.com/stellar/go/issues/2370
+	sort.Strings(addresses)
 
 	err := bulkInsert(
 		ctx,
diff --git a/services/horizon/internal/db2/history/account_loader_test.go b/services/horizon/internal/db2/history/account_loader_test.go
index 785a68f118..14d933c0ad 100644
--- a/services/horizon/internal/db2/history/account_loader_test.go
+++ b/services/horizon/internal/db2/history/account_loader_test.go
@@ -27,7 +27,7 @@ func TestAccountLoader(t *testing.T) {
 		future := loader.GetFuture(address)
 		futures = append(futures, future)
 		assert.Panics(t, func() {
-			loader.GetNow(address)
+			loader.getNow(address)
 		})
 		assert.Panics(t, func() {
 			future.Value()
@@ -42,7 +42,7 @@ func TestAccountLoader(t *testing.T) {
 	q := &Q{session}
 	for i, address := range addresses {
 		future := futures[i]
-		id := loader.GetNow(address)
+		id := loader.getNow(address)
 		val, err := future.Value()
 		assert.NoError(t, err)
 		assert.Equal(t, id, val)
diff --git a/services/horizon/internal/db2/history/asset_loader.go b/services/horizon/internal/db2/history/asset_loader.go
new file mode 100644
index 0000000000..326f1b68ba
--- /dev/null
+++ b/services/horizon/internal/db2/history/asset_loader.go
@@ -0,0 +1,188 @@
+package history
+
+import (
+	"context"
+	"database/sql/driver"
+	"fmt"
+	"sort"
+
+	sq "github.com/Masterminds/squirrel"
+
+	"github.com/stellar/go/support/collections/set"
+	"github.com/stellar/go/support/db"
+	"github.com/stellar/go/support/errors"
+	"github.com/stellar/go/support/ordered"
+)
+
+type AssetKey struct {
+	Type   string
+	Code   string
+	Issuer string
+}
+
+// FutureAssetID represents a future history asset.
+// A FutureAssetID is created by an AssetLoader and
+// the asset id is available after calling Exec() on
+// the AssetLoader.
+type FutureAssetID struct {
+	asset  AssetKey
+	loader *AssetLoader
+}
+
+// Value implements the database/sql/driver Valuer interface.
+func (a FutureAssetID) Value() (driver.Value, error) {
+	return a.loader.getNow(a.asset), nil
+}
+
+// AssetLoader will map assets to their history
+// asset ids. If there is no existing mapping for a given sset,
+// the AssetLoader will insert into the history_assets table to
+// establish a mapping.
+type AssetLoader struct {
+	sealed bool
+	set    set.Set[AssetKey]
+	ids    map[AssetKey]int64
+}
+
+// NewAssetLoader will construct a new AssetLoader instance.
+func NewAssetLoader() *AssetLoader {
+	return &AssetLoader{
+		sealed: false,
+		set:    set.Set[AssetKey]{},
+		ids:    map[AssetKey]int64{},
+	}
+}
+
+// GetFuture registers the given asset into the loader and
+// returns a FutureAssetID which will hold the history asset id for
+// the asset after Exec() is called.
+func (a *AssetLoader) GetFuture(asset AssetKey) FutureAssetID {
+	if a.sealed {
+		panic(errSealed)
+	}
+	a.set.Add(asset)
+	return FutureAssetID{
+		asset:  asset,
+		loader: a,
+	}
+}
+
+// getNow returns the history asset id for the given asset.
+// getNow should only be called on values which were registered by
+// GetFuture() calls. Also, Exec() must be called before any getNow
+// call can succeed.
+func (a *AssetLoader) getNow(asset AssetKey) int64 {
+	if id, ok := a.ids[asset]; !ok {
+		panic(fmt.Errorf("asset %v not present", asset))
+	} else {
+		return id
+	}
+}
+
+func (a *AssetLoader) lookupKeys(ctx context.Context, q *Q, keys []AssetKey) error {
+	var rows []Asset
+	for i := 0; i < len(keys); i += loaderLookupBatchSize {
+		end := ordered.Min(len(keys), i+loaderLookupBatchSize)
+		subset := keys[i:end]
+		keyStrings := make([]string, 0, len(subset))
+		for _, key := range subset {
+			keyStrings = append(keyStrings, key.Type+"/"+key.Code+"/"+key.Issuer)
+		}
+		err := q.Select(ctx, &rows, sq.Select("*").From("history_assets").Where(sq.Eq{
+			"concat(asset_type, '/', asset_code, '/', asset_issuer)": keyStrings,
+		}))
+		if err != nil {
+			return errors.Wrap(err, "could not select assets")
+		}
+
+		for _, row := range rows {
+			a.ids[AssetKey{
+				Type:   row.Type,
+				Code:   row.Code,
+				Issuer: row.Issuer,
+			}] = row.ID
+		}
+	}
+	return nil
+}
+
+// Exec will look up all the history asset ids for the assets registered in the loader.
+// If there are no history asset ids for a given set of assets, Exec will insert rows
+// into the history_assets table.
+func (a *AssetLoader) Exec(ctx context.Context, session db.SessionInterface) error {
+	a.sealed = true
+	if len(a.set) == 0 {
+		return nil
+	}
+	q := &Q{session}
+	keys := make([]AssetKey, 0, len(a.set))
+	for key := range a.set {
+		keys = append(keys, key)
+	}
+
+	if err := a.lookupKeys(ctx, q, keys); err != nil {
+		return err
+	}
+
+	assetTypes := make([]string, 0, len(a.set)-len(a.ids))
+	assetCodes := make([]string, 0, len(a.set)-len(a.ids))
+	assetIssuers := make([]string, 0, len(a.set)-len(a.ids))
+	insert := 0
+	for _, key := range keys {
+		if _, ok := a.ids[key]; ok {
+			continue
+		}
+		assetTypes = append(assetTypes, key.Type)
+		assetCodes = append(assetCodes, key.Code)
+		assetIssuers = append(assetIssuers, key.Issuer)
+		keys[insert] = key
+		insert++
+	}
+	if insert == 0 {
+		return nil
+	}
+	keys = keys[:insert]
+	// sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock
+	// https://github.com/stellar/go/issues/2370
+	sort.Slice(keys, func(i, j int) bool {
+		if keys[i].Type < keys[j].Type {
+			return true
+		}
+		if keys[i].Code < keys[j].Code {
+			return true
+		}
+		if keys[i].Issuer < keys[j].Issuer {
+			return true
+		}
+		return false
+	})
+
+	err := bulkInsert(
+		ctx,
+		q,
+		"history_assets",
+		[]string{"asset_code", "asset_type", "asset_issuer"},
+		[]bulkInsertField{
+			{
+				name:    "asset_code",
+				dbType:  "character varying(12)",
+				objects: assetCodes,
+			},
+			{
+				name:    "asset_issuer",
+				dbType:  "character varying(56)",
+				objects: assetIssuers,
+			},
+			{
+				name:    "asset_type",
+				dbType:  "character varying(64)",
+				objects: assetTypes,
+			},
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	return a.lookupKeys(ctx, q, keys)
+}
diff --git a/services/horizon/internal/db2/history/asset_loader_test.go b/services/horizon/internal/db2/history/asset_loader_test.go
new file mode 100644
index 0000000000..f51b2e27ef
--- /dev/null
+++ b/services/horizon/internal/db2/history/asset_loader_test.go
@@ -0,0 +1,73 @@
+package history
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/stellar/go/keypair"
+	"github.com/stellar/go/services/horizon/internal/test"
+	"github.com/stellar/go/xdr"
+)
+
+func TestAssetLoader(t *testing.T) {
+	tt := test.Start(t)
+	defer tt.Finish()
+	test.ResetHorizonDB(t, tt.HorizonDB)
+	session := tt.HorizonSession()
+
+	var keys []AssetKey
+	for i := 0; i < 100; i++ {
+		var key AssetKey
+		if i == 0 {
+			key.Type = "native"
+		} else if i%2 == 0 {
+			key.Type = "credit_alphanum4"
+			key.Code = fmt.Sprintf("ab%d", i)
+			key.Issuer = keypair.MustRandom().Address()
+		} else {
+			key.Type = "credit_alphanum12"
+			key.Code = fmt.Sprintf("abcdef%d", i)
+			key.Issuer = keypair.MustRandom().Address()
+		}
+		keys = append(keys, key)
+	}
+
+	loader := NewAssetLoader()
+	var futures []FutureAssetID
+	for _, key := range keys {
+		future := loader.GetFuture(key)
+		futures = append(futures, future)
+		assert.Panics(t, func() {
+			loader.getNow(key)
+		})
+		assert.Panics(t, func() {
+			future.Value()
+		})
+	}
+
+	assert.NoError(t, loader.Exec(context.Background(), session))
+	assert.Panics(t, func() {
+		loader.GetFuture(AssetKey{Type: "invalid"})
+	})
+
+	q := &Q{session}
+	for i, key := range keys {
+		future := futures[i]
+		internalID := loader.getNow(key)
+		val, err := future.Value()
+		assert.NoError(t, err)
+		assert.Equal(t, internalID, val)
+		var assetXDR xdr.Asset
+		if key.Type == "native" {
+			assetXDR = xdr.MustNewNativeAsset()
+		} else {
+			assetXDR = xdr.MustNewCreditAsset(key.Code, key.Issuer)
+		}
+		assetID, err := q.GetAssetID(context.Background(), assetXDR)
+		assert.NoError(t, err)
+		assert.Equal(t, assetID, internalID)
+	}
+}
diff --git a/services/horizon/internal/db2/history/claimable_balance_loader.go b/services/horizon/internal/db2/history/claimable_balance_loader.go
new file mode 100644
index 0000000000..a077eb683e
--- /dev/null
+++ b/services/horizon/internal/db2/history/claimable_balance_loader.go
@@ -0,0 +1,143 @@
+package history
+
+import (
+	"context"
+	"database/sql/driver"
+	"fmt"
+	"sort"
+
+	"github.com/stellar/go/support/collections/set"
+	"github.com/stellar/go/support/db"
+	"github.com/stellar/go/support/errors"
+	"github.com/stellar/go/support/ordered"
+)
+
+// FutureClaimableBalanceID represents a future history claimable balance.
+// A FutureClaimableBalanceID is created by a ClaimableBalanceLoader and
+// the claimable balance id is available after calling Exec() on
+// the ClaimableBalanceLoader.
+type FutureClaimableBalanceID struct {
+	id     string
+	loader *ClaimableBalanceLoader
+}
+
+// Value implements the database/sql/driver Valuer interface.
+func (a FutureClaimableBalanceID) Value() (driver.Value, error) {
+	return a.loader.getNow(a.id), nil
+}
+
+// ClaimableBalanceLoader will map claimable balance ids to their internal
+// history ids. If there is no existing mapping for a given claimable balance id,
+// the ClaimableBalanceLoader will insert into the history_claimable_balances table to
+// establish a mapping.
+type ClaimableBalanceLoader struct {
+	sealed bool
+	set    set.Set[string]
+	ids    map[string]int64
+}
+
+// NewClaimableBalanceLoader will construct a new ClaimableBalanceLoader instance.
+func NewClaimableBalanceLoader() *ClaimableBalanceLoader {
+	return &ClaimableBalanceLoader{
+		sealed: false,
+		set:    set.Set[string]{},
+		ids:    map[string]int64{},
+	}
+}
+
+// GetFuture registers the given claimable balance into the loader and
+// returns a FutureClaimableBalanceID which will hold the internal history id for
+// the claimable balance after Exec() is called.
+func (a *ClaimableBalanceLoader) GetFuture(id string) FutureClaimableBalanceID {
+	if a.sealed {
+		panic(errSealed)
+	}
+
+	a.set.Add(id)
+	return FutureClaimableBalanceID{
+		id:     id,
+		loader: a,
+	}
+}
+
+// getNow returns the internal history id for the given claimable balance.
+// getNow should only be called on values which were registered by
+// GetFuture() calls. Also, Exec() must be called before any getNow
+// call can succeed.
+func (a *ClaimableBalanceLoader) getNow(id string) int64 {
+	if internalID, ok := a.ids[id]; !ok {
+		panic(fmt.Errorf("id %v not present", id))
+	} else {
+		return internalID
+	}
+}
+
+func (a *ClaimableBalanceLoader) lookupKeys(ctx context.Context, q *Q, ids []string) error {
+	for i := 0; i < len(ids); i += loaderLookupBatchSize {
+		end := ordered.Min(len(ids), i+loaderLookupBatchSize)
+
+		cbs, err := q.ClaimableBalancesByIDs(ctx, ids[i:end])
+		if err != nil {
+			return errors.Wrap(err, "could not select claimable balances")
+		}
+
+		for _, cb := range cbs {
+			a.ids[cb.BalanceID] = cb.InternalID
+		}
+	}
+	return nil
+}
+
+// Exec will look up all the internal history ids for the claimable balances registered in the loader.
+// If there are no internal ids for a given set of claimable balances, Exec will insert rows
+// into the history_claimable_balances table.
+func (a *ClaimableBalanceLoader) Exec(ctx context.Context, session db.SessionInterface) error {
+	a.sealed = true
+	if len(a.set) == 0 {
+		return nil
+	}
+	q := &Q{session}
+	ids := make([]string, 0, len(a.set))
+	for id := range a.set {
+		ids = append(ids, id)
+	}
+
+	if err := a.lookupKeys(ctx, q, ids); err != nil {
+		return err
+	}
+
+	insert := 0
+	for _, id := range ids {
+		if _, ok := a.ids[id]; ok {
+			continue
+		}
+		ids[insert] = id
+		insert++
+	}
+	if insert == 0 {
+		return nil
+	}
+	ids = ids[:insert]
+	// sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock
+	// https://github.com/stellar/go/issues/2370
+	sort.Strings(ids)
+
+	err := bulkInsert(
+		ctx,
+		q,
+		"history_claimable_balances",
+		[]string{"claimable_balance_id"},
+		[]bulkInsertField{
+			{
+				name:    "claimable_balance_id",
+				dbType:  "text",
+				objects: ids,
+			},
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	return a.lookupKeys(ctx, q, ids)
+}
diff --git a/services/horizon/internal/db2/history/claimable_balance_loader_test.go b/services/horizon/internal/db2/history/claimable_balance_loader_test.go
new file mode 100644
index 0000000000..183bdb3daa
--- /dev/null
+++ b/services/horizon/internal/db2/history/claimable_balance_loader_test.go
@@ -0,0 +1,60 @@
+package history
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/stellar/go/services/horizon/internal/test"
+	"github.com/stellar/go/xdr"
+)
+
+func TestClaimableBalanceLoader(t *testing.T) {
+	tt := test.Start(t)
+	defer tt.Finish()
+	test.ResetHorizonDB(t, tt.HorizonDB)
+	session := tt.HorizonSession()
+
+	var ids []string
+	for i := 0; i < 100; i++ {
+		balanceID := xdr.ClaimableBalanceId{
+			Type: xdr.ClaimableBalanceIdTypeClaimableBalanceIdTypeV0,
+			V0:   &xdr.Hash{byte(i)},
+		}
+		id, err := xdr.MarshalHex(balanceID)
+		tt.Assert.NoError(err)
+		ids = append(ids, id)
+	}
+
+	loader := NewClaimableBalanceLoader()
+	var futures []FutureClaimableBalanceID
+	for _, id := range ids {
+		future := loader.GetFuture(id)
+		futures = append(futures, future)
+		assert.Panics(t, func() {
+			loader.getNow(id)
+		})
+		assert.Panics(t, func() {
+			future.Value()
+		})
+	}
+
+	assert.NoError(t, loader.Exec(context.Background(), session))
+	assert.Panics(t, func() {
+		loader.GetFuture("not-present")
+	})
+
+	q := &Q{session}
+	for i, id := range ids {
+		future := futures[i]
+		internalID := loader.getNow(id)
+		val, err := future.Value()
+		assert.NoError(t, err)
+		assert.Equal(t, internalID, val)
+		cb, err := q.ClaimableBalanceByID(context.Background(), id)
+		assert.NoError(t, err)
+		assert.Equal(t, cb.BalanceID, id)
+		assert.Equal(t, cb.InternalID, internalID)
+	}
+}
diff --git a/services/horizon/internal/db2/history/liquidity_pool_loader.go b/services/horizon/internal/db2/history/liquidity_pool_loader.go
new file mode 100644
index 0000000000..ac501dcfd3
--- /dev/null
+++ b/services/horizon/internal/db2/history/liquidity_pool_loader.go
@@ -0,0 +1,143 @@
+package history
+
+import (
+	"context"
+	"database/sql/driver"
+	"fmt"
+	"sort"
+
+	"github.com/stellar/go/support/collections/set"
+	"github.com/stellar/go/support/db"
+	"github.com/stellar/go/support/errors"
+	"github.com/stellar/go/support/ordered"
+)
+
+// FutureLiquidityPoolID represents a future history liquidity pool.
+// A FutureLiquidityPoolID is created by an LiquidityPoolLoader and
+// the liquidity pool id is available after calling Exec() on
+// the LiquidityPoolLoader.
+type FutureLiquidityPoolID struct {
+	id     string
+	loader *LiquidityPoolLoader
+}
+
+// Value implements the database/sql/driver Valuer interface.
+func (a FutureLiquidityPoolID) Value() (driver.Value, error) {
+	return a.loader.getNow(a.id), nil
+}
+
+// LiquidityPoolLoader will map liquidity pools to their internal
+// history ids. If there is no existing mapping for a given liquidity pool,
+// the LiquidityPoolLoader will insert into the history_liquidity_pools table to
+// establish a mapping.
+type LiquidityPoolLoader struct {
+	sealed bool
+	set    set.Set[string]
+	ids    map[string]int64
+}
+
+// NewLiquidityPoolLoader will construct a new LiquidityPoolLoader instance.
+func NewLiquidityPoolLoader() *LiquidityPoolLoader {
+	return &LiquidityPoolLoader{
+		sealed: false,
+		set:    set.Set[string]{},
+		ids:    map[string]int64{},
+	}
+}
+
+// GetFuture registers the given liquidity pool into the loader and
+// returns a FutureLiquidityPoolID which will hold the internal history id for
+// the liquidity pool after Exec() is called.
+func (a *LiquidityPoolLoader) GetFuture(id string) FutureLiquidityPoolID {
+	if a.sealed {
+		panic(errSealed)
+	}
+
+	a.set.Add(id)
+	return FutureLiquidityPoolID{
+		id:     id,
+		loader: a,
+	}
+}
+
+// getNow returns the internal history id for the given liquidity pool.
+// getNow should only be called on values which were registered by
+// GetFuture() calls. Also, Exec() must be called before any getNow
+// call can succeed.
+func (a *LiquidityPoolLoader) getNow(id string) int64 {
+	if id, ok := a.ids[id]; !ok {
+		panic(fmt.Errorf("id %v not present", id))
+	} else {
+		return id
+	}
+}
+
+func (a *LiquidityPoolLoader) lookupKeys(ctx context.Context, q *Q, ids []string) error {
+	for i := 0; i < len(ids); i += loaderLookupBatchSize {
+		end := ordered.Min(len(ids), i+loaderLookupBatchSize)
+
+		lps, err := q.LiquidityPoolsByIDs(ctx, ids[i:end])
+		if err != nil {
+			return errors.Wrap(err, "could not select accounts")
+		}
+
+		for _, lp := range lps {
+			a.ids[lp.PoolID] = lp.InternalID
+		}
+	}
+	return nil
+}
+
+// Exec will look up all the internal history ids for the liquidity pools registered in the loader.
+// If there are no internal history ids for a given set of liquidity pools, Exec will insert rows
+// into the history_liquidity_pools table.
+func (a *LiquidityPoolLoader) Exec(ctx context.Context, session db.SessionInterface) error {
+	a.sealed = true
+	if len(a.set) == 0 {
+		return nil
+	}
+	q := &Q{session}
+	ids := make([]string, 0, len(a.set))
+	for id := range a.set {
+		ids = append(ids, id)
+	}
+
+	if err := a.lookupKeys(ctx, q, ids); err != nil {
+		return err
+	}
+
+	insert := 0
+	for _, id := range ids {
+		if _, ok := a.ids[id]; ok {
+			continue
+		}
+		ids[insert] = id
+		insert++
+	}
+	if insert == 0 {
+		return nil
+	}
+	ids = ids[:insert]
+	// sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock
+	// https://github.com/stellar/go/issues/2370
+	sort.Strings(ids)
+
+	err := bulkInsert(
+		ctx,
+		q,
+		"history_liquidity_pools",
+		[]string{"liquidity_pool_id"},
+		[]bulkInsertField{
+			{
+				name:    "liquidity_pool_id",
+				dbType:  "text",
+				objects: ids,
+			},
+		},
+	)
+	if err != nil {
+		return err
+	}
+
+	return a.lookupKeys(ctx, q, ids)
+}
diff --git a/services/horizon/internal/db2/history/liquidity_pool_loader_test.go b/services/horizon/internal/db2/history/liquidity_pool_loader_test.go
new file mode 100644
index 0000000000..00664ff3e3
--- /dev/null
+++ b/services/horizon/internal/db2/history/liquidity_pool_loader_test.go
@@ -0,0 +1,57 @@
+package history
+
+import (
+	"context"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/stellar/go/services/horizon/internal/test"
+	"github.com/stellar/go/xdr"
+)
+
+func TestLiquidityPoolLoader(t *testing.T) {
+	tt := test.Start(t)
+	defer tt.Finish()
+	test.ResetHorizonDB(t, tt.HorizonDB)
+	session := tt.HorizonSession()
+
+	var ids []string
+	for i := 0; i < 100; i++ {
+		poolID := xdr.PoolId{byte(i)}
+		id, err := xdr.MarshalHex(poolID)
+		tt.Assert.NoError(err)
+		ids = append(ids, id)
+	}
+
+	loader := NewLiquidityPoolLoader()
+	var futures []FutureLiquidityPoolID
+	for _, id := range ids {
+		future := loader.GetFuture(id)
+		futures = append(futures, future)
+		assert.Panics(t, func() {
+			loader.getNow(id)
+		})
+		assert.Panics(t, func() {
+			future.Value()
+		})
+	}
+
+	assert.NoError(t, loader.Exec(context.Background(), session))
+	assert.Panics(t, func() {
+		loader.GetFuture("not-present")
+	})
+
+	q := &Q{session}
+	for i, id := range ids {
+		future := futures[i]
+		internalID := loader.getNow(id)
+		val, err := future.Value()
+		assert.NoError(t, err)
+		assert.Equal(t, internalID, val)
+		lp, err := q.LiquidityPoolByID(context.Background(), id)
+		assert.NoError(t, err)
+		assert.Equal(t, lp.PoolID, id)
+		assert.Equal(t, lp.InternalID, internalID)
+	}
+}

From 461e5a12a82cab4cd2138e77627fb588d6043191 Mon Sep 17 00:00:00 2001
From: tamirms <tamirms@gmail.com>
Date: Wed, 23 Aug 2023 07:59:14 +0100
Subject: [PATCH 10/17] services/horizon/internal/ingest/processors: Refactor
 participants processors to support new ingestion data flow (#5024)

---
 ...ration_participant_batch_insert_builder.go |   4 +-
 .../db2/history/mock_q_participants.go        |   5 +-
 ...ration_participant_batch_insert_builder.go |   4 +-
 ...n_participant_batch_insert_builder_test.go |  12 +-
 .../internal/db2/history/participants.go      |   4 +-
 .../internal/db2/history/participants_test.go |  37 +--
 services/horizon/internal/ingest/main_test.go |   2 +-
 .../processors/participants_processor.go      | 146 +++--------
 .../processors/participants_processor_test.go | 230 +++++-------------
 9 files changed, 131 insertions(+), 313 deletions(-)

diff --git a/services/horizon/internal/db2/history/mock_operation_participant_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_operation_participant_batch_insert_builder.go
index 7c98ad2729..481a731043 100644
--- a/services/horizon/internal/db2/history/mock_operation_participant_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/mock_operation_participant_batch_insert_builder.go
@@ -14,8 +14,8 @@ type MockOperationParticipantBatchInsertBuilder struct {
 }
 
 // Add mock
-func (m *MockOperationParticipantBatchInsertBuilder) Add(operationID int64, accountID int64) error {
-	a := m.Called(operationID, accountID)
+func (m *MockOperationParticipantBatchInsertBuilder) Add(operationID int64, account FutureAccountID) error {
+	a := m.Called(operationID, account)
 	return a.Error(0)
 }
 
diff --git a/services/horizon/internal/db2/history/mock_q_participants.go b/services/horizon/internal/db2/history/mock_q_participants.go
index caeb6d23d7..b871199190 100644
--- a/services/horizon/internal/db2/history/mock_q_participants.go
+++ b/services/horizon/internal/db2/history/mock_q_participants.go
@@ -3,8 +3,9 @@ package history
 import (
 	"context"
 
-	"github.com/stellar/go/support/db"
 	"github.com/stretchr/testify/mock"
+
+	"github.com/stellar/go/support/db"
 )
 
 // MockQParticipants is a mock implementation of the QParticipants interface
@@ -28,7 +29,7 @@ type MockTransactionParticipantsBatchInsertBuilder struct {
 	mock.Mock
 }
 
-func (m *MockTransactionParticipantsBatchInsertBuilder) Add(transactionID, accountID int64) error {
+func (m *MockTransactionParticipantsBatchInsertBuilder) Add(transactionID int64, accountID FutureAccountID) error {
 	a := m.Called(transactionID, accountID)
 	return a.Error(0)
 }
diff --git a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder.go b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder.go
index 78ce63a0c1..8882141426 100644
--- a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder.go
@@ -11,7 +11,7 @@ import (
 type OperationParticipantBatchInsertBuilder interface {
 	Add(
 		operationID int64,
-		accountID int64,
+		accountID FutureAccountID,
 	) error
 	Exec(ctx context.Context, session db.SessionInterface) error
 }
@@ -33,7 +33,7 @@ func (q *Q) NewOperationParticipantBatchInsertBuilder() OperationParticipantBatc
 // Add adds an operation participant to the batch
 func (i *operationParticipantBatchInsertBuilder) Add(
 	operationID int64,
-	accountID int64,
+	accountID FutureAccountID,
 ) error {
 	return i.builder.Row(map[string]interface{}{
 		"history_operation_id": operationID,
diff --git a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
index 84e114d0b0..1bcd64cceb 100644
--- a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
+++ b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
@@ -4,6 +4,8 @@ import (
 	"testing"
 
 	sq "github.com/Masterminds/squirrel"
+
+	"github.com/stellar/go/keypair"
 	"github.com/stellar/go/services/horizon/internal/test"
 )
 
@@ -13,13 +15,15 @@ func TestAddOperationParticipants(t *testing.T) {
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
 
+	accountLoader := NewAccountLoader()
+	address := keypair.MustRandom().Address()
 	tt.Assert.NoError(q.Begin())
 	builder := q.NewOperationParticipantBatchInsertBuilder()
-	err := builder.Add(240518172673, 1)
+	err := builder.Add(240518172673, accountLoader.GetFuture(address))
 	tt.Assert.NoError(err)
 
-	err = builder.Exec(tt.Ctx, q)
-	tt.Assert.NoError(err)
+	tt.Assert.NoError(accountLoader.Exec(tt.Ctx, q))
+	tt.Assert.NoError(builder.Exec(tt.Ctx, q))
 	tt.Assert.NoError(q.Commit())
 
 	type hop struct {
@@ -39,6 +43,6 @@ func TestAddOperationParticipants(t *testing.T) {
 
 		op := ops[0]
 		tt.Assert.Equal(int64(240518172673), op.OperationID)
-		tt.Assert.Equal(int64(1), op.AccountID)
+		tt.Assert.Equal(accountLoader.getNow(address), op.AccountID)
 	}
 }
diff --git a/services/horizon/internal/db2/history/participants.go b/services/horizon/internal/db2/history/participants.go
index 58b0a80892..f73b5ab577 100644
--- a/services/horizon/internal/db2/history/participants.go
+++ b/services/horizon/internal/db2/history/participants.go
@@ -16,7 +16,7 @@ type QParticipants interface {
 // TransactionParticipantsBatchInsertBuilder is used to insert transaction participants into the
 // history_transaction_participants table
 type TransactionParticipantsBatchInsertBuilder interface {
-	Add(transactionID, accountID int64) error
+	Add(transactionID int64, accountID FutureAccountID) error
 	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
@@ -34,7 +34,7 @@ func (q *Q) NewTransactionParticipantsBatchInsertBuilder() TransactionParticipan
 }
 
 // Add adds a new transaction participant to the batch
-func (i *transactionParticipantsBatchInsertBuilder) Add(transactionID, accountID int64) error {
+func (i *transactionParticipantsBatchInsertBuilder) Add(transactionID int64, accountID FutureAccountID) error {
 	return i.builder.Row(map[string]interface{}{
 		"history_transaction_id": transactionID,
 		"history_account_id":     accountID,
diff --git a/services/horizon/internal/db2/history/participants_test.go b/services/horizon/internal/db2/history/participants_test.go
index 37f50706e6..a8d87976ac 100644
--- a/services/horizon/internal/db2/history/participants_test.go
+++ b/services/horizon/internal/db2/history/participants_test.go
@@ -4,6 +4,8 @@ import (
 	"testing"
 
 	sq "github.com/Masterminds/squirrel"
+
+	"github.com/stellar/go/keypair"
 	"github.com/stellar/go/services/horizon/internal/test"
 )
 
@@ -32,29 +34,36 @@ func TestTransactionParticipantsBatch(t *testing.T) {
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
 
-	tt.Assert.NoError(q.Begin())
 	batch := q.NewTransactionParticipantsBatchInsertBuilder()
+	accountLoader := NewAccountLoader()
 
 	transactionID := int64(1)
 	otherTransactionID := int64(2)
-	accountID := int64(100)
-
+	var addresses []string
 	for i := int64(0); i < 3; i++ {
-		tt.Assert.NoError(batch.Add(transactionID, accountID+i))
+		address := keypair.MustRandom().Address()
+		addresses = append(addresses, address)
+		tt.Assert.NoError(batch.Add(transactionID, accountLoader.GetFuture(address)))
 	}
 
-	tt.Assert.NoError(batch.Add(otherTransactionID, accountID))
+	address := keypair.MustRandom().Address()
+	addresses = append(addresses, address)
+	tt.Assert.NoError(batch.Add(otherTransactionID, accountLoader.GetFuture(address)))
+
+	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(accountLoader.Exec(tt.Ctx, q))
 	tt.Assert.NoError(batch.Exec(tt.Ctx, q))
 	tt.Assert.NoError(q.Commit())
 
 	participants := getTransactionParticipants(tt, q)
-	tt.Assert.Equal(
-		[]transactionParticipant{
-			{TransactionID: 1, AccountID: 100},
-			{TransactionID: 1, AccountID: 101},
-			{TransactionID: 1, AccountID: 102},
-			{TransactionID: 2, AccountID: 100},
-		},
-		participants,
-	)
+	expected := []transactionParticipant{
+		{TransactionID: 1},
+		{TransactionID: 1},
+		{TransactionID: 1},
+		{TransactionID: 2},
+	}
+	for i := range expected {
+		expected[i].AccountID = accountLoader.getNow(addresses[i])
+	}
+	tt.Assert.ElementsMatch(expected, participants)
 }
diff --git a/services/horizon/internal/ingest/main_test.go b/services/horizon/internal/ingest/main_test.go
index 6f0138ffe3..6d29c07961 100644
--- a/services/horizon/internal/ingest/main_test.go
+++ b/services/horizon/internal/ingest/main_test.go
@@ -420,7 +420,7 @@ func (m *mockDBQ) NewTransactionParticipantsBatchInsertBuilder() history.Transac
 
 func (m *mockDBQ) NewOperationParticipantBatchInsertBuilder() history.OperationParticipantBatchInsertBuilder {
 	args := m.Called()
-	return args.Get(0).(history.TransactionParticipantsBatchInsertBuilder)
+	return args.Get(0).(history.OperationParticipantBatchInsertBuilder)
 }
 
 func (m *mockDBQ) NewTradeBatchInsertBuilder() history.TradeBatchInsertBuilder {
diff --git a/services/horizon/internal/ingest/processors/participants_processor.go b/services/horizon/internal/ingest/processors/participants_processor.go
index debb812154..45ffcea524 100644
--- a/services/horizon/internal/ingest/processors/participants_processor.go
+++ b/services/horizon/internal/ingest/processors/participants_processor.go
@@ -7,7 +7,6 @@ import (
 
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
-	set "github.com/stellar/go/support/collections/set"
 	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/toid"
@@ -17,66 +16,23 @@ import (
 // ParticipantsProcessor is a processor which ingests various participants
 // from different sources (transactions, operations, etc)
 type ParticipantsProcessor struct {
-	session        db.SessionInterface
-	participantsQ  history.QParticipants
-	sequence       uint32
-	participantSet map[string]participant
+	accountLoader *history.AccountLoader
+	txBatch       history.TransactionParticipantsBatchInsertBuilder
+	opBatch       history.OperationParticipantBatchInsertBuilder
 }
 
-func NewParticipantsProcessor(session db.SessionInterface, participantsQ history.QParticipants, sequence uint32) *ParticipantsProcessor {
+func NewParticipantsProcessor(
+	accountLoader *history.AccountLoader,
+	txBatch history.TransactionParticipantsBatchInsertBuilder,
+	opBatch history.OperationParticipantBatchInsertBuilder,
+) *ParticipantsProcessor {
 	return &ParticipantsProcessor{
-		session:        session,
-		participantsQ:  participantsQ,
-		sequence:       sequence,
-		participantSet: map[string]participant{},
+		accountLoader: accountLoader,
+		txBatch:       txBatch,
+		opBatch:       opBatch,
 	}
 }
 
-type participant struct {
-	accountID      int64
-	transactionSet set.Set[int64]
-	operationSet   set.Set[int64]
-}
-
-func (p *participant) addTransactionID(id int64) {
-	if p.transactionSet == nil {
-		p.transactionSet = set.Set[int64]{}
-	}
-	p.transactionSet.Add(id)
-}
-
-func (p *participant) addOperationID(id int64) {
-	if p.operationSet == nil {
-		p.operationSet = set.Set[int64]{}
-	}
-	p.operationSet.Add(id)
-}
-
-func (p *ParticipantsProcessor) loadAccountIDs(ctx context.Context, participantSet map[string]participant) error {
-	addresses := make([]string, 0, len(participantSet))
-	for address := range participantSet {
-		addresses = append(addresses, address)
-	}
-
-	addressToID, err := p.participantsQ.CreateAccounts(ctx, addresses, maxBatchSize)
-	if err != nil {
-		return errors.Wrap(err, "Could not create account ids")
-	}
-
-	for _, address := range addresses {
-		id, ok := addressToID[address]
-		if !ok {
-			return errors.Errorf("no id found for account address %s", address)
-		}
-
-		participantForAddress := participantSet[address]
-		participantForAddress.accountID = id
-		participantSet[address] = participantForAddress
-	}
-
-	return nil
-}
-
 func participantsForChanges(
 	changes xdr.LedgerEntryChanges,
 ) ([]xdr.AccountId, error) {
@@ -144,7 +100,6 @@ func participantsForMeta(
 }
 
 func (p *ParticipantsProcessor) addTransactionParticipants(
-	participantSet map[string]participant,
 	sequence uint32,
 	transaction ingest.LedgerTransaction,
 ) error {
@@ -158,17 +113,15 @@ func (p *ParticipantsProcessor) addTransactionParticipants(
 	}
 
 	for _, participant := range transactionParticipants {
-		address := participant.Address()
-		entry := participantSet[address]
-		entry.addTransactionID(transactionID)
-		participantSet[address] = entry
+		if err := p.txBatch.Add(transactionID, p.accountLoader.GetFuture(participant.Address())); err != nil {
+			return err
+		}
 	}
 
 	return nil
 }
 
 func (p *ParticipantsProcessor) addOperationsParticipants(
-	participantSet map[string]participant,
 	sequence uint32,
 	transaction ingest.LedgerTransaction,
 ) error {
@@ -177,82 +130,39 @@ func (p *ParticipantsProcessor) addOperationsParticipants(
 		return errors.Wrap(err, "could not determine operation participants")
 	}
 
-	for operationID, p := range participants {
-		for _, participant := range p {
+	for operationID, addresses := range participants {
+		for _, participant := range addresses {
 			address := participant.Address()
-			entry := participantSet[address]
-			entry.addOperationID(operationID)
-			participantSet[address] = entry
-		}
-	}
-
-	return nil
-}
-
-func (p *ParticipantsProcessor) insertDBTransactionParticipants(ctx context.Context, participantSet map[string]participant) error {
-	batch := p.participantsQ.NewTransactionParticipantsBatchInsertBuilder()
-
-	for _, entry := range participantSet {
-		for transactionID := range entry.transactionSet {
-			if err := batch.Add(transactionID, entry.accountID); err != nil {
-				return errors.Wrap(err, "Could not insert transaction participant in db")
+			if err := p.opBatch.Add(operationID, p.accountLoader.GetFuture(address)); err != nil {
+				return err
 			}
 		}
 	}
 
-	if err := batch.Exec(ctx, p.session); err != nil {
-		return errors.Wrap(err, "Could not flush transaction participants to db")
-	}
 	return nil
 }
 
-func (p *ParticipantsProcessor) insertDBOperationsParticipants(ctx context.Context, participantSet map[string]participant) error {
-	batch := p.participantsQ.NewOperationParticipantBatchInsertBuilder()
+func (p *ParticipantsProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error {
 
-	for _, entry := range participantSet {
-		for operationID := range entry.operationSet {
-			if err := batch.Add(operationID, entry.accountID); err != nil {
-				return errors.Wrap(err, "could not insert operation participant in db")
-			}
-		}
-	}
-
-	if err := batch.Exec(ctx, p.session); err != nil {
-		return errors.Wrap(err, "could not flush operation participants to db")
-	}
-	return nil
-}
-
-func (p *ParticipantsProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) (err error) {
-	err = p.addTransactionParticipants(p.participantSet, p.sequence, transaction)
-	if err != nil {
+	if err := p.addTransactionParticipants(lcm.LedgerSequence(), transaction); err != nil {
 		return err
 	}
 
-	err = p.addOperationsParticipants(p.participantSet, p.sequence, transaction)
-	if err != nil {
+	if err := p.addOperationsParticipants(lcm.LedgerSequence(), transaction); err != nil {
 		return err
 	}
 
 	return nil
 }
 
-func (p *ParticipantsProcessor) Commit(ctx context.Context) (err error) {
-	if len(p.participantSet) > 0 {
-		if err = p.loadAccountIDs(ctx, p.participantSet); err != nil {
-			return err
-		}
-
-		if err = p.insertDBTransactionParticipants(ctx, p.participantSet); err != nil {
-			return err
-		}
-
-		if err = p.insertDBOperationsParticipants(ctx, p.participantSet); err != nil {
-			return err
-		}
+func (p *ParticipantsProcessor) Commit(ctx context.Context, session db.SessionInterface) error {
+	if err := p.txBatch.Exec(ctx, session); err != nil {
+		return errors.Wrap(err, "Could not flush transaction participants to db")
 	}
-
-	return err
+	if err := p.opBatch.Exec(ctx, session); err != nil {
+		return errors.Wrap(err, "Could not flush operation participants to db")
+	}
+	return nil
 }
 
 func ParticipantsForTransaction(
diff --git a/services/horizon/internal/ingest/processors/participants_processor_test.go b/services/horizon/internal/ingest/processors/participants_processor_test.go
index e6a8fa7b81..28bc6a126c 100644
--- a/services/horizon/internal/ingest/processors/participants_processor_test.go
+++ b/services/horizon/internal/ingest/processors/participants_processor_test.go
@@ -6,7 +6,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
 
 	"github.com/stellar/go/ingest"
@@ -22,19 +21,20 @@ type ParticipantsProcessorTestSuiteLedger struct {
 	ctx                              context.Context
 	processor                        *ParticipantsProcessor
 	mockSession                      *db.MockSession
-	mockQ                            *history.MockQParticipants
 	mockBatchInsertBuilder           *history.MockTransactionParticipantsBatchInsertBuilder
 	mockOperationsBatchInsertBuilder *history.MockOperationParticipantBatchInsertBuilder
-
-	firstTx     ingest.LedgerTransaction
-	secondTx    ingest.LedgerTransaction
-	thirdTx     ingest.LedgerTransaction
-	firstTxID   int64
-	secondTxID  int64
-	thirdTxID   int64
-	addresses   []string
-	addressToID map[string]int64
-	txs         []ingest.LedgerTransaction
+	accountLoader                    *history.AccountLoader
+
+	lcm             xdr.LedgerCloseMeta
+	firstTx         ingest.LedgerTransaction
+	secondTx        ingest.LedgerTransaction
+	thirdTx         ingest.LedgerTransaction
+	firstTxID       int64
+	secondTxID      int64
+	thirdTxID       int64
+	addresses       []string
+	addressToFuture map[string]history.FutureAccountID
+	txs             []ingest.LedgerTransaction
 }
 
 func TestParticipantsProcessorTestSuiteLedger(t *testing.T) {
@@ -43,10 +43,18 @@ func TestParticipantsProcessorTestSuiteLedger(t *testing.T) {
 
 func (s *ParticipantsProcessorTestSuiteLedger) SetupTest() {
 	s.ctx = context.Background()
-	s.mockQ = &history.MockQParticipants{}
 	s.mockBatchInsertBuilder = &history.MockTransactionParticipantsBatchInsertBuilder{}
 	s.mockOperationsBatchInsertBuilder = &history.MockOperationParticipantBatchInsertBuilder{}
 	sequence := uint32(20)
+	s.lcm = xdr.LedgerCloseMeta{
+		V0: &xdr.LedgerCloseMetaV0{
+			LedgerHeader: xdr.LedgerHeaderHistoryEntry{
+				Header: xdr.LedgerHeader{
+					LedgerSeq: xdr.Uint32(sequence),
+				},
+			},
+		},
+	}
 
 	s.addresses = []string{
 		"GA5WBPYA5Y4WAEHXWR2UKO2UO4BUGHUQ74EUPKON2QHV4WRHOIRNKKH2",
@@ -78,16 +86,16 @@ func (s *ParticipantsProcessorTestSuiteLedger) SetupTest() {
 	s.thirdTx.Envelope.V1.Tx.SourceAccount = aid.ToMuxedAccount()
 	s.thirdTxID = toid.New(int32(sequence), 3, 0).ToInt64()
 
-	s.addressToID = map[string]int64{
-		s.addresses[0]: 2,
-		s.addresses[1]: 20,
-		s.addresses[2]: 200,
+	s.accountLoader = history.NewAccountLoader()
+	s.addressToFuture = map[string]history.FutureAccountID{}
+	for _, address := range s.addresses {
+		s.addressToFuture[address] = s.accountLoader.GetFuture(address)
 	}
 
 	s.processor = NewParticipantsProcessor(
-		s.mockSession,
-		s.mockQ,
-		sequence,
+		s.accountLoader,
+		s.mockBatchInsertBuilder,
+		s.mockOperationsBatchInsertBuilder,
 	)
 
 	s.txs = []ingest.LedgerTransaction{
@@ -98,44 +106,46 @@ func (s *ParticipantsProcessorTestSuiteLedger) SetupTest() {
 }
 
 func (s *ParticipantsProcessorTestSuiteLedger) TearDownTest() {
-	s.mockQ.AssertExpectations(s.T())
 	s.mockBatchInsertBuilder.AssertExpectations(s.T())
 	s.mockOperationsBatchInsertBuilder.AssertExpectations(s.T())
 }
 
 func (s *ParticipantsProcessorTestSuiteLedger) mockSuccessfulTransactionBatchAdds() {
 	s.mockBatchInsertBuilder.On(
-		"Add", s.firstTxID, s.addressToID[s.addresses[0]],
+		"Add", s.firstTxID, s.addressToFuture[s.addresses[0]],
 	).Return(nil).Once()
 
 	s.mockBatchInsertBuilder.On(
-		"Add", s.secondTxID, s.addressToID[s.addresses[1]],
+		"Add", s.secondTxID, s.addressToFuture[s.addresses[1]],
 	).Return(nil).Once()
 	s.mockBatchInsertBuilder.On(
-		"Add", s.secondTxID, s.addressToID[s.addresses[2]],
+		"Add", s.secondTxID, s.addressToFuture[s.addresses[2]],
 	).Return(nil).Once()
 
 	s.mockBatchInsertBuilder.On(
-		"Add", s.thirdTxID, s.addressToID[s.addresses[0]],
+		"Add", s.thirdTxID, s.addressToFuture[s.addresses[0]],
 	).Return(nil).Once()
 }
 
 func (s *ParticipantsProcessorTestSuiteLedger) mockSuccessfulOperationBatchAdds() {
 	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.firstTxID+1, s.addressToID[s.addresses[0]],
+		"Add", s.firstTxID+1, s.addressToFuture[s.addresses[0]],
 	).Return(nil).Once()
 	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.secondTxID+1, s.addressToID[s.addresses[1]],
+		"Add", s.secondTxID+1, s.addressToFuture[s.addresses[1]],
 	).Return(nil).Once()
 	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.secondTxID+1, s.addressToID[s.addresses[2]],
+		"Add", s.secondTxID+1, s.addressToFuture[s.addresses[2]],
 	).Return(nil).Once()
 	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.thirdTxID+1, s.addressToID[s.addresses[0]],
+		"Add", s.thirdTxID+1, s.addressToFuture[s.addresses[0]],
 	).Return(nil).Once()
 }
 func (s *ParticipantsProcessorTestSuiteLedger) TestEmptyParticipants() {
-	err := s.processor.Commit(s.ctx)
+	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
+	s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
+
+	err := s.processor.Commit(s.ctx, s.mockSession)
 	s.Assert().NoError(err)
 }
 
@@ -169,52 +179,21 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestFeeBumptransaction() {
 	feeBumpTx.Result.Result.Result.Results = nil
 	feeBumpTxID := toid.New(20, 1, 0).ToInt64()
 
-	addresses := s.addresses[:2]
-	addressToID := map[string]int64{
-		addresses[0]: s.addressToID[addresses[0]],
-		addresses[1]: s.addressToID[addresses[1]],
-	}
-	s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]string)
-			s.Assert().ElementsMatch(
-				addresses,
-				arg,
-			)
-		}).Return(addressToID, nil).Once()
-	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder").
-		Return(s.mockBatchInsertBuilder).Once()
-	s.mockQ.On("NewOperationParticipantBatchInsertBuilder").
-		Return(s.mockOperationsBatchInsertBuilder).Once()
-
 	s.mockBatchInsertBuilder.On(
-		"Add", feeBumpTxID, addressToID[addresses[0]],
+		"Add", feeBumpTxID, s.addressToFuture[s.addresses[0]],
 	).Return(nil).Once()
 	s.mockBatchInsertBuilder.On(
-		"Add", feeBumpTxID, addressToID[addresses[1]],
+		"Add", feeBumpTxID, s.addressToFuture[s.addresses[1]],
 	).Return(nil).Once()
 
 	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 	s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
-	s.Assert().NoError(s.processor.ProcessTransaction(s.ctx, feeBumpTx))
-	s.Assert().NoError(s.processor.Commit(s.ctx))
+	s.Assert().NoError(s.processor.ProcessTransaction(s.lcm, feeBumpTx))
+	s.Assert().NoError(s.processor.Commit(s.ctx, s.mockSession))
 }
 
 func (s *ParticipantsProcessorTestSuiteLedger) TestIngestParticipantsSucceeds() {
-	s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]string)
-			s.Assert().ElementsMatch(
-				s.addresses,
-				arg,
-			)
-		}).Return(s.addressToID, nil).Once()
-	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder").
-		Return(s.mockBatchInsertBuilder).Once()
-	s.mockQ.On("NewOperationParticipantBatchInsertBuilder").
-		Return(s.mockOperationsBatchInsertBuilder).Once()
-
 	s.mockSuccessfulTransactionBatchAdds()
 	s.mockSuccessfulOperationBatchAdds()
 
@@ -222,135 +201,50 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestIngestParticipantsSucceeds()
 	s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(s.ctx, tx)
+		err := s.processor.ProcessTransaction(s.lcm, tx)
 		s.Assert().NoError(err)
 	}
-	err := s.processor.Commit(s.ctx)
+	err := s.processor.Commit(s.ctx, s.mockSession)
 	s.Assert().NoError(err)
 }
 
-func (s *ParticipantsProcessorTestSuiteLedger) TestCreateAccountsFails() {
-	s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Return(s.addressToID, errors.New("transient error")).Once()
-	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(s.ctx, tx)
-		s.Assert().NoError(err)
-	}
-	err := s.processor.Commit(s.ctx)
-	s.Assert().EqualError(err, "Could not create account ids: transient error")
-}
-
 func (s *ParticipantsProcessorTestSuiteLedger) TestBatchAddFails() {
-	s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]string)
-			s.Assert().ElementsMatch(
-				s.addresses,
-				arg,
-			)
-		}).Return(s.addressToID, nil).Once()
-	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder").
-		Return(s.mockBatchInsertBuilder).Once()
-
 	s.mockBatchInsertBuilder.On(
-		"Add", s.firstTxID, s.addressToID[s.addresses[0]],
+		"Add", s.firstTxID, s.addressToFuture[s.addresses[0]],
 	).Return(errors.New("transient error")).Once()
 
-	s.mockBatchInsertBuilder.On(
-		"Add", s.secondTxID, s.addressToID[s.addresses[1]],
-	).Return(nil).Maybe()
-	s.mockBatchInsertBuilder.On(
-		"Add", s.secondTxID, s.addressToID[s.addresses[2]],
-	).Return(nil).Maybe()
-
-	s.mockBatchInsertBuilder.On(
-		"Add", s.thirdTxID, s.addressToID[s.addresses[0]],
-	).Return(nil).Maybe()
-	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(s.ctx, tx)
-		s.Assert().NoError(err)
-	}
-	err := s.processor.Commit(s.ctx)
-	s.Assert().EqualError(err, "Could not insert transaction participant in db: transient error")
+	err := s.processor.ProcessTransaction(s.lcm, s.txs[0])
+	s.Assert().EqualError(err, "transient error")
 }
 
 func (s *ParticipantsProcessorTestSuiteLedger) TestOperationParticipantsBatchAddFails() {
-	s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]string)
-			s.Assert().ElementsMatch(
-				s.addresses,
-				arg,
-			)
-		}).Return(s.addressToID, nil).Once()
-	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder").
-		Return(s.mockBatchInsertBuilder).Once()
-	s.mockQ.On("NewOperationParticipantBatchInsertBuilder").
-		Return(s.mockOperationsBatchInsertBuilder).Once()
-
-	s.mockSuccessfulTransactionBatchAdds()
+	s.mockBatchInsertBuilder.On(
+		"Add", s.firstTxID, s.addressToFuture[s.addresses[0]],
+	).Return(nil).Once()
 
 	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.firstTxID+1, s.addressToID[s.addresses[0]],
+		"Add", s.firstTxID+1, s.addressToFuture[s.addresses[0]],
 	).Return(errors.New("transient error")).Once()
-	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.secondTxID+1, s.addressToID[s.addresses[1]],
-	).Return(nil).Maybe()
-	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.secondTxID+1, s.addressToID[s.addresses[2]],
-	).Return(nil).Maybe()
-	s.mockOperationsBatchInsertBuilder.On(
-		"Add", s.thirdTxID+1, s.addressToID[s.addresses[0]],
-	).Return(nil).Maybe()
-
-	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
-	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(s.ctx, tx)
-		s.Assert().NoError(err)
-	}
-	err := s.processor.Commit(s.ctx)
-	s.Assert().EqualError(err, "could not insert operation participant in db: transient error")
+	err := s.processor.ProcessTransaction(s.lcm, s.txs[0])
+	s.Assert().EqualError(err, "transient error")
 }
 
 func (s *ParticipantsProcessorTestSuiteLedger) TestBatchAddExecFails() {
-	s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]string)
-			s.Assert().ElementsMatch(
-				s.addresses,
-				arg,
-			)
-		}).Return(s.addressToID, nil).Once()
-	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder").
-		Return(s.mockBatchInsertBuilder).Once()
-
 	s.mockSuccessfulTransactionBatchAdds()
+	s.mockSuccessfulOperationBatchAdds()
 
 	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(errors.New("transient error")).Once()
 
 	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(s.ctx, tx)
+		err := s.processor.ProcessTransaction(s.lcm, tx)
 		s.Assert().NoError(err)
 	}
-	err := s.processor.Commit(s.ctx)
+	err := s.processor.Commit(s.ctx, s.mockSession)
 	s.Assert().EqualError(err, "Could not flush transaction participants to db: transient error")
 }
 
-func (s *ParticipantsProcessorTestSuiteLedger) TestOpeartionBatchAddExecFails() {
-	s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]string)
-			s.Assert().ElementsMatch(
-				s.addresses,
-				arg,
-			)
-		}).Return(s.addressToID, nil).Once()
-	s.mockQ.On("NewTransactionParticipantsBatchInsertBuilder").
-		Return(s.mockBatchInsertBuilder).Once()
-	s.mockQ.On("NewOperationParticipantBatchInsertBuilder").
-		Return(s.mockOperationsBatchInsertBuilder).Once()
-
+func (s *ParticipantsProcessorTestSuiteLedger) TestOperationBatchAddExecFails() {
 	s.mockSuccessfulTransactionBatchAdds()
 	s.mockSuccessfulOperationBatchAdds()
 
@@ -358,9 +252,9 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestOpeartionBatchAddExecFails()
 	s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(errors.New("transient error")).Once()
 
 	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(s.ctx, tx)
+		err := s.processor.ProcessTransaction(s.lcm, tx)
 		s.Assert().NoError(err)
 	}
-	err := s.processor.Commit(s.ctx)
-	s.Assert().EqualError(err, "could not flush operation participants to db: transient error")
+	err := s.processor.Commit(s.ctx, s.mockSession)
+	s.Assert().EqualError(err, "Could not flush operation participants to db: transient error")
 }

From 21d016f3f7b01ca1e2f341417762b18c1a059e22 Mon Sep 17 00:00:00 2001
From: tamirms <tamirms@gmail.com>
Date: Fri, 25 Aug 2023 19:31:50 +0100
Subject: [PATCH 11/17] services/horizon/internal/ingest/processors: Refactor
 liquidity pools, trades, and claimable balances processors to support new
 ingestion data flow (#5025)

---
 .../internal/db2/history/account_loader.go    |  27 +-
 .../db2/history/account_loader_test.go        |   6 +-
 .../internal/db2/history/asset_loader.go      |  37 ++-
 .../internal/db2/history/asset_loader_test.go |   6 +-
 .../history/claimable_balance_loader_test.go  |   2 +
 .../internal/db2/history/effect_test.go       |  14 +-
 .../db2/history/history_claimable_balances.go |  12 +-
 .../db2/history/history_liquidity_pools.go    |  13 +-
 .../db2/history/liquidity_pool_loader.go      |  27 +-
 .../db2/history/liquidity_pool_loader_test.go |   6 +-
 .../mock_q_history_claimable_balances.go      |   8 +-
 .../history/mock_q_history_liquidity_pools.go |   8 +-
 ...n_participant_batch_insert_builder_test.go |   2 +-
 .../internal/db2/history/operation_test.go    |  18 +-
 .../internal/db2/history/participants_test.go |   2 +-
 .../internal/db2/history/transaction_test.go  |  20 +-
 ...laimable_balances_transaction_processor.go | 189 +++--------
 ...ble_balances_transaction_processor_test.go |  89 +++--
 .../liquidity_pools_transaction_processor.go  | 182 +++-------
 ...uidity_pools_transaction_processor_test.go |  82 +++--
 .../processors/participants_processor.go      |   2 +-
 .../processors/participants_processor_test.go |  10 +-
 .../ingest/processors/trades_processor.go     | 102 +++---
 .../processors/trades_processor_test.go       | 311 +++++-------------
 24 files changed, 420 insertions(+), 755 deletions(-)

diff --git a/services/horizon/internal/db2/history/account_loader.go b/services/horizon/internal/db2/history/account_loader.go
index 14bcfc5243..e9fd9bedea 100644
--- a/services/horizon/internal/db2/history/account_loader.go
+++ b/services/horizon/internal/db2/history/account_loader.go
@@ -28,7 +28,7 @@ const loaderLookupBatchSize = 50000
 
 // Value implements the database/sql/driver Valuer interface.
 func (a FutureAccountID) Value() (driver.Value, error) {
-	return a.loader.getNow(a.address), nil
+	return a.loader.GetNow(a.address), nil
 }
 
 // AccountLoader will map account addresses to their history
@@ -67,11 +67,11 @@ func (a *AccountLoader) GetFuture(address string) FutureAccountID {
 	}
 }
 
-// getNow returns the history account id for the given address.
-// getNow should only be called on values which were registered by
-// GetFuture() calls. Also, Exec() must be called before any getNow
+// GetNow returns the history account id for the given address.
+// GetNow should only be called on values which were registered by
+// GetFuture() calls. Also, Exec() must be called before any GetNow
 // call can succeed.
-func (a *AccountLoader) getNow(address string) int64 {
+func (a *AccountLoader) GetNow(address string) int64 {
 	if id, ok := a.ids[address]; !ok {
 		panic(fmt.Errorf("address %v not present", address))
 	} else {
@@ -190,3 +190,20 @@ func bulkInsert(ctx context.Context, q *Q, table string, conflictFields []string
 	)
 	return err
 }
+
+// AccountLoaderStub is a stub wrapper around AccountLoader which allows
+// you to manually configure the mapping of addresses to history account ids
+type AccountLoaderStub struct {
+	Loader *AccountLoader
+}
+
+// NewAccountLoaderStub returns a new AccountLoaderStub instance
+func NewAccountLoaderStub() AccountLoaderStub {
+	return AccountLoaderStub{Loader: NewAccountLoader()}
+}
+
+// Insert updates the wrapped AccountLoader so that the given account
+// address is mapped to the provided history account id
+func (a AccountLoaderStub) Insert(address string, id int64) {
+	a.Loader.ids[address] = id
+}
diff --git a/services/horizon/internal/db2/history/account_loader_test.go b/services/horizon/internal/db2/history/account_loader_test.go
index 14d933c0ad..11047f3be2 100644
--- a/services/horizon/internal/db2/history/account_loader_test.go
+++ b/services/horizon/internal/db2/history/account_loader_test.go
@@ -27,11 +27,13 @@ func TestAccountLoader(t *testing.T) {
 		future := loader.GetFuture(address)
 		futures = append(futures, future)
 		assert.Panics(t, func() {
-			loader.getNow(address)
+			loader.GetNow(address)
 		})
 		assert.Panics(t, func() {
 			future.Value()
 		})
+		duplicateFuture := loader.GetFuture(address)
+		assert.Equal(t, future, duplicateFuture)
 	}
 
 	assert.NoError(t, loader.Exec(context.Background(), session))
@@ -42,7 +44,7 @@ func TestAccountLoader(t *testing.T) {
 	q := &Q{session}
 	for i, address := range addresses {
 		future := futures[i]
-		id := loader.getNow(address)
+		id := loader.GetNow(address)
 		val, err := future.Value()
 		assert.NoError(t, err)
 		assert.Equal(t, id, val)
diff --git a/services/horizon/internal/db2/history/asset_loader.go b/services/horizon/internal/db2/history/asset_loader.go
index 326f1b68ba..6ef3d7a350 100644
--- a/services/horizon/internal/db2/history/asset_loader.go
+++ b/services/horizon/internal/db2/history/asset_loader.go
@@ -12,6 +12,7 @@ import (
 	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/support/ordered"
+	"github.com/stellar/go/xdr"
 )
 
 type AssetKey struct {
@@ -20,6 +21,15 @@ type AssetKey struct {
 	Issuer string
 }
 
+// AssetKeyFromXDR constructs an AssetKey from an xdr asset
+func AssetKeyFromXDR(asset xdr.Asset) AssetKey {
+	return AssetKey{
+		Type:   xdr.AssetTypeToString[asset.Type],
+		Code:   asset.GetCode(),
+		Issuer: asset.GetIssuer(),
+	}
+}
+
 // FutureAssetID represents a future history asset.
 // A FutureAssetID is created by an AssetLoader and
 // the asset id is available after calling Exec() on
@@ -31,7 +41,7 @@ type FutureAssetID struct {
 
 // Value implements the database/sql/driver Valuer interface.
 func (a FutureAssetID) Value() (driver.Value, error) {
-	return a.loader.getNow(a.asset), nil
+	return a.loader.GetNow(a.asset), nil
 }
 
 // AssetLoader will map assets to their history
@@ -67,11 +77,11 @@ func (a *AssetLoader) GetFuture(asset AssetKey) FutureAssetID {
 	}
 }
 
-// getNow returns the history asset id for the given asset.
-// getNow should only be called on values which were registered by
-// GetFuture() calls. Also, Exec() must be called before any getNow
+// GetNow returns the history asset id for the given asset.
+// GetNow should only be called on values which were registered by
+// GetFuture() calls. Also, Exec() must be called before any GetNow
 // call can succeed.
-func (a *AssetLoader) getNow(asset AssetKey) int64 {
+func (a *AssetLoader) GetNow(asset AssetKey) int64 {
 	if id, ok := a.ids[asset]; !ok {
 		panic(fmt.Errorf("asset %v not present", asset))
 	} else {
@@ -186,3 +196,20 @@ func (a *AssetLoader) Exec(ctx context.Context, session db.SessionInterface) err
 
 	return a.lookupKeys(ctx, q, keys)
 }
+
+// AssetLoaderStub is a stub wrapper around AssetLoader which allows
+// you to manually configure the mapping of assets to history asset ids
+type AssetLoaderStub struct {
+	Loader *AssetLoader
+}
+
+// NewAssetLoaderStub returns a new AssetLoaderStub instance
+func NewAssetLoaderStub() AssetLoaderStub {
+	return AssetLoaderStub{Loader: NewAssetLoader()}
+}
+
+// Insert updates the wrapped AssetLoaderStub so that the given asset
+// address is mapped to the provided history asset id
+func (a AssetLoaderStub) Insert(asset AssetKey, id int64) {
+	a.Loader.ids[asset] = id
+}
diff --git a/services/horizon/internal/db2/history/asset_loader_test.go b/services/horizon/internal/db2/history/asset_loader_test.go
index f51b2e27ef..99f510266c 100644
--- a/services/horizon/internal/db2/history/asset_loader_test.go
+++ b/services/horizon/internal/db2/history/asset_loader_test.go
@@ -41,11 +41,13 @@ func TestAssetLoader(t *testing.T) {
 		future := loader.GetFuture(key)
 		futures = append(futures, future)
 		assert.Panics(t, func() {
-			loader.getNow(key)
+			loader.GetNow(key)
 		})
 		assert.Panics(t, func() {
 			future.Value()
 		})
+		duplicateFuture := loader.GetFuture(key)
+		assert.Equal(t, future, duplicateFuture)
 	}
 
 	assert.NoError(t, loader.Exec(context.Background(), session))
@@ -56,7 +58,7 @@ func TestAssetLoader(t *testing.T) {
 	q := &Q{session}
 	for i, key := range keys {
 		future := futures[i]
-		internalID := loader.getNow(key)
+		internalID := loader.GetNow(key)
 		val, err := future.Value()
 		assert.NoError(t, err)
 		assert.Equal(t, internalID, val)
diff --git a/services/horizon/internal/db2/history/claimable_balance_loader_test.go b/services/horizon/internal/db2/history/claimable_balance_loader_test.go
index 183bdb3daa..b119daa674 100644
--- a/services/horizon/internal/db2/history/claimable_balance_loader_test.go
+++ b/services/horizon/internal/db2/history/claimable_balance_loader_test.go
@@ -38,6 +38,8 @@ func TestClaimableBalanceLoader(t *testing.T) {
 		assert.Panics(t, func() {
 			future.Value()
 		})
+		duplicateFuture := loader.GetFuture(id)
+		assert.Equal(t, future, duplicateFuture)
 	}
 
 	assert.NoError(t, loader.Exec(context.Background(), session))
diff --git a/services/horizon/internal/db2/history/effect_test.go b/services/horizon/internal/db2/history/effect_test.go
index 06cfc2adcb..bf893a100b 100644
--- a/services/horizon/internal/db2/history/effect_test.go
+++ b/services/horizon/internal/db2/history/effect_test.go
@@ -6,6 +6,7 @@ import (
 	"testing"
 
 	"github.com/guregu/null"
+
 	"github.com/stellar/go/protocols/horizon/effects"
 	"github.com/stellar/go/services/horizon/internal/db2"
 	"github.com/stellar/go/services/horizon/internal/test"
@@ -47,17 +48,12 @@ func TestEffectsForLiquidityPool(t *testing.T) {
 
 	// Insert Liquidity Pool history
 	liquidityPoolID := "abcde"
-	toInternalID, err := q.CreateHistoryLiquidityPools(tt.Ctx, []string{liquidityPoolID}, 2)
-	tt.Assert.NoError(err)
+	lpLoader := NewLiquidityPoolLoader()
 
 	operationBuilder := q.NewOperationLiquidityPoolBatchInsertBuilder()
-	tt.Assert.NoError(err)
-	internalID, ok := toInternalID[liquidityPoolID]
-	tt.Assert.True(ok)
-	err = operationBuilder.Add(opID, internalID)
-	tt.Assert.NoError(err)
-	err = operationBuilder.Exec(tt.Ctx, q)
-	tt.Assert.NoError(err)
+	tt.Assert.NoError(operationBuilder.Add(opID, lpLoader.GetFuture(liquidityPoolID)))
+	tt.Assert.NoError(lpLoader.Exec(tt.Ctx, q))
+	tt.Assert.NoError(operationBuilder.Exec(tt.Ctx, q))
 
 	tt.Assert.NoError(q.Commit())
 
diff --git a/services/horizon/internal/db2/history/history_claimable_balances.go b/services/horizon/internal/db2/history/history_claimable_balances.go
index 162c31a70b..5d2076f3fd 100644
--- a/services/horizon/internal/db2/history/history_claimable_balances.go
+++ b/services/horizon/internal/db2/history/history_claimable_balances.go
@@ -92,7 +92,7 @@ func (q *Q) ClaimableBalanceByID(ctx context.Context, id string) (dest HistoryCl
 }
 
 type OperationClaimableBalanceBatchInsertBuilder interface {
-	Add(operationID, internalID int64) error
+	Add(operationID int64, claimableBalance FutureClaimableBalanceID) error
 	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
@@ -109,10 +109,10 @@ func (q *Q) NewOperationClaimableBalanceBatchInsertBuilder() OperationClaimableB
 }
 
 // Add adds a new operation claimable balance to the batch
-func (i *operationClaimableBalanceBatchInsertBuilder) Add(operationID, internalID int64) error {
+func (i *operationClaimableBalanceBatchInsertBuilder) Add(operationID int64, claimableBalance FutureClaimableBalanceID) error {
 	return i.builder.Row(map[string]interface{}{
 		"history_operation_id":         operationID,
-		"history_claimable_balance_id": internalID,
+		"history_claimable_balance_id": claimableBalance,
 	})
 }
 
@@ -122,7 +122,7 @@ func (i *operationClaimableBalanceBatchInsertBuilder) Exec(ctx context.Context,
 }
 
 type TransactionClaimableBalanceBatchInsertBuilder interface {
-	Add(transactionID, internalID int64) error
+	Add(transactionID int64, claimableBalance FutureClaimableBalanceID) error
 	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
@@ -139,10 +139,10 @@ func (q *Q) NewTransactionClaimableBalanceBatchInsertBuilder() TransactionClaima
 }
 
 // Add adds a new transaction claimable balance to the batch
-func (i *transactionClaimableBalanceBatchInsertBuilder) Add(transactionID, internalID int64) error {
+func (i *transactionClaimableBalanceBatchInsertBuilder) Add(transactionID int64, claimableBalance FutureClaimableBalanceID) error {
 	return i.builder.Row(map[string]interface{}{
 		"history_transaction_id":       transactionID,
-		"history_claimable_balance_id": internalID,
+		"history_claimable_balance_id": claimableBalance,
 	})
 }
 
diff --git a/services/horizon/internal/db2/history/history_liquidity_pools.go b/services/horizon/internal/db2/history/history_liquidity_pools.go
index 3953280aa9..bb13ac59f9 100644
--- a/services/horizon/internal/db2/history/history_liquidity_pools.go
+++ b/services/horizon/internal/db2/history/history_liquidity_pools.go
@@ -5,6 +5,7 @@ import (
 	"sort"
 
 	sq "github.com/Masterminds/squirrel"
+
 	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 )
@@ -101,7 +102,7 @@ func (q *Q) LiquidityPoolByID(ctx context.Context, poolID string) (dest HistoryL
 }
 
 type OperationLiquidityPoolBatchInsertBuilder interface {
-	Add(operationID, internalID int64) error
+	Add(operationID int64, lp FutureLiquidityPoolID) error
 	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
@@ -118,10 +119,10 @@ func (q *Q) NewOperationLiquidityPoolBatchInsertBuilder() OperationLiquidityPool
 }
 
 // Add adds a new operation claimable balance to the batch
-func (i *operationLiquidityPoolBatchInsertBuilder) Add(operationID, internalID int64) error {
+func (i *operationLiquidityPoolBatchInsertBuilder) Add(operationID int64, lp FutureLiquidityPoolID) error {
 	return i.builder.Row(map[string]interface{}{
 		"history_operation_id":      operationID,
-		"history_liquidity_pool_id": internalID,
+		"history_liquidity_pool_id": lp,
 	})
 }
 
@@ -131,7 +132,7 @@ func (i *operationLiquidityPoolBatchInsertBuilder) Exec(ctx context.Context, ses
 }
 
 type TransactionLiquidityPoolBatchInsertBuilder interface {
-	Add(transactionID, internalID int64) error
+	Add(transactionID int64, lp FutureLiquidityPoolID) error
 	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
@@ -148,10 +149,10 @@ func (q *Q) NewTransactionLiquidityPoolBatchInsertBuilder() TransactionLiquidity
 }
 
 // Add adds a new transaction claimable balance to the batch
-func (i *transactionLiquidityPoolBatchInsertBuilder) Add(transactionID, internalID int64) error {
+func (i *transactionLiquidityPoolBatchInsertBuilder) Add(transactionID int64, lp FutureLiquidityPoolID) error {
 	return i.builder.Row(map[string]interface{}{
 		"history_transaction_id":    transactionID,
-		"history_liquidity_pool_id": internalID,
+		"history_liquidity_pool_id": lp,
 	})
 }
 
diff --git a/services/horizon/internal/db2/history/liquidity_pool_loader.go b/services/horizon/internal/db2/history/liquidity_pool_loader.go
index ac501dcfd3..7c2fe6fd4d 100644
--- a/services/horizon/internal/db2/history/liquidity_pool_loader.go
+++ b/services/horizon/internal/db2/history/liquidity_pool_loader.go
@@ -23,7 +23,7 @@ type FutureLiquidityPoolID struct {
 
 // Value implements the database/sql/driver Valuer interface.
 func (a FutureLiquidityPoolID) Value() (driver.Value, error) {
-	return a.loader.getNow(a.id), nil
+	return a.loader.GetNow(a.id), nil
 }
 
 // LiquidityPoolLoader will map liquidity pools to their internal
@@ -60,11 +60,11 @@ func (a *LiquidityPoolLoader) GetFuture(id string) FutureLiquidityPoolID {
 	}
 }
 
-// getNow returns the internal history id for the given liquidity pool.
-// getNow should only be called on values which were registered by
-// GetFuture() calls. Also, Exec() must be called before any getNow
+// GetNow returns the internal history id for the given liquidity pool.
+// GetNow should only be called on values which were registered by
+// GetFuture() calls. Also, Exec() must be called before any GetNow
 // call can succeed.
-func (a *LiquidityPoolLoader) getNow(id string) int64 {
+func (a *LiquidityPoolLoader) GetNow(id string) int64 {
 	if id, ok := a.ids[id]; !ok {
 		panic(fmt.Errorf("id %v not present", id))
 	} else {
@@ -141,3 +141,20 @@ func (a *LiquidityPoolLoader) Exec(ctx context.Context, session db.SessionInterf
 
 	return a.lookupKeys(ctx, q, ids)
 }
+
+// LiquidityPoolLoaderStub is a stub wrapper around LiquidityPoolLoader which allows
+// you to manually configure the mapping of liquidity pools to history liquidity ppol ids
+type LiquidityPoolLoaderStub struct {
+	Loader *LiquidityPoolLoader
+}
+
+// NewLiquidityPoolLoaderStub returns a new LiquidityPoolLoader instance
+func NewLiquidityPoolLoaderStub() LiquidityPoolLoaderStub {
+	return LiquidityPoolLoaderStub{Loader: NewLiquidityPoolLoader()}
+}
+
+// Insert updates the wrapped LiquidityPoolLoader so that the given liquidity pool
+// is mapped to the provided history liquidity pool id
+func (a LiquidityPoolLoaderStub) Insert(lp string, id int64) {
+	a.Loader.ids[lp] = id
+}
diff --git a/services/horizon/internal/db2/history/liquidity_pool_loader_test.go b/services/horizon/internal/db2/history/liquidity_pool_loader_test.go
index 00664ff3e3..e2b1e05beb 100644
--- a/services/horizon/internal/db2/history/liquidity_pool_loader_test.go
+++ b/services/horizon/internal/db2/history/liquidity_pool_loader_test.go
@@ -30,11 +30,13 @@ func TestLiquidityPoolLoader(t *testing.T) {
 		future := loader.GetFuture(id)
 		futures = append(futures, future)
 		assert.Panics(t, func() {
-			loader.getNow(id)
+			loader.GetNow(id)
 		})
 		assert.Panics(t, func() {
 			future.Value()
 		})
+		duplicateFuture := loader.GetFuture(id)
+		assert.Equal(t, future, duplicateFuture)
 	}
 
 	assert.NoError(t, loader.Exec(context.Background(), session))
@@ -45,7 +47,7 @@ func TestLiquidityPoolLoader(t *testing.T) {
 	q := &Q{session}
 	for i, id := range ids {
 		future := futures[i]
-		internalID := loader.getNow(id)
+		internalID := loader.GetNow(id)
 		val, err := future.Value()
 		assert.NoError(t, err)
 		assert.Equal(t, internalID, val)
diff --git a/services/horizon/internal/db2/history/mock_q_history_claimable_balances.go b/services/horizon/internal/db2/history/mock_q_history_claimable_balances.go
index 6ce3926c91..6607456af2 100644
--- a/services/horizon/internal/db2/history/mock_q_history_claimable_balances.go
+++ b/services/horizon/internal/db2/history/mock_q_history_claimable_balances.go
@@ -29,8 +29,8 @@ type MockTransactionClaimableBalanceBatchInsertBuilder struct {
 	mock.Mock
 }
 
-func (m *MockTransactionClaimableBalanceBatchInsertBuilder) Add(transactionID, accountID int64) error {
-	a := m.Called(transactionID, accountID)
+func (m *MockTransactionClaimableBalanceBatchInsertBuilder) Add(transactionID int64, claimableBalance FutureClaimableBalanceID) error {
+	a := m.Called(transactionID, claimableBalance)
 	return a.Error(0)
 }
 
@@ -51,8 +51,8 @@ type MockOperationClaimableBalanceBatchInsertBuilder struct {
 	mock.Mock
 }
 
-func (m *MockOperationClaimableBalanceBatchInsertBuilder) Add(transactionID, accountID int64) error {
-	a := m.Called(transactionID, accountID)
+func (m *MockOperationClaimableBalanceBatchInsertBuilder) Add(operationID int64, claimableBalance FutureClaimableBalanceID) error {
+	a := m.Called(operationID, claimableBalance)
 	return a.Error(0)
 }
 
diff --git a/services/horizon/internal/db2/history/mock_q_history_liquidity_pools.go b/services/horizon/internal/db2/history/mock_q_history_liquidity_pools.go
index 33f5c8a46b..bf000a22e9 100644
--- a/services/horizon/internal/db2/history/mock_q_history_liquidity_pools.go
+++ b/services/horizon/internal/db2/history/mock_q_history_liquidity_pools.go
@@ -29,8 +29,8 @@ type MockTransactionLiquidityPoolBatchInsertBuilder struct {
 	mock.Mock
 }
 
-func (m *MockTransactionLiquidityPoolBatchInsertBuilder) Add(transactionID, accountID int64) error {
-	a := m.Called(transactionID, accountID)
+func (m *MockTransactionLiquidityPoolBatchInsertBuilder) Add(transactionID int64, lp FutureLiquidityPoolID) error {
+	a := m.Called(transactionID, lp)
 	return a.Error(0)
 }
 
@@ -51,8 +51,8 @@ type MockOperationLiquidityPoolBatchInsertBuilder struct {
 	mock.Mock
 }
 
-func (m *MockOperationLiquidityPoolBatchInsertBuilder) Add(transactionID, accountID int64) error {
-	a := m.Called(transactionID, accountID)
+func (m *MockOperationLiquidityPoolBatchInsertBuilder) Add(operationID int64, lp FutureLiquidityPoolID) error {
+	a := m.Called(operationID, lp)
 	return a.Error(0)
 }
 
diff --git a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
index 1bcd64cceb..fc2ca9c831 100644
--- a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
+++ b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
@@ -43,6 +43,6 @@ func TestAddOperationParticipants(t *testing.T) {
 
 		op := ops[0]
 		tt.Assert.Equal(int64(240518172673), op.OperationID)
-		tt.Assert.Equal(accountLoader.getNow(address), op.AccountID)
+		tt.Assert.Equal(accountLoader.GetNow(address), op.AccountID)
 	}
 }
diff --git a/services/horizon/internal/db2/history/operation_test.go b/services/horizon/internal/db2/history/operation_test.go
index a996ac49b7..40c6e5262e 100644
--- a/services/horizon/internal/db2/history/operation_test.go
+++ b/services/horizon/internal/db2/history/operation_test.go
@@ -5,6 +5,7 @@ import (
 
 	sq "github.com/Masterminds/squirrel"
 	"github.com/guregu/null"
+
 	"github.com/stellar/go/services/horizon/internal/db2"
 	"github.com/stellar/go/services/horizon/internal/test"
 	"github.com/stellar/go/toid"
@@ -122,18 +123,13 @@ func TestOperationByLiquidityPool(t *testing.T) {
 
 	// Insert Liquidity Pool history
 	liquidityPoolID := "a2f38836a839de008cf1d782c81f45e1253cc5d3dad9110b872965484fec0a49"
-	toInternalID, err := q.CreateHistoryLiquidityPools(tt.Ctx, []string{liquidityPoolID}, 2)
-	tt.Assert.NoError(err)
+	lpLoader := NewLiquidityPoolLoader()
+
 	lpOperationBuilder := q.NewOperationLiquidityPoolBatchInsertBuilder()
-	tt.Assert.NoError(err)
-	internalID, ok := toInternalID[liquidityPoolID]
-	tt.Assert.True(ok)
-	err = lpOperationBuilder.Add(opID1, internalID)
-	tt.Assert.NoError(err)
-	err = lpOperationBuilder.Add(opID2, internalID)
-	tt.Assert.NoError(err)
-	err = lpOperationBuilder.Exec(tt.Ctx, q)
-	tt.Assert.NoError(err)
+	tt.Assert.NoError(lpOperationBuilder.Add(opID1, lpLoader.GetFuture(liquidityPoolID)))
+	tt.Assert.NoError(lpOperationBuilder.Add(opID2, lpLoader.GetFuture(liquidityPoolID)))
+	tt.Assert.NoError(lpLoader.Exec(tt.Ctx, q))
+	tt.Assert.NoError(lpOperationBuilder.Exec(tt.Ctx, q))
 
 	tt.Assert.NoError(q.Commit())
 
diff --git a/services/horizon/internal/db2/history/participants_test.go b/services/horizon/internal/db2/history/participants_test.go
index a8d87976ac..16671098bf 100644
--- a/services/horizon/internal/db2/history/participants_test.go
+++ b/services/horizon/internal/db2/history/participants_test.go
@@ -63,7 +63,7 @@ func TestTransactionParticipantsBatch(t *testing.T) {
 		{TransactionID: 2},
 	}
 	for i := range expected {
-		expected[i].AccountID = accountLoader.getNow(addresses[i])
+		expected[i].AccountID = accountLoader.GetNow(addresses[i])
 	}
 	tt.Assert.ElementsMatch(expected, participants)
 }
diff --git a/services/horizon/internal/db2/history/transaction_test.go b/services/horizon/internal/db2/history/transaction_test.go
index 5bc423006e..a145a13d43 100644
--- a/services/horizon/internal/db2/history/transaction_test.go
+++ b/services/horizon/internal/db2/history/transaction_test.go
@@ -8,6 +8,7 @@ import (
 
 	sq "github.com/Masterminds/squirrel"
 	"github.com/guregu/null"
+
 	"github.com/stellar/go/xdr"
 
 	"github.com/stellar/go/ingest"
@@ -77,22 +78,17 @@ func TestTransactionByLiquidityPool(t *testing.T) {
 
 	// Insert Liquidity Pool history
 	liquidityPoolID := "a2f38836a839de008cf1d782c81f45e1253cc5d3dad9110b872965484fec0a49"
-	toInternalID, err := q.CreateHistoryLiquidityPools(tt.Ctx, []string{liquidityPoolID}, 2)
-	tt.Assert.NoError(err)
+	lpLoader := NewLiquidityPoolLoader()
 	lpTransactionBuilder := q.NewTransactionLiquidityPoolBatchInsertBuilder()
-	tt.Assert.NoError(err)
-	internalID, ok := toInternalID[liquidityPoolID]
-	tt.Assert.True(ok)
-	err = lpTransactionBuilder.Add(txID, internalID)
-	tt.Assert.NoError(err)
-	err = lpTransactionBuilder.Exec(tt.Ctx, q)
-	tt.Assert.NoError(err)
-
+	tt.Assert.NoError(lpTransactionBuilder.Add(txID, lpLoader.GetFuture(liquidityPoolID)))
+	tt.Assert.NoError(lpLoader.Exec(tt.Ctx, q))
+	tt.Assert.NoError(lpTransactionBuilder.Exec(tt.Ctx, q))
 	tt.Assert.NoError(q.Commit())
 
 	var records []Transaction
-	err = q.Transactions().ForLiquidityPool(tt.Ctx, liquidityPoolID).Select(tt.Ctx, &records)
-	tt.Assert.NoError(err)
+	tt.Assert.NoError(
+		q.Transactions().ForLiquidityPool(tt.Ctx, liquidityPoolID).Select(tt.Ctx, &records),
+	)
 	tt.Assert.Len(records, 1)
 
 }
diff --git a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor.go b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor.go
index aecb25f88d..394d2e0f9b 100644
--- a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor.go
+++ b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor.go
@@ -5,56 +5,39 @@ import (
 
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
-	set "github.com/stellar/go/support/collections/set"
 	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
 )
 
-type claimableBalance struct {
-	internalID     int64 // Bigint auto-generated by postgres
-	transactionSet set.Set[int64]
-	operationSet   set.Set[int64]
-}
-
-func (b *claimableBalance) addTransactionID(id int64) {
-	if b.transactionSet == nil {
-		b.transactionSet = set.Set[int64]{}
-	}
-	b.transactionSet.Add(id)
-}
-
-func (b *claimableBalance) addOperationID(id int64) {
-	if b.operationSet == nil {
-		b.operationSet = set.Set[int64]{}
-	}
-	b.operationSet.Add(id)
-}
-
 type ClaimableBalancesTransactionProcessor struct {
-	session             db.SessionInterface
-	sequence            uint32
-	claimableBalanceSet map[string]claimableBalance
-	qClaimableBalances  history.QHistoryClaimableBalances
+	cbLoader *history.ClaimableBalanceLoader
+	txBatch  history.TransactionClaimableBalanceBatchInsertBuilder
+	opBatch  history.OperationClaimableBalanceBatchInsertBuilder
 }
 
-func NewClaimableBalancesTransactionProcessor(session db.SessionInterface, Q history.QHistoryClaimableBalances, sequence uint32) *ClaimableBalancesTransactionProcessor {
+func NewClaimableBalancesTransactionProcessor(
+	cbLoader *history.ClaimableBalanceLoader,
+	txBatch history.TransactionClaimableBalanceBatchInsertBuilder,
+	opBatch history.OperationClaimableBalanceBatchInsertBuilder,
+) *ClaimableBalancesTransactionProcessor {
 	return &ClaimableBalancesTransactionProcessor{
-		session:             session,
-		qClaimableBalances:  Q,
-		sequence:            sequence,
-		claimableBalanceSet: map[string]claimableBalance{},
+		cbLoader: cbLoader,
+		txBatch:  txBatch,
+		opBatch:  opBatch,
 	}
 }
 
-func (p *ClaimableBalancesTransactionProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error {
-	err := p.addTransactionClaimableBalances(p.claimableBalanceSet, p.sequence, transaction)
+func (p *ClaimableBalancesTransactionProcessor) ProcessTransaction(
+	lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction,
+) error {
+	err := p.addTransactionClaimableBalances(lcm.LedgerSequence(), transaction)
 	if err != nil {
 		return err
 	}
 
-	err = p.addOperationClaimableBalances(p.claimableBalanceSet, p.sequence, transaction)
+	err = p.addOperationClaimableBalances(lcm.LedgerSequence(), transaction)
 	if err != nil {
 		return err
 	}
@@ -62,27 +45,25 @@ func (p *ClaimableBalancesTransactionProcessor) ProcessTransaction(ctx context.C
 	return nil
 }
 
-func (p *ClaimableBalancesTransactionProcessor) addTransactionClaimableBalances(cbSet map[string]claimableBalance, sequence uint32, transaction ingest.LedgerTransaction) error {
+func (p *ClaimableBalancesTransactionProcessor) addTransactionClaimableBalances(
+	sequence uint32, transaction ingest.LedgerTransaction,
+) error {
 	transactionID := toid.New(int32(sequence), int32(transaction.Index), 0).ToInt64()
-	transactionClaimableBalances, err := claimableBalancesForTransaction(
-		sequence,
-		transaction,
-	)
+	transactionClaimableBalances, err := claimableBalancesForTransaction(transaction)
 	if err != nil {
 		return errors.Wrap(err, "Could not determine claimable balances for transaction")
 	}
 
-	for _, cb := range transactionClaimableBalances {
-		entry := cbSet[cb]
-		entry.addTransactionID(transactionID)
-		cbSet[cb] = entry
+	for _, cb := range dedupeStrings(transactionClaimableBalances) {
+		if err = p.txBatch.Add(transactionID, p.cbLoader.GetFuture(cb)); err != nil {
+			return err
+		}
 	}
 
 	return nil
 }
 
 func claimableBalancesForTransaction(
-	sequence uint32,
 	transaction ingest.LedgerTransaction,
 ) ([]string, error) {
 	changes, err := transaction.GetChanges()
@@ -93,19 +74,7 @@ func claimableBalancesForTransaction(
 	if err != nil {
 		return nil, errors.Wrapf(err, "reading transaction %v claimable balances", transaction.Index)
 	}
-	return dedupeClaimableBalances(cbs)
-}
-
-func dedupeClaimableBalances(in []string) (out []string, err error) {
-	set := set.Set[string]{}
-	for _, id := range in {
-		set.Add(id)
-	}
-
-	for id := range set {
-		out = append(out, id)
-	}
-	return
+	return cbs, nil
 }
 
 func claimableBalancesForChanges(
@@ -139,26 +108,9 @@ func claimableBalancesForChanges(
 	return cbs, nil
 }
 
-func (p *ClaimableBalancesTransactionProcessor) addOperationClaimableBalances(cbSet map[string]claimableBalance, sequence uint32, transaction ingest.LedgerTransaction) error {
-	claimableBalances, err := claimableBalancesForOperations(transaction, sequence)
-	if err != nil {
-		return errors.Wrap(err, "could not determine operation claimable balances")
-	}
-
-	for operationID, cbs := range claimableBalances {
-		for _, cb := range cbs {
-			entry := cbSet[cb]
-			entry.addOperationID(operationID)
-			cbSet[cb] = entry
-		}
-	}
-
-	return nil
-}
-
-func claimableBalancesForOperations(transaction ingest.LedgerTransaction, sequence uint32) (map[int64][]string, error) {
-	cbs := map[int64][]string{}
-
+func (p *ClaimableBalancesTransactionProcessor) addOperationClaimableBalances(
+	sequence uint32, transaction ingest.LedgerTransaction,
+) error {
 	for opi, op := range transaction.Envelope.Operations() {
 		operation := transactionOperationWrapper{
 			index:          uint32(opi),
@@ -169,92 +121,33 @@ func claimableBalancesForOperations(transaction ingest.LedgerTransaction, sequen
 
 		changes, err := transaction.GetOperationChanges(uint32(opi))
 		if err != nil {
-			return cbs, err
-		}
-		c, err := claimableBalancesForChanges(changes)
-		if err != nil {
-			return cbs, errors.Wrapf(err, "reading operation %v claimable balances", operation.ID())
-		}
-		cbs[operation.ID()] = c
-	}
-
-	return cbs, nil
-}
-
-func (p *ClaimableBalancesTransactionProcessor) Commit(ctx context.Context) error {
-	if len(p.claimableBalanceSet) > 0 {
-		if err := p.loadClaimableBalanceIDs(ctx, p.claimableBalanceSet); err != nil {
 			return err
 		}
-
-		if err := p.insertDBTransactionClaimableBalances(ctx, p.claimableBalanceSet); err != nil {
-			return err
+		cbs, err := claimableBalancesForChanges(changes)
+		if err != nil {
+			return errors.Wrapf(err, "reading operation %v claimable balances", operation.ID())
 		}
 
-		if err := p.insertDBOperationsClaimableBalances(ctx, p.claimableBalanceSet); err != nil {
-			return err
+		for _, cb := range dedupeStrings(cbs) {
+			if err = p.opBatch.Add(operation.ID(), p.cbLoader.GetFuture(cb)); err != nil {
+				return err
+			}
 		}
 	}
 
 	return nil
 }
 
-func (p *ClaimableBalancesTransactionProcessor) loadClaimableBalanceIDs(ctx context.Context, claimableBalanceSet map[string]claimableBalance) error {
-	ids := make([]string, 0, len(claimableBalanceSet))
-	for id := range claimableBalanceSet {
-		ids = append(ids, id)
-	}
-
-	toInternalID, err := p.qClaimableBalances.CreateHistoryClaimableBalances(ctx, ids, maxBatchSize)
+func (p *ClaimableBalancesTransactionProcessor) Flush(ctx context.Context, session db.SessionInterface) error {
+	err := p.txBatch.Exec(ctx, session)
 	if err != nil {
-		return errors.Wrap(err, "Could not create claimable balance ids")
-	}
-
-	for _, id := range ids {
-		internalID, ok := toInternalID[id]
-		if !ok {
-			// TODO: Figure out the right way to convert the id to a string here. %v will be nonsense.
-			return errors.Errorf("no internal id found for claimable balance %v", id)
-		}
-
-		cb := claimableBalanceSet[id]
-		cb.internalID = internalID
-		claimableBalanceSet[id] = cb
-	}
-
-	return nil
-}
-
-func (p ClaimableBalancesTransactionProcessor) insertDBTransactionClaimableBalances(ctx context.Context, claimableBalanceSet map[string]claimableBalance) error {
-	batch := p.qClaimableBalances.NewTransactionClaimableBalanceBatchInsertBuilder()
-
-	for _, entry := range claimableBalanceSet {
-		for transactionID := range entry.transactionSet {
-			if err := batch.Add(transactionID, entry.internalID); err != nil {
-				return errors.Wrap(err, "could not insert transaction claimable balance in db")
-			}
-		}
-	}
-
-	if err := batch.Exec(ctx, p.session); err != nil {
-		return errors.Wrap(err, "could not flush transaction claimable balances to db")
+		return err
 	}
-	return nil
-}
 
-func (p ClaimableBalancesTransactionProcessor) insertDBOperationsClaimableBalances(ctx context.Context, claimableBalanceSet map[string]claimableBalance) error {
-	batch := p.qClaimableBalances.NewOperationClaimableBalanceBatchInsertBuilder()
-
-	for _, entry := range claimableBalanceSet {
-		for operationID := range entry.operationSet {
-			if err := batch.Add(operationID, entry.internalID); err != nil {
-				return errors.Wrap(err, "could not insert operation claimable balance in db")
-			}
-		}
+	err = p.opBatch.Exec(ctx, session)
+	if err != nil {
+		return err
 	}
 
-	if err := batch.Exec(ctx, p.session); err != nil {
-		return errors.Wrap(err, "could not flush operation claimable balances to db")
-	}
 	return nil
 }
diff --git a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go
index 9c771c7e8a..11ce54505a 100644
--- a/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go
+++ b/services/horizon/internal/ingest/processors/claimable_balances_transaction_processor_test.go
@@ -6,7 +6,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
 
 	"github.com/stellar/go/services/horizon/internal/db2/history"
@@ -20,11 +19,11 @@ type ClaimableBalancesTransactionProcessorTestSuiteLedger struct {
 	ctx                               context.Context
 	processor                         *ClaimableBalancesTransactionProcessor
 	mockSession                       *db.MockSession
-	mockQ                             *history.MockQHistoryClaimableBalances
 	mockTransactionBatchInsertBuilder *history.MockTransactionClaimableBalanceBatchInsertBuilder
 	mockOperationBatchInsertBuilder   *history.MockOperationClaimableBalanceBatchInsertBuilder
+	cbLoader                          *history.ClaimableBalanceLoader
 
-	sequence uint32
+	lcm xdr.LedgerCloseMeta
 }
 
 func TestClaimableBalancesTransactionProcessorTestSuiteLedger(t *testing.T) {
@@ -33,41 +32,41 @@ func TestClaimableBalancesTransactionProcessorTestSuiteLedger(t *testing.T) {
 
 func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) SetupTest() {
 	s.ctx = context.Background()
-	s.mockQ = &history.MockQHistoryClaimableBalances{}
 	s.mockTransactionBatchInsertBuilder = &history.MockTransactionClaimableBalanceBatchInsertBuilder{}
 	s.mockOperationBatchInsertBuilder = &history.MockOperationClaimableBalanceBatchInsertBuilder{}
-	s.sequence = 20
+	sequence := uint32(20)
+	s.lcm = xdr.LedgerCloseMeta{
+		V0: &xdr.LedgerCloseMetaV0{
+			LedgerHeader: xdr.LedgerHeaderHistoryEntry{
+				Header: xdr.LedgerHeader{
+					LedgerSeq: xdr.Uint32(sequence),
+				},
+			},
+		},
+	}
+	s.cbLoader = history.NewClaimableBalanceLoader()
 
 	s.processor = NewClaimableBalancesTransactionProcessor(
-		s.mockSession,
-		s.mockQ,
-		s.sequence,
+		s.cbLoader,
+		s.mockTransactionBatchInsertBuilder,
+		s.mockOperationBatchInsertBuilder,
 	)
 }
 
 func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) TearDownTest() {
-	s.mockQ.AssertExpectations(s.T())
 	s.mockTransactionBatchInsertBuilder.AssertExpectations(s.T())
 	s.mockOperationBatchInsertBuilder.AssertExpectations(s.T())
 }
 
-func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) mockTransactionBatchAdd(transactionID, internalID int64, err error) {
-	s.mockTransactionBatchInsertBuilder.On("Add", transactionID, internalID).Return(err).Once()
-}
-
-func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) mockOperationBatchAdd(operationID, internalID int64, err error) {
-	s.mockOperationBatchInsertBuilder.On("Add", operationID, internalID).Return(err).Once()
-}
-
 func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) TestEmptyClaimableBalances() {
-	// What is this expecting? Doesn't seem to assert anything meaningful...
-	err := s.processor.Commit(context.Background())
-	s.Assert().NoError(err)
+	s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
+	s.mockOperationBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
+
+	s.Assert().NoError(s.processor.Flush(s.ctx, s.mockSession))
 }
 
 func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) testOperationInserts(balanceID xdr.ClaimableBalanceId, body xdr.OperationBody, change xdr.LedgerEntryChange) {
 	// Setup the transaction
-	internalID := int64(1234)
 	txn := createTransaction(true, 1)
 	txn.Envelope.Operations()[0].Body = body
 	txn.UnsafeMeta.V = 2
@@ -85,6 +84,20 @@ func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) testOperationInse
 				},
 			},
 			change,
+			// add a duplicate change to test that the processor
+			// does not insert duplicate rows
+			{
+				Type: xdr.LedgerEntryChangeTypeLedgerEntryState,
+				State: &xdr.LedgerEntry{
+					Data: xdr.LedgerEntryData{
+						Type: xdr.LedgerEntryTypeClaimableBalance,
+						ClaimableBalance: &xdr.ClaimableBalanceEntry{
+							BalanceId: balanceID,
+						},
+					},
+				},
+			},
+			change,
 		}},
 	}
 
@@ -104,46 +117,28 @@ func (s *ClaimableBalancesTransactionProcessorTestSuiteLedger) testOperationInse
 				},
 			}
 	}
-	txnID := toid.New(int32(s.sequence), int32(txn.Index), 0).ToInt64()
+	txnID := toid.New(int32(s.lcm.LedgerSequence()), int32(txn.Index), 0).ToInt64()
 	opID := (&transactionOperationWrapper{
 		index:          uint32(0),
 		transaction:    txn,
 		operation:      txn.Envelope.Operations()[0],
-		ledgerSequence: s.sequence,
+		ledgerSequence: s.lcm.LedgerSequence(),
 	}).ID()
 
-	hexID, _ := xdr.MarshalHex(balanceID)
+	hexID, err := xdr.MarshalHex(balanceID)
+	s.Assert().NoError(err)
 
-	// Setup a q
-	s.mockQ.On("CreateHistoryClaimableBalances", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]string)
-			s.Assert().ElementsMatch(
-				[]string{
-					hexID,
-				},
-				arg,
-			)
-		}).Return(map[string]int64{
-		hexID: internalID,
-	}, nil).Once()
-
-	// Prepare to process transactions successfully
-	s.mockQ.On("NewTransactionClaimableBalanceBatchInsertBuilder").
-		Return(s.mockTransactionBatchInsertBuilder).Once()
-	s.mockTransactionBatchAdd(txnID, internalID, nil)
+	s.mockTransactionBatchInsertBuilder.On("Add", txnID, s.cbLoader.GetFuture(hexID)).Return(nil).Once()
 	s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	// Prepare to process operations successfully
-	s.mockQ.On("NewOperationClaimableBalanceBatchInsertBuilder").
-		Return(s.mockOperationBatchInsertBuilder).Once()
-	s.mockOperationBatchAdd(opID, internalID, nil)
+	s.mockOperationBatchInsertBuilder.On("Add", opID, s.cbLoader.GetFuture(hexID)).Return(nil).Once()
 	s.mockOperationBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	// Process the transaction
-	err := s.processor.ProcessTransaction(s.ctx, txn)
+	err = s.processor.ProcessTransaction(s.lcm, txn)
 	s.Assert().NoError(err)
-	err = s.processor.Commit(s.ctx)
+	err = s.processor.Flush(s.ctx, s.mockSession)
 	s.Assert().NoError(err)
 }
 
diff --git a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor.go b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor.go
index a55cb95934..0a38215f08 100644
--- a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor.go
+++ b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor.go
@@ -5,56 +5,38 @@ import (
 
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
-	set "github.com/stellar/go/support/collections/set"
+	"github.com/stellar/go/support/collections/set"
 	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
 )
 
-type liquidityPool struct {
-	internalID     int64 // Bigint auto-generated by postgres
-	transactionSet set.Set[int64]
-	operationSet   set.Set[int64]
-}
-
-func (b *liquidityPool) addTransactionID(id int64) {
-	if b.transactionSet == nil {
-		b.transactionSet = set.Set[int64]{}
-	}
-	b.transactionSet.Add(id)
-}
-
-func (b *liquidityPool) addOperationID(id int64) {
-	if b.operationSet == nil {
-		b.operationSet = set.Set[int64]{}
-	}
-	b.operationSet.Add(id)
-}
-
 type LiquidityPoolsTransactionProcessor struct {
-	session          db.SessionInterface
-	sequence         uint32
-	liquidityPoolSet map[string]liquidityPool
-	qLiquidityPools  history.QHistoryLiquidityPools
+	lpLoader *history.LiquidityPoolLoader
+	txBatch  history.TransactionLiquidityPoolBatchInsertBuilder
+	opBatch  history.OperationLiquidityPoolBatchInsertBuilder
 }
 
-func NewLiquidityPoolsTransactionProcessor(session db.SessionInterface, Q history.QHistoryLiquidityPools, sequence uint32) *LiquidityPoolsTransactionProcessor {
+func NewLiquidityPoolsTransactionProcessor(
+	lpLoader *history.LiquidityPoolLoader,
+	txBatch history.TransactionLiquidityPoolBatchInsertBuilder,
+	opBatch history.OperationLiquidityPoolBatchInsertBuilder,
+) *LiquidityPoolsTransactionProcessor {
 	return &LiquidityPoolsTransactionProcessor{
-		session:          session,
-		qLiquidityPools:  Q,
-		sequence:         sequence,
-		liquidityPoolSet: map[string]liquidityPool{},
+		lpLoader: lpLoader,
+		txBatch:  txBatch,
+		opBatch:  opBatch,
 	}
 }
 
-func (p *LiquidityPoolsTransactionProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error {
-	err := p.addTransactionLiquidityPools(p.liquidityPoolSet, p.sequence, transaction)
+func (p *LiquidityPoolsTransactionProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error {
+	err := p.addTransactionLiquidityPools(lcm.LedgerSequence(), transaction)
 	if err != nil {
 		return err
 	}
 
-	err = p.addOperationLiquidityPools(p.liquidityPoolSet, p.sequence, transaction)
+	err = p.addOperationLiquidityPools(lcm.LedgerSequence(), transaction)
 	if err != nil {
 		return err
 	}
@@ -62,29 +44,23 @@ func (p *LiquidityPoolsTransactionProcessor) ProcessTransaction(ctx context.Cont
 	return nil
 }
 
-func (p *LiquidityPoolsTransactionProcessor) addTransactionLiquidityPools(lpSet map[string]liquidityPool, sequence uint32, transaction ingest.LedgerTransaction) error {
+func (p *LiquidityPoolsTransactionProcessor) addTransactionLiquidityPools(sequence uint32, transaction ingest.LedgerTransaction) error {
 	transactionID := toid.New(int32(sequence), int32(transaction.Index), 0).ToInt64()
-	transactionLiquidityPools, err := liquidityPoolsForTransaction(
-		sequence,
-		transaction,
-	)
+	lps, err := liquidityPoolsForTransaction(transaction)
 	if err != nil {
 		return errors.Wrap(err, "Could not determine liquidity pools for transaction")
 	}
 
-	for _, lp := range transactionLiquidityPools {
-		entry := lpSet[lp]
-		entry.addTransactionID(transactionID)
-		lpSet[lp] = entry
+	for _, lp := range dedupeStrings(lps) {
+		if err = p.txBatch.Add(transactionID, p.lpLoader.GetFuture(lp)); err != nil {
+			return err
+		}
 	}
 
 	return nil
 }
 
-func liquidityPoolsForTransaction(
-	sequence uint32,
-	transaction ingest.LedgerTransaction,
-) ([]string, error) {
+func liquidityPoolsForTransaction(transaction ingest.LedgerTransaction) ([]string, error) {
 	changes, err := transaction.GetChanges()
 	if err != nil {
 		return nil, err
@@ -93,19 +69,20 @@ func liquidityPoolsForTransaction(
 	if err != nil {
 		return nil, errors.Wrapf(err, "reading transaction %v liquidity pools", transaction.Index)
 	}
-	return dedupeLiquidityPools(lps)
+	return lps, nil
 }
 
-func dedupeLiquidityPools(in []string) (out []string, err error) {
+func dedupeStrings(in []string) []string {
 	set := set.Set[string]{}
 	for _, id := range in {
 		set.Add(id)
 	}
 
+	out := make([]string, 0, len(in))
 	for id := range set {
 		out = append(out, id)
 	}
-	return
+	return out
 }
 
 func liquidityPoolsForChanges(
@@ -135,26 +112,7 @@ func liquidityPoolsForChanges(
 	return lps, nil
 }
 
-func (p *LiquidityPoolsTransactionProcessor) addOperationLiquidityPools(lpSet map[string]liquidityPool, sequence uint32, transaction ingest.LedgerTransaction) error {
-	liquidityPools, err := liquidityPoolsForOperations(transaction, sequence)
-	if err != nil {
-		return errors.Wrap(err, "could not determine operation liquidity pools")
-	}
-
-	for operationID, lps := range liquidityPools {
-		for _, lp := range lps {
-			entry := lpSet[lp]
-			entry.addOperationID(operationID)
-			lpSet[lp] = entry
-		}
-	}
-
-	return nil
-}
-
-func liquidityPoolsForOperations(transaction ingest.LedgerTransaction, sequence uint32) (map[int64][]string, error) {
-	lps := map[int64][]string{}
-
+func (p *LiquidityPoolsTransactionProcessor) addOperationLiquidityPools(sequence uint32, transaction ingest.LedgerTransaction) error {
 	for opi, op := range transaction.Envelope.Operations() {
 		operation := transactionOperationWrapper{
 			index:          uint32(opi),
@@ -165,91 +123,29 @@ func liquidityPoolsForOperations(transaction ingest.LedgerTransaction, sequence
 
 		changes, err := transaction.GetOperationChanges(uint32(opi))
 		if err != nil {
-			return lps, err
-		}
-		c, err := liquidityPoolsForChanges(changes)
-		if err != nil {
-			return lps, errors.Wrapf(err, "reading operation %v liquidity pools", operation.ID())
-		}
-		lps[operation.ID()] = c
-	}
-
-	return lps, nil
-}
-
-func (p *LiquidityPoolsTransactionProcessor) Commit(ctx context.Context) error {
-	if len(p.liquidityPoolSet) > 0 {
-		if err := p.loadLiquidityPoolIDs(ctx, p.liquidityPoolSet); err != nil {
 			return err
 		}
-
-		if err := p.insertDBTransactionLiquidityPools(ctx, p.liquidityPoolSet); err != nil {
-			return err
-		}
-
-		if err := p.insertDBOperationsLiquidityPools(ctx, p.liquidityPoolSet); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
-func (p *LiquidityPoolsTransactionProcessor) loadLiquidityPoolIDs(ctx context.Context, liquidityPoolSet map[string]liquidityPool) error {
-	ids := make([]string, 0, len(liquidityPoolSet))
-	for id := range liquidityPoolSet {
-		ids = append(ids, id)
-	}
-
-	toInternalID, err := p.qLiquidityPools.CreateHistoryLiquidityPools(ctx, ids, maxBatchSize)
-	if err != nil {
-		return errors.Wrap(err, "Could not create liquidity pool ids")
-	}
-
-	for _, id := range ids {
-		internalID, ok := toInternalID[id]
-		if !ok {
-			return errors.Errorf("no internal id found for liquidity pool %s", id)
+		lps, err := liquidityPoolsForChanges(changes)
+		if err != nil {
+			return errors.Wrapf(err, "reading operation %v liquidity pools", operation.ID())
 		}
-
-		lp := liquidityPoolSet[id]
-		lp.internalID = internalID
-		liquidityPoolSet[id] = lp
-	}
-
-	return nil
-}
-
-func (p LiquidityPoolsTransactionProcessor) insertDBTransactionLiquidityPools(ctx context.Context, liquidityPoolSet map[string]liquidityPool) error {
-	batch := p.qLiquidityPools.NewTransactionLiquidityPoolBatchInsertBuilder()
-
-	for _, entry := range liquidityPoolSet {
-		for transactionID := range entry.transactionSet {
-			if err := batch.Add(transactionID, entry.internalID); err != nil {
-				return errors.Wrap(err, "could not insert transaction liquidity pool in db")
+		for _, lp := range dedupeStrings(lps) {
+			if err := p.opBatch.Add(operation.ID(), p.lpLoader.GetFuture(lp)); err != nil {
+				return err
 			}
 		}
 	}
 
-	if err := batch.Exec(ctx, p.session); err != nil {
-		return errors.Wrap(err, "could not flush transaction liquidity pools to db")
-	}
 	return nil
 }
 
-func (p LiquidityPoolsTransactionProcessor) insertDBOperationsLiquidityPools(ctx context.Context, liquidityPoolSet map[string]liquidityPool) error {
-	batch := p.qLiquidityPools.NewOperationLiquidityPoolBatchInsertBuilder()
-
-	for _, entry := range liquidityPoolSet {
-		for operationID := range entry.operationSet {
-			if err := batch.Add(operationID, entry.internalID); err != nil {
-				return errors.Wrap(err, "could not insert operation liquidity pool in db")
-			}
-		}
+func (p *LiquidityPoolsTransactionProcessor) Flush(ctx context.Context, session db.SessionInterface) error {
+	if err := p.txBatch.Exec(ctx, session); err != nil {
+		return errors.Wrap(err, "Could not flush transaction liquidity pools to db")
 	}
-
-	if err := batch.Exec(ctx, p.session); err != nil {
-		return errors.Wrap(err, "could not flush operation liquidity pools to db")
+	if err := p.opBatch.Exec(ctx, session); err != nil {
+		return errors.Wrap(err, "Could not flush operation liquidity pools to db")
 	}
+
 	return nil
 }
diff --git a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go
index f86845e931..485d890dca 100644
--- a/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go
+++ b/services/horizon/internal/ingest/processors/liquidity_pools_transaction_processor_test.go
@@ -6,7 +6,6 @@ import (
 	"context"
 	"testing"
 
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
 
 	"github.com/stellar/go/services/horizon/internal/db2/history"
@@ -20,11 +19,11 @@ type LiquidityPoolsTransactionProcessorTestSuiteLedger struct {
 	ctx                               context.Context
 	processor                         *LiquidityPoolsTransactionProcessor
 	mockSession                       *db.MockSession
-	mockQ                             *history.MockQHistoryLiquidityPools
+	lpLoader                          *history.LiquidityPoolLoader
 	mockTransactionBatchInsertBuilder *history.MockTransactionLiquidityPoolBatchInsertBuilder
 	mockOperationBatchInsertBuilder   *history.MockOperationLiquidityPoolBatchInsertBuilder
 
-	sequence uint32
+	lcm xdr.LedgerCloseMeta
 }
 
 func TestLiquidityPoolsTransactionProcessorTestSuiteLedger(t *testing.T) {
@@ -33,41 +32,42 @@ func TestLiquidityPoolsTransactionProcessorTestSuiteLedger(t *testing.T) {
 
 func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) SetupTest() {
 	s.ctx = context.Background()
-	s.mockQ = &history.MockQHistoryLiquidityPools{}
 	s.mockTransactionBatchInsertBuilder = &history.MockTransactionLiquidityPoolBatchInsertBuilder{}
 	s.mockOperationBatchInsertBuilder = &history.MockOperationLiquidityPoolBatchInsertBuilder{}
-	s.sequence = 20
+	sequence := uint32(20)
+	s.lcm = xdr.LedgerCloseMeta{
+		V0: &xdr.LedgerCloseMetaV0{
+			LedgerHeader: xdr.LedgerHeaderHistoryEntry{
+				Header: xdr.LedgerHeader{
+					LedgerSeq: xdr.Uint32(sequence),
+				},
+			},
+		},
+	}
+	s.lpLoader = history.NewLiquidityPoolLoader()
 
 	s.processor = NewLiquidityPoolsTransactionProcessor(
-		s.mockSession,
-		s.mockQ,
-		s.sequence,
+		s.lpLoader,
+		s.mockTransactionBatchInsertBuilder,
+		s.mockOperationBatchInsertBuilder,
 	)
 }
 
 func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) TearDownTest() {
-	s.mockQ.AssertExpectations(s.T())
 	s.mockTransactionBatchInsertBuilder.AssertExpectations(s.T())
 	s.mockOperationBatchInsertBuilder.AssertExpectations(s.T())
 }
 
-func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) mockTransactionBatchAdd(transactionID, internalID int64, err error) {
-	s.mockTransactionBatchInsertBuilder.On("Add", transactionID, internalID).Return(err).Once()
-}
-
-func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) mockOperationBatchAdd(operationID, internalID int64, err error) {
-	s.mockOperationBatchInsertBuilder.On("Add", operationID, internalID).Return(err).Once()
-}
-
 func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) TestEmptyLiquidityPools() {
-	// What is this expecting? Doesn't seem to assert anything meaningful...
-	err := s.processor.Commit(context.Background())
+	s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
+	s.mockOperationBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
+
+	err := s.processor.Flush(context.Background(), s.mockSession)
 	s.Assert().NoError(err)
 }
 
 func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) testOperationInserts(poolID xdr.PoolId, body xdr.OperationBody, change xdr.LedgerEntryChange) {
 	// Setup the transaction
-	internalID := int64(1234)
 	txn := createTransaction(true, 1)
 	txn.Envelope.Operations()[0].Body = body
 	txn.UnsafeMeta.V = 2
@@ -85,6 +85,20 @@ func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) testOperationInserts
 				},
 			},
 			change,
+			// add a duplicate change to test that the processor
+			// does not insert duplicate rows
+			{
+				Type: xdr.LedgerEntryChangeTypeLedgerEntryState,
+				State: &xdr.LedgerEntry{
+					Data: xdr.LedgerEntryData{
+						Type: xdr.LedgerEntryTypeLiquidityPool,
+						LiquidityPool: &xdr.LiquidityPoolEntry{
+							LiquidityPoolId: poolID,
+						},
+					},
+				},
+			},
+			change,
 		}},
 	}
 
@@ -103,44 +117,28 @@ func (s *LiquidityPoolsTransactionProcessorTestSuiteLedger) testOperationInserts
 				},
 			}
 	}
-	txnID := toid.New(int32(s.sequence), int32(txn.Index), 0).ToInt64()
+	txnID := toid.New(int32(s.lcm.LedgerSequence()), int32(txn.Index), 0).ToInt64()
 	opID := (&transactionOperationWrapper{
 		index:          uint32(0),
 		transaction:    txn,
 		operation:      txn.Envelope.Operations()[0],
-		ledgerSequence: s.sequence,
+		ledgerSequence: s.lcm.LedgerSequence(),
 	}).ID()
 
 	hexID := PoolIDToString(poolID)
 
-	// Setup a q
-	s.mockQ.On("CreateHistoryLiquidityPools", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]string)
-			s.Assert().ElementsMatch(
-				[]string{hexID},
-				arg,
-			)
-		}).Return(map[string]int64{
-		hexID: internalID,
-	}, nil).Once()
-
 	// Prepare to process transactions successfully
-	s.mockQ.On("NewTransactionLiquidityPoolBatchInsertBuilder").
-		Return(s.mockTransactionBatchInsertBuilder).Once()
-	s.mockTransactionBatchAdd(txnID, internalID, nil)
+	s.mockTransactionBatchInsertBuilder.On("Add", txnID, s.lpLoader.GetFuture(hexID)).Return(nil).Once()
 	s.mockTransactionBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	// Prepare to process operations successfully
-	s.mockQ.On("NewOperationLiquidityPoolBatchInsertBuilder").
-		Return(s.mockOperationBatchInsertBuilder).Once()
-	s.mockOperationBatchAdd(opID, internalID, nil)
+	s.mockOperationBatchInsertBuilder.On("Add", opID, s.lpLoader.GetFuture(hexID)).Return(nil).Once()
 	s.mockOperationBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	// Process the transaction
-	err := s.processor.ProcessTransaction(s.ctx, txn)
+	err := s.processor.ProcessTransaction(s.lcm, txn)
 	s.Assert().NoError(err)
-	err = s.processor.Commit(s.ctx)
+	err = s.processor.Flush(s.ctx, s.mockSession)
 	s.Assert().NoError(err)
 }
 
diff --git a/services/horizon/internal/ingest/processors/participants_processor.go b/services/horizon/internal/ingest/processors/participants_processor.go
index 45ffcea524..7d4ae7fe39 100644
--- a/services/horizon/internal/ingest/processors/participants_processor.go
+++ b/services/horizon/internal/ingest/processors/participants_processor.go
@@ -155,7 +155,7 @@ func (p *ParticipantsProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, tran
 	return nil
 }
 
-func (p *ParticipantsProcessor) Commit(ctx context.Context, session db.SessionInterface) error {
+func (p *ParticipantsProcessor) Flush(ctx context.Context, session db.SessionInterface) error {
 	if err := p.txBatch.Exec(ctx, session); err != nil {
 		return errors.Wrap(err, "Could not flush transaction participants to db")
 	}
diff --git a/services/horizon/internal/ingest/processors/participants_processor_test.go b/services/horizon/internal/ingest/processors/participants_processor_test.go
index 28bc6a126c..2348b79eaf 100644
--- a/services/horizon/internal/ingest/processors/participants_processor_test.go
+++ b/services/horizon/internal/ingest/processors/participants_processor_test.go
@@ -145,7 +145,7 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestEmptyParticipants() {
 	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 	s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
-	err := s.processor.Commit(s.ctx, s.mockSession)
+	err := s.processor.Flush(s.ctx, s.mockSession)
 	s.Assert().NoError(err)
 }
 
@@ -190,7 +190,7 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestFeeBumptransaction() {
 	s.mockOperationsBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
 
 	s.Assert().NoError(s.processor.ProcessTransaction(s.lcm, feeBumpTx))
-	s.Assert().NoError(s.processor.Commit(s.ctx, s.mockSession))
+	s.Assert().NoError(s.processor.Flush(s.ctx, s.mockSession))
 }
 
 func (s *ParticipantsProcessorTestSuiteLedger) TestIngestParticipantsSucceeds() {
@@ -204,7 +204,7 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestIngestParticipantsSucceeds()
 		err := s.processor.ProcessTransaction(s.lcm, tx)
 		s.Assert().NoError(err)
 	}
-	err := s.processor.Commit(s.ctx, s.mockSession)
+	err := s.processor.Flush(s.ctx, s.mockSession)
 	s.Assert().NoError(err)
 }
 
@@ -240,7 +240,7 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestBatchAddExecFails() {
 		err := s.processor.ProcessTransaction(s.lcm, tx)
 		s.Assert().NoError(err)
 	}
-	err := s.processor.Commit(s.ctx, s.mockSession)
+	err := s.processor.Flush(s.ctx, s.mockSession)
 	s.Assert().EqualError(err, "Could not flush transaction participants to db: transient error")
 }
 
@@ -255,6 +255,6 @@ func (s *ParticipantsProcessorTestSuiteLedger) TestOperationBatchAddExecFails()
 		err := s.processor.ProcessTransaction(s.lcm, tx)
 		s.Assert().NoError(err)
 	}
-	err := s.processor.Commit(s.ctx, s.mockSession)
+	err := s.processor.Flush(s.ctx, s.mockSession)
 	s.Assert().EqualError(err, "Could not flush operation participants to db: transient error")
 }
diff --git a/services/horizon/internal/ingest/processors/trades_processor.go b/services/horizon/internal/ingest/processors/trades_processor.go
index 10aed5d9ea..b1084c6e08 100644
--- a/services/horizon/internal/ingest/processors/trades_processor.go
+++ b/services/horizon/internal/ingest/processors/trades_processor.go
@@ -19,18 +19,25 @@ import (
 
 // TradeProcessor operations processor
 type TradeProcessor struct {
-	session db.SessionInterface
-	tradesQ history.QTrades
-	ledger  xdr.LedgerHeaderHistoryEntry
-	trades  []ingestTrade
-	stats   TradeStats
+	accountLoader *history.AccountLoader
+	lpLoader      *history.LiquidityPoolLoader
+	assetLoader   *history.AssetLoader
+	batch         history.TradeBatchInsertBuilder
+	trades        []ingestTrade
+	stats         TradeStats
 }
 
-func NewTradeProcessor(session db.SessionInterface, tradesQ history.QTrades, ledger xdr.LedgerHeaderHistoryEntry) *TradeProcessor {
+func NewTradeProcessor(
+	accountLoader *history.AccountLoader,
+	lpLoader *history.LiquidityPoolLoader,
+	assetLoader *history.AssetLoader,
+	batch history.TradeBatchInsertBuilder,
+) *TradeProcessor {
 	return &TradeProcessor{
-		session: session,
-		tradesQ: tradesQ,
-		ledger:  ledger,
+		accountLoader: accountLoader,
+		lpLoader:      lpLoader,
+		assetLoader:   assetLoader,
+		batch:         batch,
 	}
 }
 
@@ -48,79 +55,54 @@ func (stats *TradeStats) Map() map[string]interface{} {
 }
 
 // ProcessTransaction process the given transaction
-func (p *TradeProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) (err error) {
+func (p *TradeProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) (err error) {
 	if !transaction.Result.Successful() {
 		return nil
 	}
 
-	trades, err := p.extractTrades(p.ledger, transaction)
+	trades, err := p.extractTrades(lcm.LedgerHeaderHistoryEntry(), transaction)
 	if err != nil {
 		return err
 	}
 
-	p.trades = append(p.trades, trades...)
-	p.stats.count += int64(len(trades))
-	return nil
-}
-
-func (p *TradeProcessor) Commit(ctx context.Context) error {
-	if len(p.trades) == 0 {
-		return nil
-	}
-
-	batch := p.tradesQ.NewTradeBatchInsertBuilder()
-	var poolIDs, accounts []string
-	var assets []xdr.Asset
-	for _, trade := range p.trades {
+	for _, trade := range trades {
 		if trade.buyerAccount != "" {
-			accounts = append(accounts, trade.buyerAccount)
+			p.accountLoader.GetFuture(trade.buyerAccount)
 		}
 		if trade.sellerAccount != "" {
-			accounts = append(accounts, trade.sellerAccount)
+			p.accountLoader.GetFuture(trade.sellerAccount)
 		}
 		if trade.liquidityPoolID != "" {
-			poolIDs = append(poolIDs, trade.liquidityPoolID)
+			p.lpLoader.GetFuture(trade.liquidityPoolID)
 		}
-		assets = append(assets, trade.boughtAsset)
-		assets = append(assets, trade.soldAsset)
-	}
-
-	accountSet, err := p.tradesQ.CreateAccounts(ctx, accounts, maxBatchSize)
-	if err != nil {
-		return errors.Wrap(err, "Error creating account ids")
+		p.assetLoader.GetFuture(history.AssetKeyFromXDR(trade.boughtAsset))
+		p.assetLoader.GetFuture(history.AssetKeyFromXDR(trade.soldAsset))
 	}
 
-	var assetMap map[string]history.Asset
-	assetMap, err = p.tradesQ.CreateAssets(ctx, assets, maxBatchSize)
-	if err != nil {
-		return errors.Wrap(err, "Error creating asset ids")
-	}
+	p.trades = append(p.trades, trades...)
+	p.stats.count += int64(len(trades))
+	return nil
+}
 
-	var poolMap map[string]int64
-	poolMap, err = p.tradesQ.CreateHistoryLiquidityPools(ctx, poolIDs, maxBatchSize)
-	if err != nil {
-		return errors.Wrap(err, "Error creating pool ids")
+func (p *TradeProcessor) Flush(ctx context.Context, session db.SessionInterface) error {
+	if len(p.trades) == 0 {
+		return nil
 	}
 
 	for _, trade := range p.trades {
 		row := trade.row
-		if id, ok := accountSet[trade.sellerAccount]; ok {
-			row.BaseAccountID = null.IntFrom(id)
-		} else if len(trade.sellerAccount) > 0 {
-			return errors.Errorf("Could not find history account id for %s", trade.sellerAccount)
+		if trade.sellerAccount != "" {
+			row.BaseAccountID = null.IntFrom(p.accountLoader.GetNow(trade.sellerAccount))
 		}
-		if id, ok := accountSet[trade.buyerAccount]; ok {
-			row.CounterAccountID = null.IntFrom(id)
-		} else if len(trade.buyerAccount) > 0 {
-			return errors.Errorf("Could not find history account id for %s", trade.buyerAccount)
+		if trade.buyerAccount != "" {
+			row.CounterAccountID = null.IntFrom(p.accountLoader.GetNow(trade.buyerAccount))
 		}
-		if id, ok := poolMap[trade.liquidityPoolID]; ok {
-			row.BaseLiquidityPoolID = null.IntFrom(id)
-		} else if len(trade.liquidityPoolID) > 0 {
-			return errors.Errorf("Could not find history liquidity pool id for %s", trade.liquidityPoolID)
+		if trade.liquidityPoolID != "" {
+			row.BaseLiquidityPoolID = null.IntFrom(p.lpLoader.GetNow(trade.liquidityPoolID))
 		}
-		row.BaseAssetID = assetMap[trade.soldAsset.String()].ID
-		row.CounterAssetID = assetMap[trade.boughtAsset.String()].ID
+
+		row.BaseAssetID = p.assetLoader.GetNow(history.AssetKeyFromXDR(trade.soldAsset))
+		row.CounterAssetID = p.assetLoader.GetNow(history.AssetKeyFromXDR(trade.boughtAsset))
 
 		if row.BaseAssetID > row.CounterAssetID {
 			row.BaseIsSeller = false
@@ -136,12 +118,12 @@ func (p *TradeProcessor) Commit(ctx context.Context) error {
 			}
 		}
 
-		if err = batch.Add(row); err != nil {
+		if err := p.batch.Add(row); err != nil {
 			return errors.Wrap(err, "Error adding trade to batch")
 		}
 	}
 
-	if err = batch.Exec(ctx, p.session); err != nil {
+	if err := p.batch.Exec(ctx, session); err != nil {
 		return errors.Wrap(err, "Error flushing operation batch")
 	}
 	return nil
diff --git a/services/horizon/internal/ingest/processors/trades_processor_test.go b/services/horizon/internal/ingest/processors/trades_processor_test.go
index d0cf299d30..5b2a2f20e3 100644
--- a/services/horizon/internal/ingest/processors/trades_processor_test.go
+++ b/services/horizon/internal/ingest/processors/trades_processor_test.go
@@ -10,20 +10,23 @@ import (
 
 	"github.com/guregu/null"
 
+	"github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/suite"
+
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
-	"github.com/stretchr/testify/mock"
-	"github.com/stretchr/testify/suite"
 )
 
 type TradeProcessorTestSuiteLedger struct {
 	suite.Suite
 	processor              *TradeProcessor
 	mockSession            *db.MockSession
-	mockQ                  *history.MockQTrades
+	accountLoader          history.AccountLoaderStub
+	lpLoader               history.LiquidityPoolLoaderStub
+	assetLoader            history.AssetLoaderStub
 	mockBatchInsertBuilder *history.MockTradeBatchInsertBuilder
 
 	unmuxedSourceAccount       xdr.AccountId
@@ -45,9 +48,10 @@ type TradeProcessorTestSuiteLedger struct {
 
 	lpToID             map[xdr.PoolId]int64
 	unmuxedAccountToID map[string]int64
-	assetToID          map[string]history.Asset
+	assetToID          map[history.AssetKey]history.Asset
 
 	txs []ingest.LedgerTransaction
+	lcm xdr.LedgerCloseMeta
 }
 
 func TestTradeProcessorTestSuiteLedger(t *testing.T) {
@@ -55,7 +59,6 @@ func TestTradeProcessorTestSuiteLedger(t *testing.T) {
 }
 
 func (s *TradeProcessorTestSuiteLedger) SetupTest() {
-	s.mockQ = &history.MockQTrades{}
 	s.mockBatchInsertBuilder = &history.MockTradeBatchInsertBuilder{}
 
 	s.unmuxedSourceAccount = xdr.MustAddress("GAUJETIZVEP2NRYLUESJ3LS66NVCEGMON4UDCBCSBEVPIID773P2W6AY")
@@ -165,7 +168,7 @@ func (s *TradeProcessorTestSuiteLedger) SetupTest() {
 		s.unmuxedSourceAccount.Address():   1000,
 		s.unmuxedOpSourceAccount.Address(): 1001,
 	}
-	s.assetToID = map[string]history.Asset{}
+	s.assetToID = map[history.AssetKey]history.Asset{}
 	s.allTrades = []xdr.ClaimAtom{
 		s.strictReceiveTrade,
 		s.strictSendTrade,
@@ -190,43 +193,51 @@ func (s *TradeProcessorTestSuiteLedger) SetupTest() {
 			s.sellPrices = append(s.sellPrices, xdr.Price{N: xdr.Int32(trade.AmountBought()), D: xdr.Int32(trade.AmountSold())})
 		}
 		if i%2 == 0 {
-			s.assetToID[trade.AssetSold().String()] = history.Asset{ID: int64(10000 + i)}
-			s.assetToID[trade.AssetBought().String()] = history.Asset{ID: int64(100 + i)}
+			s.assetToID[history.AssetKeyFromXDR(trade.AssetSold())] = history.Asset{ID: int64(10000 + i)}
+			s.assetToID[history.AssetKeyFromXDR(trade.AssetBought())] = history.Asset{ID: int64(100 + i)}
 		} else {
-			s.assetToID[trade.AssetSold().String()] = history.Asset{ID: int64(100 + i)}
-			s.assetToID[trade.AssetBought().String()] = history.Asset{ID: int64(10000 + i)}
+			s.assetToID[history.AssetKeyFromXDR(trade.AssetSold())] = history.Asset{ID: int64(100 + i)}
+			s.assetToID[history.AssetKeyFromXDR(trade.AssetBought())] = history.Asset{ID: int64(10000 + i)}
 		}
 		s.assets = append(s.assets, trade.AssetSold(), trade.AssetBought())
 	}
 
-	s.processor = NewTradeProcessor(
-		s.mockSession,
-		s.mockQ,
-		xdr.LedgerHeaderHistoryEntry{
-			Header: xdr.LedgerHeader{
-				LedgerSeq: 100,
+	s.lcm = xdr.LedgerCloseMeta{
+		V0: &xdr.LedgerCloseMetaV0{
+			LedgerHeader: xdr.LedgerHeaderHistoryEntry{
+				Header: xdr.LedgerHeader{
+					LedgerSeq: xdr.Uint32(100),
+				},
 			},
 		},
+	}
+
+	s.accountLoader = history.NewAccountLoaderStub()
+	s.assetLoader = history.NewAssetLoaderStub()
+	s.lpLoader = history.NewLiquidityPoolLoaderStub()
+	s.processor = NewTradeProcessor(
+		s.accountLoader.Loader,
+		s.lpLoader.Loader,
+		s.assetLoader.Loader,
+		s.mockBatchInsertBuilder,
 	)
 }
 
 func (s *TradeProcessorTestSuiteLedger) TearDownTest() {
-	s.mockQ.AssertExpectations(s.T())
 	s.mockBatchInsertBuilder.AssertExpectations(s.T())
 }
 
 func (s *TradeProcessorTestSuiteLedger) TestIgnoreFailedTransactions() {
 	ctx := context.Background()
-	err := s.processor.ProcessTransaction(ctx, createTransaction(false, 1))
+	err := s.processor.ProcessTransaction(s.lcm, createTransaction(false, 1))
 	s.Assert().NoError(err)
 
-	err = s.processor.Commit(ctx)
+	err = s.processor.Flush(ctx, s.mockSession)
 	s.Assert().NoError(err)
 }
 
-func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions(
-	ledger xdr.LedgerHeaderHistoryEntry,
-) []history.InsertTrade {
+func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions() []history.InsertTrade {
+	ledger := s.lcm.LedgerHeaderHistoryEntry()
 	closeTime := time.Unix(int64(ledger.Header.ScpValue.CloseTime), 0).UTC()
 	inserts := []history.InsertTrade{
 		{
@@ -235,11 +246,11 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions(
 			LedgerCloseTime:    closeTime,
 			BaseAmount:         int64(s.strictReceiveTrade.AmountBought()),
 			BaseAccountID:      null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]),
-			BaseAssetID:        s.assetToID[s.strictReceiveTrade.AssetBought().String()].ID,
+			BaseAssetID:        s.assetToID[history.AssetKeyFromXDR(s.strictReceiveTrade.AssetBought())].ID,
 			BaseOfferID:        null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 2).ToInt64()), TOIDType)),
 			CounterAmount:      int64(s.strictReceiveTrade.AmountSold()),
 			CounterAccountID:   null.IntFrom(s.unmuxedAccountToID[s.strictReceiveTrade.SellerId().Address()]),
-			CounterAssetID:     s.assetToID[s.strictReceiveTrade.AssetSold().String()].ID,
+			CounterAssetID:     s.assetToID[history.AssetKeyFromXDR(s.strictReceiveTrade.AssetSold())].ID,
 			CounterOfferID:     null.IntFrom(int64(s.strictReceiveTrade.OfferId())),
 			BaseIsSeller:       false,
 			BaseIsExact:        null.BoolFrom(false),
@@ -253,11 +264,11 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions(
 			LedgerCloseTime:    closeTime,
 			CounterAmount:      int64(s.strictSendTrade.AmountBought()),
 			CounterAccountID:   null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]),
-			CounterAssetID:     s.assetToID[s.strictSendTrade.AssetBought().String()].ID,
+			CounterAssetID:     s.assetToID[history.AssetKeyFromXDR(s.strictSendTrade.AssetBought())].ID,
 			CounterOfferID:     null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 3).ToInt64()), TOIDType)),
 			BaseAmount:         int64(s.strictSendTrade.AmountSold()),
 			BaseAccountID:      null.IntFrom(s.unmuxedAccountToID[s.strictSendTrade.SellerId().Address()]),
-			BaseAssetID:        s.assetToID[s.strictSendTrade.AssetSold().String()].ID,
+			BaseAssetID:        s.assetToID[history.AssetKeyFromXDR(s.strictSendTrade.AssetSold())].ID,
 			BaseIsSeller:       true,
 			BaseIsExact:        null.BoolFrom(false),
 			BaseOfferID:        null.IntFrom(int64(s.strictSendTrade.OfferId())),
@@ -272,10 +283,10 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions(
 			BaseOfferID:        null.IntFrom(879136),
 			BaseAmount:         int64(s.buyOfferTrade.AmountBought()),
 			BaseAccountID:      null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]),
-			BaseAssetID:        s.assetToID[s.buyOfferTrade.AssetBought().String()].ID,
+			BaseAssetID:        s.assetToID[history.AssetKeyFromXDR(s.buyOfferTrade.AssetBought())].ID,
 			CounterAmount:      int64(s.buyOfferTrade.AmountSold()),
 			CounterAccountID:   null.IntFrom(s.unmuxedAccountToID[s.buyOfferTrade.SellerId().Address()]),
-			CounterAssetID:     s.assetToID[s.buyOfferTrade.AssetSold().String()].ID,
+			CounterAssetID:     s.assetToID[history.AssetKeyFromXDR(s.buyOfferTrade.AssetSold())].ID,
 			BaseIsSeller:       false,
 			CounterOfferID:     null.IntFrom(int64(s.buyOfferTrade.OfferId())),
 			PriceN:             int64(s.sellPrices[2].D),
@@ -287,12 +298,12 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions(
 			Order:              2,
 			LedgerCloseTime:    closeTime,
 			CounterAmount:      int64(s.sellOfferTrade.AmountBought()),
-			CounterAssetID:     s.assetToID[s.sellOfferTrade.AssetBought().String()].ID,
+			CounterAssetID:     s.assetToID[history.AssetKeyFromXDR(s.sellOfferTrade.AssetBought())].ID,
 			CounterAccountID:   null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]),
 			CounterOfferID:     null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 5).ToInt64()), TOIDType)),
 			BaseAmount:         int64(s.sellOfferTrade.AmountSold()),
 			BaseAccountID:      null.IntFrom(s.unmuxedAccountToID[s.sellOfferTrade.SellerId().Address()]),
-			BaseAssetID:        s.assetToID[s.sellOfferTrade.AssetSold().String()].ID,
+			BaseAssetID:        s.assetToID[history.AssetKeyFromXDR(s.sellOfferTrade.AssetSold())].ID,
 			BaseIsSeller:       true,
 			BaseOfferID:        null.IntFrom(int64(s.sellOfferTrade.OfferId())),
 			PriceN:             int64(s.sellPrices[3].N),
@@ -304,12 +315,12 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions(
 			Order:              0,
 			LedgerCloseTime:    closeTime,
 			BaseAmount:         int64(s.passiveSellOfferTrade.AmountBought()),
-			BaseAssetID:        s.assetToID[s.passiveSellOfferTrade.AssetBought().String()].ID,
+			BaseAssetID:        s.assetToID[history.AssetKeyFromXDR(s.passiveSellOfferTrade.AssetBought())].ID,
 			BaseAccountID:      null.IntFrom(s.unmuxedAccountToID[s.unmuxedSourceAccount.Address()]),
 			BaseOfferID:        null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 6).ToInt64()), TOIDType)),
 			CounterAmount:      int64(s.passiveSellOfferTrade.AmountSold()),
 			CounterAccountID:   null.IntFrom(s.unmuxedAccountToID[s.passiveSellOfferTrade.SellerId().Address()]),
-			CounterAssetID:     s.assetToID[s.passiveSellOfferTrade.AssetSold().String()].ID,
+			CounterAssetID:     s.assetToID[history.AssetKeyFromXDR(s.passiveSellOfferTrade.AssetSold())].ID,
 			BaseIsSeller:       false,
 			CounterOfferID:     null.IntFrom(int64(s.passiveSellOfferTrade.OfferId())),
 			PriceN:             int64(s.sellPrices[4].D),
@@ -323,12 +334,12 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions(
 			LedgerCloseTime:    closeTime,
 
 			CounterAmount:    int64(s.otherPassiveSellOfferTrade.AmountBought()),
-			CounterAssetID:   s.assetToID[s.otherPassiveSellOfferTrade.AssetBought().String()].ID,
+			CounterAssetID:   s.assetToID[history.AssetKeyFromXDR(s.otherPassiveSellOfferTrade.AssetBought())].ID,
 			CounterAccountID: null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]),
 			CounterOfferID:   null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 7).ToInt64()), TOIDType)),
 			BaseAmount:       int64(s.otherPassiveSellOfferTrade.AmountSold()),
 			BaseAccountID:    null.IntFrom(s.unmuxedAccountToID[s.otherPassiveSellOfferTrade.SellerId().Address()]),
-			BaseAssetID:      s.assetToID[s.otherPassiveSellOfferTrade.AssetSold().String()].ID,
+			BaseAssetID:      s.assetToID[history.AssetKeyFromXDR(s.otherPassiveSellOfferTrade.AssetSold())].ID,
 			BaseIsSeller:     true,
 			BaseOfferID:      null.IntFrom(int64(s.otherPassiveSellOfferTrade.OfferId())),
 			PriceN:           int64(s.sellPrices[5].N),
@@ -340,12 +351,12 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions(
 			Order:                  1,
 			LedgerCloseTime:        closeTime,
 			BaseAmount:             int64(s.strictReceiveTradeLP.AmountBought()),
-			BaseAssetID:            s.assetToID[s.strictReceiveTradeLP.AssetBought().String()].ID,
+			BaseAssetID:            s.assetToID[history.AssetKeyFromXDR(s.strictReceiveTradeLP.AssetBought())].ID,
 			BaseAccountID:          null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]),
 			BaseOfferID:            null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 8).ToInt64()), TOIDType)),
 			CounterAmount:          int64(s.strictReceiveTradeLP.AmountSold()),
 			CounterLiquidityPoolID: null.IntFrom(s.lpToID[s.strictReceiveTradeLP.MustLiquidityPool().LiquidityPoolId]),
-			CounterAssetID:         s.assetToID[s.strictReceiveTradeLP.AssetSold().String()].ID,
+			CounterAssetID:         s.assetToID[history.AssetKeyFromXDR(s.strictReceiveTradeLP.AssetSold())].ID,
 			BaseIsSeller:           false,
 			BaseIsExact:            null.BoolFrom(false),
 			LiquidityPoolFee:       null.IntFrom(int64(xdr.LiquidityPoolFeeV18)),
@@ -359,12 +370,12 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions(
 			Order:               0,
 			LedgerCloseTime:     closeTime,
 			CounterAmount:       int64(s.strictSendTradeLP.AmountBought()),
-			CounterAssetID:      s.assetToID[s.strictSendTradeLP.AssetBought().String()].ID,
+			CounterAssetID:      s.assetToID[history.AssetKeyFromXDR(s.strictSendTradeLP.AssetBought())].ID,
 			CounterAccountID:    null.IntFrom(s.unmuxedAccountToID[s.unmuxedOpSourceAccount.Address()]),
 			CounterOfferID:      null.IntFrom(EncodeOfferId(uint64(toid.New(int32(ledger.Header.LedgerSeq), 1, 9).ToInt64()), TOIDType)),
 			BaseAmount:          int64(s.strictSendTradeLP.AmountSold()),
 			BaseLiquidityPoolID: null.IntFrom(s.lpToID[s.strictSendTradeLP.MustLiquidityPool().LiquidityPoolId]),
-			BaseAssetID:         s.assetToID[s.strictSendTradeLP.AssetSold().String()].ID,
+			BaseAssetID:         s.assetToID[history.AssetKeyFromXDR(s.strictSendTradeLP.AssetSold())].ID,
 			BaseIsSeller:        true,
 			BaseIsExact:         null.BoolFrom(false),
 			LiquidityPoolFee:    null.IntFrom(int64(xdr.LiquidityPoolFeeV18)),
@@ -712,243 +723,75 @@ func (s *TradeProcessorTestSuiteLedger) mockReadTradeTransactions(
 		tx,
 	}
 
-	s.mockQ.On("NewTradeBatchInsertBuilder").
-		Return(s.mockBatchInsertBuilder).Once()
-
 	return inserts
 }
 
-func mapKeysToList(set map[string]int64) []string {
-	keys := make([]string, 0, len(set))
-	for key := range set {
-		keys = append(keys, key)
+func (s *TradeProcessorTestSuiteLedger) stubLoaders() {
+	for key, id := range s.unmuxedAccountToID {
+		s.accountLoader.Insert(key, id)
 	}
-	return keys
-}
-
-func uniq(list []string) []string {
-	var deduped []string
-	set := map[string]bool{}
-	for _, s := range list {
-		if set[s] {
-			continue
-		}
-		deduped = append(deduped, s)
-		set[s] = true
+	for key, id := range s.assetToID {
+		s.assetLoader.Insert(key, id.ID)
+	}
+	for key, id := range s.lpToID {
+		s.lpLoader.Insert(PoolIDToString(key), id)
 	}
-	return deduped
 }
 
 func (s *TradeProcessorTestSuiteLedger) TestIngestTradesSucceeds() {
 	ctx := context.Background()
-	inserts := s.mockReadTradeTransactions(s.processor.ledger)
+	inserts := s.mockReadTradeTransactions()
 
-	s.mockCreateAccounts(ctx)
-
-	s.mockCreateAssets(ctx)
-
-	s.mockCreateHistoryLiquidityPools(ctx)
+	for _, tx := range s.txs {
+		err := s.processor.ProcessTransaction(s.lcm, tx)
+		s.Assert().NoError(err)
+	}
 
 	for _, insert := range inserts {
 		s.mockBatchInsertBuilder.On("Add", []history.InsertTrade{
 			insert,
 		}).Return(nil).Once()
 	}
-
 	s.mockBatchInsertBuilder.On("Exec", ctx, s.mockSession).Return(nil).Once()
+	s.stubLoaders()
 
-	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(ctx, tx)
-		s.Assert().NoError(err)
-	}
-
-	err := s.processor.Commit(ctx)
+	err := s.processor.Flush(ctx, s.mockSession)
 	s.Assert().NoError(err)
 }
 
-func (s *TradeProcessorTestSuiteLedger) mockCreateHistoryLiquidityPools(ctx context.Context) {
-	lpIDs, lpStrToID := s.extractLpIDs()
-	s.mockQ.On("CreateHistoryLiquidityPools", ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]string)
-			s.Assert().ElementsMatch(
-				lpIDs,
-				arg,
-			)
-		}).Return(lpStrToID, nil).Once()
-}
-
-func (s *TradeProcessorTestSuiteLedger) extractLpIDs() ([]string, map[string]int64) {
-	var lpIDs []string
-	lpStrToID := map[string]int64{}
-	for lpID, id := range s.lpToID {
-		lpIDStr := PoolIDToString(lpID)
-		lpIDs = append(lpIDs, lpIDStr)
-		lpStrToID[lpIDStr] = id
-	}
-	return lpIDs, lpStrToID
-}
-
-func (s *TradeProcessorTestSuiteLedger) TestCreateAccountsError() {
-	ctx := context.Background()
-	s.mockReadTradeTransactions(s.processor.ledger)
-
-	s.mockQ.On("CreateAccounts", ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]string)
-			s.Assert().ElementsMatch(
-				mapKeysToList(s.unmuxedAccountToID),
-				uniq(arg),
-			)
-		}).Return(map[string]int64{}, fmt.Errorf("create accounts error")).Once()
-
-	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(ctx, tx)
-		s.Assert().NoError(err)
-	}
-
-	err := s.processor.Commit(ctx)
-
-	s.Assert().EqualError(err, "Error creating account ids: create accounts error")
-}
-
-func (s *TradeProcessorTestSuiteLedger) TestCreateAssetsError() {
-	ctx := context.Background()
-	s.mockReadTradeTransactions(s.processor.ledger)
-
-	s.mockCreateAccounts(ctx)
-
-	s.mockQ.On("CreateAssets", ctx, mock.AnythingOfType("[]xdr.Asset"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]xdr.Asset)
-			s.Assert().ElementsMatch(
-				s.assets,
-				arg,
-			)
-		}).Return(s.assetToID, fmt.Errorf("create assets error")).Once()
-
-	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(ctx, tx)
-		s.Assert().NoError(err)
-	}
-
-	err := s.processor.Commit(ctx)
-	s.Assert().EqualError(err, "Error creating asset ids: create assets error")
-}
-
-func (s *TradeProcessorTestSuiteLedger) TestCreateHistoryLiquidityPoolsError() {
+func (s *TradeProcessorTestSuiteLedger) TestBatchAddError() {
 	ctx := context.Background()
-	s.mockReadTradeTransactions(s.processor.ledger)
-
-	s.mockCreateAccounts(ctx)
-
-	s.mockCreateAssets(ctx)
-
-	lpIDs, lpStrToID := s.extractLpIDs()
-	s.mockQ.On("CreateHistoryLiquidityPools", ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]string)
-			s.Assert().ElementsMatch(
-				lpIDs,
-				arg,
-			)
-		}).Return(lpStrToID, fmt.Errorf("create liqudity pool id error")).Once()
+	s.mockReadTradeTransactions()
 
 	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(ctx, tx)
+		err := s.processor.ProcessTransaction(s.lcm, tx)
 		s.Assert().NoError(err)
 	}
 
-	err := s.processor.Commit(ctx)
-	s.Assert().EqualError(err, "Error creating pool ids: create liqudity pool id error")
-}
-
-func (s *TradeProcessorTestSuiteLedger) mockCreateAssets(ctx context.Context) {
-	s.mockQ.On("CreateAssets", ctx, mock.AnythingOfType("[]xdr.Asset"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]xdr.Asset)
-			s.Assert().ElementsMatch(
-				s.assets,
-				arg,
-			)
-		}).Return(s.assetToID, nil).Once()
-}
-
-func (s *TradeProcessorTestSuiteLedger) mockCreateAccounts(ctx context.Context) {
-	s.mockQ.On("CreateAccounts", ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Run(func(args mock.Arguments) {
-			arg := args.Get(1).([]string)
-			s.Assert().ElementsMatch(
-				mapKeysToList(s.unmuxedAccountToID),
-				uniq(arg),
-			)
-		}).Return(s.unmuxedAccountToID, nil).Once()
-}
-
-func (s *TradeProcessorTestSuiteLedger) TestBatchAddError() {
-	ctx := context.Background()
-	s.mockReadTradeTransactions(s.processor.ledger)
-
-	s.mockCreateAccounts(ctx)
-
-	s.mockCreateAssets(ctx)
-
-	s.mockCreateHistoryLiquidityPools(ctx)
-
+	s.stubLoaders()
 	s.mockBatchInsertBuilder.On("Add", mock.AnythingOfType("[]history.InsertTrade")).
 		Return(fmt.Errorf("batch add error")).Once()
 
-	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(ctx, tx)
-		s.Assert().NoError(err)
-	}
-
-	err := s.processor.Commit(ctx)
+	err := s.processor.Flush(ctx, s.mockSession)
 	s.Assert().EqualError(err, "Error adding trade to batch: batch add error")
 }
 
 func (s *TradeProcessorTestSuiteLedger) TestBatchExecError() {
 	ctx := context.Background()
-	insert := s.mockReadTradeTransactions(s.processor.ledger)
-
-	s.mockCreateAccounts(ctx)
-
-	s.mockCreateAssets(ctx)
+	insert := s.mockReadTradeTransactions()
 
-	s.mockCreateHistoryLiquidityPools(ctx)
-
-	s.mockBatchInsertBuilder.On("Add", mock.AnythingOfType("[]history.InsertTrade")).
-		Return(nil).Times(len(insert))
-	s.mockBatchInsertBuilder.On("Exec", ctx, s.mockSession).Return(fmt.Errorf("exec error")).Once()
 	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(ctx, tx)
+		err := s.processor.ProcessTransaction(s.lcm, tx)
 		s.Assert().NoError(err)
 	}
 
-	err := s.processor.Commit(ctx)
-	s.Assert().EqualError(err, "Error flushing operation batch: exec error")
-}
-
-func (s *TradeProcessorTestSuiteLedger) TestIgnoreCheckIfSmallLedger() {
-	ctx := context.Background()
-	insert := s.mockReadTradeTransactions(s.processor.ledger)
-
-	s.mockCreateAccounts(ctx)
-
-	s.mockCreateAssets(ctx)
-
-	s.mockCreateHistoryLiquidityPools(ctx)
+	s.stubLoaders()
 	s.mockBatchInsertBuilder.On("Add", mock.AnythingOfType("[]history.InsertTrade")).
 		Return(nil).Times(len(insert))
-	s.mockBatchInsertBuilder.On("Exec", ctx, s.mockSession).Return(nil).Once()
-
-	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(ctx, tx)
-		s.Assert().NoError(err)
-	}
+	s.mockBatchInsertBuilder.On("Exec", ctx, s.mockSession).Return(fmt.Errorf("exec error")).Once()
 
-	err := s.processor.Commit(ctx)
-	s.Assert().NoError(err)
+	err := s.processor.Flush(ctx, s.mockSession)
+	s.Assert().EqualError(err, "Error flushing operation batch: exec error")
 }
 
 func TestTradeProcessor_ProcessTransaction_MuxedAccount(t *testing.T) {
@@ -976,7 +819,7 @@ func TestTradeProcessor_RoundingSlippage_Big(t *testing.T) {
 	s := &TradeProcessorTestSuiteLedger{}
 	s.SetT(t)
 	s.SetupTest()
-	s.mockReadTradeTransactions(s.processor.ledger)
+	s.mockReadTradeTransactions()
 
 	assetDeposited := xdr.MustNewCreditAsset("MAD", s.unmuxedSourceAccount.Address())
 	assetDisbursed := xdr.MustNewCreditAsset("GRE", s.unmuxedSourceAccount.Address())
@@ -1008,7 +851,7 @@ func TestTradeProcessor_RoundingSlippage_Small(t *testing.T) {
 	s := &TradeProcessorTestSuiteLedger{}
 	s.SetT(t)
 	s.SetupTest()
-	s.mockReadTradeTransactions(s.processor.ledger)
+	s.mockReadTradeTransactions()
 
 	assetDeposited := xdr.MustNewCreditAsset("MAD", s.unmuxedSourceAccount.Address())
 	assetDisbursed := xdr.MustNewCreditAsset("GRE", s.unmuxedSourceAccount.Address())

From 877564802ce10ee6c7e6b9121d0a3ee7f471a84a Mon Sep 17 00:00:00 2001
From: tamirms <tamirms@gmail.com>
Date: Thu, 31 Aug 2023 09:23:32 +0100
Subject: [PATCH 12/17] services/horizon/internal/ingest/processors: Refactor
 effects processor to support new ingestion data flow (#5028)

---
 .../history/effect_batch_insert_builder.go    |   5 +-
 .../effect_batch_insert_builder_test.go       |   7 +-
 .../internal/db2/history/effect_test.go       |  35 +-
 .../internal/db2/history/fee_bump_scenario.go |   9 +-
 .../mock_effect_batch_insert_builder.go       |   2 +-
 .../ingest/processors/effects_processor.go    | 570 ++++++++++--------
 .../processors/effects_processor_test.go      | 215 +++----
 .../ingest/processors/operations_processor.go |  68 ++-
 8 files changed, 474 insertions(+), 437 deletions(-)

diff --git a/services/horizon/internal/db2/history/effect_batch_insert_builder.go b/services/horizon/internal/db2/history/effect_batch_insert_builder.go
index e3e5896e7f..bd9aa0687a 100644
--- a/services/horizon/internal/db2/history/effect_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/effect_batch_insert_builder.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/guregu/null"
+
 	"github.com/stellar/go/support/db"
 )
 
@@ -11,7 +12,7 @@ import (
 // history_effects table
 type EffectBatchInsertBuilder interface {
 	Add(
-		accountID int64,
+		accountID FutureAccountID,
 		muxedAccount null.String,
 		operationID int64,
 		order uint32,
@@ -37,7 +38,7 @@ func (q *Q) NewEffectBatchInsertBuilder() EffectBatchInsertBuilder {
 
 // Add adds a effect to the batch
 func (i *effectBatchInsertBuilder) Add(
-	accountID int64,
+	accountID FutureAccountID,
 	muxedAccount null.String,
 	operationID int64,
 	order uint32,
diff --git a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go
index 78988db2b4..cef5f3745d 100644
--- a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go
+++ b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go
@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/guregu/null"
+
 	"github.com/stellar/go/services/horizon/internal/test"
 	"github.com/stellar/go/toid"
 )
@@ -18,8 +19,7 @@ func TestAddEffect(t *testing.T) {
 
 	address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY"
 	muxedAddres := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26"
-	accounIDs, err := q.CreateAccounts(tt.Ctx, []string{address}, 1)
-	tt.Assert.NoError(err)
+	accountLoader := NewAccountLoader()
 
 	builder := q.NewEffectBatchInsertBuilder()
 	sequence := int32(56)
@@ -29,7 +29,7 @@ func TestAddEffect(t *testing.T) {
 	})
 
 	err = builder.Add(
-		accounIDs[address],
+		accountLoader.GetFuture(address),
 		null.StringFrom(muxedAddres),
 		toid.New(sequence, 1, 1).ToInt64(),
 		1,
@@ -38,6 +38,7 @@ func TestAddEffect(t *testing.T) {
 	)
 	tt.Assert.NoError(err)
 
+	tt.Assert.NoError(accountLoader.Exec(tt.Ctx, q))
 	tt.Assert.NoError(builder.Exec(tt.Ctx, q))
 	tt.Assert.NoError(q.Commit())
 
diff --git a/services/horizon/internal/db2/history/effect_test.go b/services/horizon/internal/db2/history/effect_test.go
index bf893a100b..96d68208cb 100644
--- a/services/horizon/internal/db2/history/effect_test.go
+++ b/services/horizon/internal/db2/history/effect_test.go
@@ -23,8 +23,7 @@ func TestEffectsForLiquidityPool(t *testing.T) {
 	// Insert Effect
 	address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY"
 	muxedAddres := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26"
-	accountIDs, err := q.CreateAccounts(tt.Ctx, []string{address}, 1)
-	tt.Assert.NoError(err)
+	accountLoader := NewAccountLoader()
 
 	builder := q.NewEffectBatchInsertBuilder()
 	sequence := int32(56)
@@ -32,19 +31,19 @@ func TestEffectsForLiquidityPool(t *testing.T) {
 		"amount":     "1000.0000000",
 		"asset_type": "native",
 	})
+	tt.Assert.NoError(err)
 	opID := toid.New(sequence, 1, 1).ToInt64()
-	err = builder.Add(
-		accountIDs[address],
+	tt.Assert.NoError(builder.Add(
+		accountLoader.GetFuture(address),
 		null.StringFrom(muxedAddres),
 		opID,
 		1,
 		3,
 		details,
-	)
-	tt.Assert.NoError(err)
+	))
 
-	err = builder.Exec(tt.Ctx, q)
-	tt.Assert.NoError(err)
+	tt.Assert.NoError(accountLoader.Exec(tt.Ctx, q))
+	tt.Assert.NoError(builder.Exec(tt.Ctx, q))
 
 	// Insert Liquidity Pool history
 	liquidityPoolID := "abcde"
@@ -79,8 +78,7 @@ func TestEffectsForTrustlinesSponsorshipEmptyAssetType(t *testing.T) {
 
 	address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY"
 	muxedAddres := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26"
-	accountIDs, err := q.CreateAccounts(tt.Ctx, []string{address}, 1)
-	tt.Assert.NoError(err)
+	accountLoader := NewAccountLoader()
 
 	builder := q.NewEffectBatchInsertBuilder()
 	sequence := int32(56)
@@ -142,27 +140,24 @@ func TestEffectsForTrustlinesSponsorshipEmptyAssetType(t *testing.T) {
 
 	for i, test := range tests {
 		var bytes []byte
-		bytes, err = json.Marshal(test.details)
+		bytes, err := json.Marshal(test.details)
 		tt.Require.NoError(err)
 
-		err = builder.Add(
-			accountIDs[address],
+		tt.Require.NoError(builder.Add(
+			accountLoader.GetFuture(address),
 			null.StringFrom(muxedAddres),
 			opID,
 			uint32(i),
 			test.effectType,
 			bytes,
-		)
-		tt.Require.NoError(err)
+		))
 	}
-
-	err = builder.Exec(tt.Ctx, q)
-	tt.Require.NoError(err)
+	tt.Require.NoError(accountLoader.Exec(tt.Ctx, q))
+	tt.Require.NoError(builder.Exec(tt.Ctx, q))
 	tt.Assert.NoError(q.Commit())
 
 	var results []Effect
-	err = q.Effects().Select(tt.Ctx, &results)
-	tt.Require.NoError(err)
+	tt.Require.NoError(q.Effects().Select(tt.Ctx, &results))
 	tt.Require.Len(results, len(tests))
 
 	for i, test := range tests {
diff --git a/services/horizon/internal/db2/history/fee_bump_scenario.go b/services/horizon/internal/db2/history/fee_bump_scenario.go
index 95fc7ac1b4..5d155ac5e8 100644
--- a/services/horizon/internal/db2/history/fee_bump_scenario.go
+++ b/services/horizon/internal/db2/history/fee_bump_scenario.go
@@ -10,12 +10,13 @@ import (
 	sq "github.com/Masterminds/squirrel"
 	"github.com/guregu/null"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/network"
 	"github.com/stellar/go/services/horizon/internal/test"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
-	"github.com/stretchr/testify/assert"
 )
 
 func ledgerToMap(ledger Ledger) map[string]interface{} {
@@ -285,11 +286,10 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 	details, err = json.Marshal(map[string]interface{}{"new_seq": 98})
 	tt.Assert.NoError(err)
 
-	accounIDs, err := q.CreateAccounts(ctx, []string{account.Address()}, 1)
-	tt.Assert.NoError(err)
+	accountLoader := NewAccountLoader()
 
 	err = effectBuilder.Add(
-		accounIDs[account.Address()],
+		accountLoader.GetFuture(account.Address()),
 		null.String{},
 		toid.New(fixture.Ledger.Sequence, 1, 1).ToInt64(),
 		1,
@@ -297,6 +297,7 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 		details,
 	)
 	tt.Assert.NoError(err)
+	tt.Assert.NoError(accountLoader.Exec(ctx, q))
 	tt.Assert.NoError(effectBuilder.Exec(ctx, q))
 
 	tt.Assert.NoError(q.Commit())
diff --git a/services/horizon/internal/db2/history/mock_effect_batch_insert_builder.go b/services/horizon/internal/db2/history/mock_effect_batch_insert_builder.go
index f97e4f5a0d..35168a775a 100644
--- a/services/horizon/internal/db2/history/mock_effect_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/mock_effect_batch_insert_builder.go
@@ -16,7 +16,7 @@ type MockEffectBatchInsertBuilder struct {
 
 // Add mock
 func (m *MockEffectBatchInsertBuilder) Add(
-	accountID int64,
+	accountID FutureAccountID,
 	muxedAccount null.String,
 	operationID int64,
 	order uint32,
diff --git a/services/horizon/internal/ingest/processors/effects_processor.go b/services/horizon/internal/ingest/processors/effects_processor.go
index f7703016e3..6382461ef6 100644
--- a/services/horizon/internal/ingest/processors/effects_processor.go
+++ b/services/horizon/internal/ingest/processors/effects_processor.go
@@ -10,6 +10,7 @@ import (
 	"strconv"
 
 	"github.com/guregu/null"
+
 	"github.com/stellar/go/amount"
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/keypair"
@@ -22,148 +23,51 @@ import (
 
 // EffectProcessor process effects
 type EffectProcessor struct {
-	effects  []effect
-	session  db.SessionInterface
-	effectsQ history.QEffects
-	sequence uint32
+	accountLoader *history.AccountLoader
+	batch         history.EffectBatchInsertBuilder
 }
 
-func NewEffectProcessor(session db.SessionInterface, effectsQ history.QEffects, sequence uint32) *EffectProcessor {
+func NewEffectProcessor(
+	accountLoader *history.AccountLoader,
+	batch history.EffectBatchInsertBuilder,
+) *EffectProcessor {
 	return &EffectProcessor{
-		session:  session,
-		effectsQ: effectsQ,
-		sequence: sequence,
+		accountLoader: accountLoader,
+		batch:         batch,
 	}
 }
 
-func (p *EffectProcessor) loadAccountIDs(ctx context.Context, accountSet map[string]int64) error {
-	addresses := make([]string, 0, len(accountSet))
-	for address := range accountSet {
-		addresses = append(addresses, address)
-	}
-
-	addressToID, err := p.effectsQ.CreateAccounts(ctx, addresses, maxBatchSize)
-	if err != nil {
-		return errors.Wrap(err, "Could not create account ids")
-	}
-
-	for _, address := range addresses {
-		id, ok := addressToID[address]
-		if !ok {
-			return errors.Errorf("no id found for account address %s", address)
-		}
-
-		accountSet[address] = id
+func (p *EffectProcessor) ProcessTransaction(
+	lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction,
+) error {
+	// Failed transactions don't have operation effects
+	if !transaction.Result.Successful() {
+		return nil
 	}
 
-	return nil
-}
-
-func operationsEffects(transaction ingest.LedgerTransaction, sequence uint32) ([]effect, error) {
-	effects := []effect{}
-
 	for opi, op := range transaction.Envelope.Operations() {
 		operation := transactionOperationWrapper{
 			index:          uint32(opi),
 			transaction:    transaction,
 			operation:      op,
-			ledgerSequence: sequence,
+			ledgerSequence: lcm.LedgerSequence(),
 		}
-
-		p, err := operation.effects()
-		if err != nil {
-			return effects, errors.Wrapf(err, "reading operation %v effects", operation.ID())
+		if err := operation.ingestEffects(p.accountLoader, p.batch); err != nil {
+			return errors.Wrapf(err, "reading operation %v effects", operation.ID())
 		}
-		effects = append(effects, p...)
 	}
 
-	return effects, nil
-}
-
-func (p *EffectProcessor) insertDBOperationsEffects(ctx context.Context, effects []effect, accountSet map[string]int64) error {
-	batch := p.effectsQ.NewEffectBatchInsertBuilder()
-
-	for _, effect := range effects {
-		accountID, found := accountSet[effect.address]
-
-		if !found {
-			return errors.Errorf("Error finding history_account_id for address %v", effect.address)
-		}
-
-		var detailsJSON []byte
-		detailsJSON, err := json.Marshal(effect.details)
-
-		if err != nil {
-			return errors.Wrapf(err, "Error marshaling details for operation effect %v", effect.operationID)
-		}
-
-		if err := batch.Add(
-			accountID,
-			effect.addressMuxed,
-			effect.operationID,
-			effect.order,
-			effect.effectType,
-			detailsJSON,
-		); err != nil {
-			return errors.Wrap(err, "could not insert operation effect in db")
-		}
-	}
-
-	if err := batch.Exec(ctx, p.session); err != nil {
-		return errors.Wrap(err, "could not flush operation effects to db")
-	}
 	return nil
 }
 
-func (p *EffectProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) (err error) {
-	// Failed transactions don't have operation effects
-	if !transaction.Result.Successful() {
-		return nil
-	}
-
-	var effectsForTx []effect
-	effectsForTx, err = operationsEffects(transaction, p.sequence)
-	if err != nil {
-		return err
-	}
-	p.effects = append(p.effects, effectsForTx...)
-
-	return nil
+func (p *EffectProcessor) Flush(ctx context.Context, session db.SessionInterface) (err error) {
+	return p.batch.Exec(ctx, session)
 }
 
-func (p *EffectProcessor) Commit(ctx context.Context) (err error) {
-	if len(p.effects) > 0 {
-		accountSet := map[string]int64{}
-
-		for _, effect := range p.effects {
-			accountSet[effect.address] = 0
-		}
-
-		if err = p.loadAccountIDs(ctx, accountSet); err != nil {
-			return err
-		}
-
-		if err = p.insertDBOperationsEffects(ctx, p.effects, accountSet); err != nil {
-			return err
-		}
-	}
-
-	return err
-}
-
-type effect struct {
-	address      string
-	addressMuxed null.String
-	operationID  int64
-	details      map[string]interface{}
-	effectType   history.EffectType
-	order        uint32
-}
-
-// Effects returns the operation effects
-func (operation *transactionOperationWrapper) effects() ([]effect, error) {
+// ingestEffects adds effects from the operation to the given EffectBatchInsertBuilder
+func (operation *transactionOperationWrapper) ingestEffects(accountLoader *history.AccountLoader, batch history.EffectBatchInsertBuilder) error {
 	if !operation.transaction.Result.Successful() {
-		return []effect{}, nil
+		return nil
 	}
 	var (
 		op  = operation.operation
@@ -172,19 +76,21 @@ func (operation *transactionOperationWrapper) effects() ([]effect, error) {
 
 	changes, err := operation.transaction.GetOperationChanges(operation.index)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	wrapper := &effectsWrapper{
-		effects:   []effect{},
-		operation: operation,
+		accountLoader: accountLoader,
+		batch:         batch,
+		order:         1,
+		operation:     operation,
 	}
 
 	switch operation.OperationType() {
 	case xdr.OperationTypeCreateAccount:
-		wrapper.addAccountCreatedEffects()
+		err = wrapper.addAccountCreatedEffects()
 	case xdr.OperationTypePayment:
-		wrapper.addPaymentEffects()
+		err = wrapper.addPaymentEffects()
 	case xdr.OperationTypePathPaymentStrictReceive:
 		err = wrapper.pathPaymentStrictReceiveEffects()
 	case xdr.OperationTypePathPaymentStrictSend:
@@ -196,15 +102,15 @@ func (operation *transactionOperationWrapper) effects() ([]effect, error) {
 	case xdr.OperationTypeCreatePassiveSellOffer:
 		err = wrapper.addCreatePassiveSellOfferEffect()
 	case xdr.OperationTypeSetOptions:
-		wrapper.addSetOptionsEffects()
+		err = wrapper.addSetOptionsEffects()
 	case xdr.OperationTypeChangeTrust:
 		err = wrapper.addChangeTrustEffects()
 	case xdr.OperationTypeAllowTrust:
 		err = wrapper.addAllowTrustEffects()
 	case xdr.OperationTypeAccountMerge:
-		wrapper.addAccountMergeEffects()
+		err = wrapper.addAccountMergeEffects()
 	case xdr.OperationTypeInflation:
-		wrapper.addInflationEffects()
+		err = wrapper.addInflationEffects()
 	case xdr.OperationTypeManageData:
 		err = wrapper.addManageDataEffects()
 	case xdr.OperationTypeBumpSequence:
@@ -226,10 +132,10 @@ func (operation *transactionOperationWrapper) effects() ([]effect, error) {
 	case xdr.OperationTypeLiquidityPoolWithdraw:
 		err = wrapper.addLiquidityPoolWithdrawEffect()
 	default:
-		return nil, fmt.Errorf("Unknown operation type: %s", op.Body.Type)
+		err = fmt.Errorf("Unknown operation type: %s", op.Body.Type)
 	}
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	// Effects generated for multiple operations. Keep the effect categories
@@ -238,48 +144,63 @@ func (operation *transactionOperationWrapper) effects() ([]effect, error) {
 
 	// Sponsorships
 	for _, change := range changes {
-		if err = wrapper.addLedgerEntrySponsorshipEffects(change); err != nil {
-			return nil, err
+		if err := wrapper.addLedgerEntrySponsorshipEffects(change); err != nil {
+			return err
+		}
+		if err := wrapper.addSignerSponsorshipEffects(change); err != nil {
+			return err
 		}
-		wrapper.addSignerSponsorshipEffects(change)
 	}
 
 	// Liquidity pools
 	for _, change := range changes {
 		// Effects caused by ChangeTrust (creation), AllowTrust and SetTrustlineFlags (removal through revocation)
-		wrapper.addLedgerEntryLiquidityPoolEffects(change)
+		if err := wrapper.addLedgerEntryLiquidityPoolEffects(change); err != nil {
+			return err
+		}
 	}
 
-	return wrapper.effects, nil
+	return nil
 }
 
 type effectsWrapper struct {
-	effects   []effect
-	operation *transactionOperationWrapper
+	accountLoader *history.AccountLoader
+	batch         history.EffectBatchInsertBuilder
+	order         uint32
+	operation     *transactionOperationWrapper
 }
 
-func (e *effectsWrapper) add(address string, addressMuxed null.String, effectType history.EffectType, details map[string]interface{}) {
-	e.effects = append(e.effects, effect{
-		address:      address,
-		addressMuxed: addressMuxed,
-		operationID:  e.operation.ID(),
-		effectType:   effectType,
-		order:        uint32(len(e.effects) + 1),
-		details:      details,
-	})
+func (e *effectsWrapper) add(address string, addressMuxed null.String, effectType history.EffectType, details map[string]interface{}) error {
+	detailsJSON, err := json.Marshal(details)
+	if err != nil {
+		return errors.Wrapf(err, "Error marshaling details for operation effect %v", e.operation.ID())
+	}
+
+	if err := e.batch.Add(
+		e.accountLoader.GetFuture(address),
+		addressMuxed,
+		e.operation.ID(),
+		e.order,
+		effectType,
+		detailsJSON,
+	); err != nil {
+		return errors.Wrap(err, "could not insert operation effect in db")
+	}
+	e.order++
+	return nil
 }
 
-func (e *effectsWrapper) addUnmuxed(address *xdr.AccountId, effectType history.EffectType, details map[string]interface{}) {
-	e.add(address.Address(), null.String{}, effectType, details)
+func (e *effectsWrapper) addUnmuxed(address *xdr.AccountId, effectType history.EffectType, details map[string]interface{}) error {
+	return e.add(address.Address(), null.String{}, effectType, details)
 }
 
-func (e *effectsWrapper) addMuxed(address *xdr.MuxedAccount, effectType history.EffectType, details map[string]interface{}) {
+func (e *effectsWrapper) addMuxed(address *xdr.MuxedAccount, effectType history.EffectType, details map[string]interface{}) error {
 	var addressMuxed null.String
 	if address.Type == xdr.CryptoKeyTypeKeyTypeMuxedEd25519 {
 		addressMuxed = null.StringFrom(address.Address())
 	}
 	accID := address.ToAccountId()
-	e.add(accID.Address(), addressMuxed, effectType, details)
+	return e.add(accID.Address(), addressMuxed, effectType, details)
 }
 
 var sponsoringEffectsTable = map[xdr.LedgerEntryType]struct {
@@ -310,9 +231,9 @@ var sponsoringEffectsTable = map[xdr.LedgerEntryType]struct {
 	// entries because we don't generate creation effects for them.
 }
 
-func (e *effectsWrapper) addSignerSponsorshipEffects(change ingest.Change) {
+func (e *effectsWrapper) addSignerSponsorshipEffects(change ingest.Change) error {
 	if change.Type != xdr.LedgerEntryTypeAccount {
-		return
+		return nil
 	}
 
 	preSigners := map[string]xdr.AccountId{}
@@ -350,12 +271,16 @@ func (e *effectsWrapper) addSignerSponsorshipEffects(change ingest.Change) {
 			details["sponsor"] = post.Address()
 			details["signer"] = signer
 			srcAccount := change.Post.Data.MustAccount().AccountId
-			e.addUnmuxed(&srcAccount, history.EffectSignerSponsorshipCreated, details)
+			if err := e.addUnmuxed(&srcAccount, history.EffectSignerSponsorshipCreated, details); err != nil {
+				return err
+			}
 		case !foundPost && foundPre:
 			details["former_sponsor"] = pre.Address()
 			details["signer"] = signer
 			srcAccount := change.Pre.Data.MustAccount().AccountId
-			e.addUnmuxed(&srcAccount, history.EffectSignerSponsorshipRemoved, details)
+			if err := e.addUnmuxed(&srcAccount, history.EffectSignerSponsorshipRemoved, details); err != nil {
+				return err
+			}
 		case foundPre && foundPost:
 			formerSponsor := pre.Address()
 			newSponsor := post.Address()
@@ -367,9 +292,12 @@ func (e *effectsWrapper) addSignerSponsorshipEffects(change ingest.Change) {
 			details["new_sponsor"] = newSponsor
 			details["signer"] = signer
 			srcAccount := change.Post.Data.MustAccount().AccountId
-			e.addUnmuxed(&srcAccount, history.EffectSignerSponsorshipUpdated, details)
+			if err := e.addUnmuxed(&srcAccount, history.EffectSignerSponsorshipUpdated, details); err != nil {
+				return err
+			}
 		}
 	}
+	return nil
 }
 
 func (e *effectsWrapper) addLedgerEntrySponsorshipEffects(change ingest.Change) error {
@@ -447,9 +375,13 @@ func (e *effectsWrapper) addLedgerEntrySponsorshipEffects(change ingest.Change)
 	}
 
 	if accountID != nil {
-		e.addUnmuxed(accountID, effectType, details)
+		if err := e.addUnmuxed(accountID, effectType, details); err != nil {
+			return err
+		}
 	} else {
-		e.addMuxed(muxedAccount, effectType, details)
+		if err := e.addMuxed(muxedAccount, effectType, details); err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -477,55 +409,64 @@ func (e *effectsWrapper) addLedgerEntryLiquidityPoolEffects(change ingest.Change
 	default:
 		return nil
 	}
-	e.addMuxed(
+	return e.addMuxed(
 		e.operation.SourceAccount(),
 		effectType,
 		details,
 	)
-
-	return nil
 }
 
-func (e *effectsWrapper) addAccountCreatedEffects() {
+func (e *effectsWrapper) addAccountCreatedEffects() error {
 	op := e.operation.operation.Body.MustCreateAccountOp()
 
-	e.addUnmuxed(
+	if err := e.addUnmuxed(
 		&op.Destination,
 		history.EffectAccountCreated,
 		map[string]interface{}{
 			"starting_balance": amount.String(op.StartingBalance),
 		},
-	)
-	e.addMuxed(
+	); err != nil {
+		return err
+	}
+	if err := e.addMuxed(
 		e.operation.SourceAccount(),
 		history.EffectAccountDebited,
 		map[string]interface{}{
 			"asset_type": "native",
 			"amount":     amount.String(op.StartingBalance),
 		},
-	)
-	e.addUnmuxed(
+	); err != nil {
+		return err
+	}
+	if err := e.addUnmuxed(
 		&op.Destination,
 		history.EffectSignerCreated,
 		map[string]interface{}{
 			"public_key": op.Destination.Address(),
 			"weight":     keypair.DefaultSignerWeight,
 		},
-	)
+	); err != nil {
+		return err
+	}
+	return nil
 }
 
-func (e *effectsWrapper) addPaymentEffects() {
+func (e *effectsWrapper) addPaymentEffects() error {
 	op := e.operation.operation.Body.MustPaymentOp()
 
 	details := map[string]interface{}{"amount": amount.String(op.Amount)}
-	addAssetDetails(details, op.Asset, "")
+	if err := addAssetDetails(details, op.Asset, ""); err != nil {
+		return err
+	}
 
-	e.addMuxed(
+	if err := e.addMuxed(
 		&op.Destination,
 		history.EffectAccountCredited,
 		details,
-	)
-	e.addMuxed(
+	); err != nil {
+		return err
+	}
+	return e.addMuxed(
 		e.operation.SourceAccount(),
 		history.EffectAccountDebited,
 		details,
@@ -538,23 +479,31 @@ func (e *effectsWrapper) pathPaymentStrictReceiveEffects() error {
 	source := e.operation.SourceAccount()
 
 	details := map[string]interface{}{"amount": amount.String(op.DestAmount)}
-	addAssetDetails(details, op.DestAsset, "")
+	if err := addAssetDetails(details, op.DestAsset, ""); err != nil {
+		return err
+	}
 
-	e.addMuxed(
+	if err := e.addMuxed(
 		&op.Destination,
 		history.EffectAccountCredited,
 		details,
-	)
+	); err != nil {
+		return err
+	}
 
 	result := e.operation.OperationResult().MustPathPaymentStrictReceiveResult()
 	details = map[string]interface{}{"amount": amount.String(result.SendAmount())}
-	addAssetDetails(details, op.SendAsset, "")
+	if err := addAssetDetails(details, op.SendAsset, ""); err != nil {
+		return err
+	}
 
-	e.addMuxed(
+	if err := e.addMuxed(
 		source,
 		history.EffectAccountDebited,
 		details,
-	)
+	); err != nil {
+		return err
+	}
 
 	return e.addIngestTradeEffects(*source, resultSuccess.Offers)
 }
@@ -566,12 +515,20 @@ func (e *effectsWrapper) addPathPaymentStrictSendEffects() error {
 	result := e.operation.OperationResult().MustPathPaymentStrictSendResult()
 
 	details := map[string]interface{}{"amount": amount.String(result.DestAmount())}
-	addAssetDetails(details, op.DestAsset, "")
-	e.addMuxed(&op.Destination, history.EffectAccountCredited, details)
+	if err := addAssetDetails(details, op.DestAsset, ""); err != nil {
+		return err
+	}
+	if err := e.addMuxed(&op.Destination, history.EffectAccountCredited, details); err != nil {
+		return err
+	}
 
 	details = map[string]interface{}{"amount": amount.String(op.SendAmount)}
-	addAssetDetails(details, op.SendAsset, "")
-	e.addMuxed(source, history.EffectAccountDebited, details)
+	if err := addAssetDetails(details, op.SendAsset, ""); err != nil {
+		return err
+	}
+	if err := e.addMuxed(source, history.EffectAccountDebited, details); err != nil {
+		return err
+	}
 
 	return e.addIngestTradeEffects(*source, resultSuccess.Offers)
 }
@@ -610,11 +567,13 @@ func (e *effectsWrapper) addSetOptionsEffects() error {
 	op := e.operation.operation.Body.MustSetOptionsOp()
 
 	if op.HomeDomain != nil {
-		e.addMuxed(source, history.EffectAccountHomeDomainUpdated,
+		if err := e.addMuxed(source, history.EffectAccountHomeDomainUpdated,
 			map[string]interface{}{
 				"home_domain": string(*op.HomeDomain),
 			},
-		)
+		); err != nil {
+			return err
+		}
 	}
 
 	thresholdDetails := map[string]interface{}{}
@@ -632,7 +591,9 @@ func (e *effectsWrapper) addSetOptionsEffects() error {
 	}
 
 	if len(thresholdDetails) > 0 {
-		e.addMuxed(source, history.EffectAccountThresholdsUpdated, thresholdDetails)
+		if err := e.addMuxed(source, history.EffectAccountThresholdsUpdated, thresholdDetails); err != nil {
+			return err
+		}
 	}
 
 	flagDetails := map[string]interface{}{}
@@ -644,15 +605,19 @@ func (e *effectsWrapper) addSetOptionsEffects() error {
 	}
 
 	if len(flagDetails) > 0 {
-		e.addMuxed(source, history.EffectAccountFlagsUpdated, flagDetails)
+		if err := e.addMuxed(source, history.EffectAccountFlagsUpdated, flagDetails); err != nil {
+			return err
+		}
 	}
 
 	if op.InflationDest != nil {
-		e.addMuxed(source, history.EffectAccountInflationDestinationUpdated,
+		if err := e.addMuxed(source, history.EffectAccountInflationDestinationUpdated,
 			map[string]interface{}{
 				"inflation_destination": op.InflationDest.Address(),
 			},
-		)
+		); err != nil {
+			return err
+		}
 	}
 	changes, err := e.operation.transaction.GetOperationChanges(e.operation.index)
 	if err != nil {
@@ -675,7 +640,7 @@ func (e *effectsWrapper) addSetOptionsEffects() error {
 			continue
 		}
 
-		beforeSortedSigners := []string{}
+		var beforeSortedSigners []string
 		for signer := range before {
 			beforeSortedSigners = append(beforeSortedSigners, signer)
 		}
@@ -684,21 +649,25 @@ func (e *effectsWrapper) addSetOptionsEffects() error {
 		for _, addy := range beforeSortedSigners {
 			weight, ok := after[addy]
 			if !ok {
-				e.addMuxed(source, history.EffectSignerRemoved, map[string]interface{}{
+				if err := e.addMuxed(source, history.EffectSignerRemoved, map[string]interface{}{
 					"public_key": addy,
-				})
+				}); err != nil {
+					return err
+				}
 				continue
 			}
 
 			if weight != before[addy] {
-				e.addMuxed(source, history.EffectSignerUpdated, map[string]interface{}{
+				if err := e.addMuxed(source, history.EffectSignerUpdated, map[string]interface{}{
 					"public_key": addy,
 					"weight":     weight,
-				})
+				}); err != nil {
+					return err
+				}
 			}
 		}
 
-		afterSortedSigners := []string{}
+		var afterSortedSigners []string
 		for signer := range after {
 			afterSortedSigners = append(afterSortedSigners, signer)
 		}
@@ -713,10 +682,12 @@ func (e *effectsWrapper) addSetOptionsEffects() error {
 				continue
 			}
 
-			e.addMuxed(source, history.EffectSignerCreated, map[string]interface{}{
+			if err := e.addMuxed(source, history.EffectSignerCreated, map[string]interface{}{
 				"public_key": addy,
 				"weight":     weight,
-			})
+			}); err != nil {
+				return err
+			}
 		}
 	}
 	return nil
@@ -772,10 +743,14 @@ func (e *effectsWrapper) addChangeTrustEffects() error {
 				return err
 			}
 		} else {
-			addAssetDetails(details, op.Line.ToAsset(), "")
+			if err := addAssetDetails(details, op.Line.ToAsset(), ""); err != nil {
+				return err
+			}
 		}
 
-		e.addMuxed(source, effect, details)
+		if err := e.addMuxed(source, effect, details); err != nil {
+			return err
+		}
 		break
 	}
 
@@ -789,33 +764,47 @@ func (e *effectsWrapper) addAllowTrustEffects() error {
 	details := map[string]interface{}{
 		"trustor": op.Trustor.Address(),
 	}
-	addAssetDetails(details, asset, "")
+	if err := addAssetDetails(details, asset, ""); err != nil {
+		return err
+	}
 
 	switch {
 	case xdr.TrustLineFlags(op.Authorize).IsAuthorized():
-		e.addMuxed(source, history.EffectTrustlineAuthorized, details)
+		if err := e.addMuxed(source, history.EffectTrustlineAuthorized, details); err != nil {
+			return err
+		}
 		// Forward compatibility
 		setFlags := xdr.Uint32(xdr.TrustLineFlagsAuthorizedFlag)
-		e.addTrustLineFlagsEffect(source, &op.Trustor, asset, &setFlags, nil)
+		if err := e.addTrustLineFlagsEffect(source, &op.Trustor, asset, &setFlags, nil); err != nil {
+			return err
+		}
 	case xdr.TrustLineFlags(op.Authorize).IsAuthorizedToMaintainLiabilitiesFlag():
-		e.addMuxed(
+		if err := e.addMuxed(
 			source,
 			history.EffectTrustlineAuthorizedToMaintainLiabilities,
 			details,
-		)
+		); err != nil {
+			return err
+		}
 		// Forward compatibility
 		setFlags := xdr.Uint32(xdr.TrustLineFlagsAuthorizedToMaintainLiabilitiesFlag)
-		e.addTrustLineFlagsEffect(source, &op.Trustor, asset, &setFlags, nil)
+		if err := e.addTrustLineFlagsEffect(source, &op.Trustor, asset, &setFlags, nil); err != nil {
+			return err
+		}
 	default:
-		e.addMuxed(source, history.EffectTrustlineDeauthorized, details)
+		if err := e.addMuxed(source, history.EffectTrustlineDeauthorized, details); err != nil {
+			return err
+		}
 		// Forward compatibility, show both as cleared
 		clearFlags := xdr.Uint32(xdr.TrustLineFlagsAuthorizedFlag | xdr.TrustLineFlagsAuthorizedToMaintainLiabilitiesFlag)
-		e.addTrustLineFlagsEffect(source, &op.Trustor, asset, nil, &clearFlags)
+		if err := e.addTrustLineFlagsEffect(source, &op.Trustor, asset, nil, &clearFlags); err != nil {
+			return err
+		}
 	}
 	return e.addLiquidityPoolRevokedEffect()
 }
 
-func (e *effectsWrapper) addAccountMergeEffects() {
+func (e *effectsWrapper) addAccountMergeEffects() error {
 	source := e.operation.SourceAccount()
 
 	dest := e.operation.operation.Body.MustDestination()
@@ -825,21 +814,31 @@ func (e *effectsWrapper) addAccountMergeEffects() {
 		"asset_type": "native",
 	}
 
-	e.addMuxed(source, history.EffectAccountDebited, details)
-	e.addMuxed(&dest, history.EffectAccountCredited, details)
-	e.addMuxed(source, history.EffectAccountRemoved, map[string]interface{}{})
+	if err := e.addMuxed(source, history.EffectAccountDebited, details); err != nil {
+		return err
+	}
+	if err := e.addMuxed(&dest, history.EffectAccountCredited, details); err != nil {
+		return err
+	}
+	if err := e.addMuxed(source, history.EffectAccountRemoved, map[string]interface{}{}); err != nil {
+		return err
+	}
+	return nil
 }
 
-func (e *effectsWrapper) addInflationEffects() {
+func (e *effectsWrapper) addInflationEffects() error {
 	payouts := e.operation.OperationResult().MustInflationResult().MustPayouts()
 	for _, payout := range payouts {
-		e.addUnmuxed(&payout.Destination, history.EffectAccountCredited,
+		if err := e.addUnmuxed(&payout.Destination, history.EffectAccountCredited,
 			map[string]interface{}{
 				"amount":     amount.String(payout.Amount),
 				"asset_type": "native",
 			},
-		)
+		); err != nil {
+			return err
+		}
 	}
+	return nil
 }
 
 func (e *effectsWrapper) addManageDataEffects() error {
@@ -879,8 +878,7 @@ func (e *effectsWrapper) addManageDataEffects() error {
 		break
 	}
 
-	e.addMuxed(source, effect, details)
-	return nil
+	return e.addMuxed(source, effect, details)
 }
 
 func (e *effectsWrapper) addBumpSequenceEffects() error {
@@ -903,7 +901,9 @@ func (e *effectsWrapper) addBumpSequenceEffects() error {
 
 		if beforeAccount.SeqNum != afterAccount.SeqNum {
 			details := map[string]interface{}{"new_seq": afterAccount.SeqNum}
-			e.addMuxed(source, history.EffectSequenceBumped, details)
+			if err := e.addMuxed(source, history.EffectSequenceBumped, details); err != nil {
+				return err
+			}
 		}
 		break
 	}
@@ -926,7 +926,9 @@ func (e *effectsWrapper) addCreateClaimableBalanceEffects(changes []ingest.Chang
 			continue
 		}
 		cb = change.Post.Data.ClaimableBalance
-		e.addClaimableBalanceEntryCreatedEffects(source, cb)
+		if err := e.addClaimableBalanceEntryCreatedEffects(source, cb); err != nil {
+			return err
+		}
 		break
 	}
 	if cb == nil {
@@ -936,14 +938,14 @@ func (e *effectsWrapper) addCreateClaimableBalanceEffects(changes []ingest.Chang
 	details := map[string]interface{}{
 		"amount": amount.String(cb.Amount),
 	}
-	addAssetDetails(details, cb.Asset, "")
-	e.addMuxed(
+	if err := addAssetDetails(details, cb.Asset, ""); err != nil {
+		return err
+	}
+	return e.addMuxed(
 		source,
 		history.EffectAccountDebited,
 		details,
 	)
-
-	return nil
 }
 
 func (e *effectsWrapper) addClaimableBalanceEntryCreatedEffects(source *xdr.MuxedAccount, cb *xdr.ClaimableBalanceEntry) error {
@@ -957,11 +959,13 @@ func (e *effectsWrapper) addClaimableBalanceEntryCreatedEffects(source *xdr.Muxe
 		"asset":      cb.Asset.StringCanonical(),
 	}
 	setClaimableBalanceFlagDetails(details, cb.Flags())
-	e.addMuxed(
+	if err := e.addMuxed(
 		source,
 		history.EffectClaimableBalanceCreated,
 		details,
-	)
+	); err != nil {
+		return err
+	}
 	// EffectClaimableBalanceClaimantCreated can be generated by
 	// `create_claimable_balance` operation but also by `liquidity_pool_withdraw`
 	// operation causing a revocation.
@@ -977,7 +981,7 @@ func (e *effectsWrapper) addClaimableBalanceEntryCreatedEffects(source *xdr.Muxe
 	}
 	for _, c := range claimants {
 		cv0 := c.MustV0()
-		e.addUnmuxed(
+		if err := e.addUnmuxed(
 			&cv0.Destination,
 			history.EffectClaimableBalanceClaimantCreated,
 			map[string]interface{}{
@@ -986,9 +990,11 @@ func (e *effectsWrapper) addClaimableBalanceEntryCreatedEffects(source *xdr.Muxe
 				"predicate":  cv0.Predicate,
 				"asset":      cb.Asset.StringCanonical(),
 			},
-		)
+		); err != nil {
+			return err
+		}
 	}
-	return err
+	return nil
 }
 
 func (e *effectsWrapper) addClaimClaimableBalanceEffects(changes []ingest.Change) error {
@@ -1031,23 +1037,25 @@ func (e *effectsWrapper) addClaimClaimableBalanceEffects(changes []ingest.Change
 	}
 	setClaimableBalanceFlagDetails(details, cBalance.Flags())
 	source := e.operation.SourceAccount()
-	e.addMuxed(
+	if err := e.addMuxed(
 		source,
 		history.EffectClaimableBalanceClaimed,
 		details,
-	)
+	); err != nil {
+		return err
+	}
 
 	details = map[string]interface{}{
 		"amount": amount.String(cBalance.Amount),
 	}
-	addAssetDetails(details, cBalance.Asset, "")
-	e.addMuxed(
+	if err := addAssetDetails(details, cBalance.Asset, ""); err != nil {
+		return err
+	}
+	return e.addMuxed(
 		source,
 		history.EffectAccountCredited,
 		details,
 	)
-
-	return nil
 }
 
 func (e *effectsWrapper) addIngestTradeEffects(buyer xdr.MuxedAccount, claims []xdr.ClaimAtom) error {
@@ -1061,23 +1069,30 @@ func (e *effectsWrapper) addIngestTradeEffects(buyer xdr.MuxedAccount, claims []
 				return err
 			}
 		default:
-			e.addClaimTradeEffects(buyer, claim)
+			if err := e.addClaimTradeEffects(buyer, claim); err != nil {
+				return err
+			}
 		}
 	}
 	return nil
 }
 
-func (e *effectsWrapper) addClaimTradeEffects(buyer xdr.MuxedAccount, claim xdr.ClaimAtom) {
+func (e *effectsWrapper) addClaimTradeEffects(buyer xdr.MuxedAccount, claim xdr.ClaimAtom) error {
 	seller := claim.SellerId()
-	bd, sd := tradeDetails(buyer, seller, claim)
+	bd, sd, err := tradeDetails(buyer, seller, claim)
+	if err != nil {
+		return err
+	}
 
-	e.addMuxed(
+	if err := e.addMuxed(
 		&buyer,
 		history.EffectTrade,
 		bd,
-	)
+	); err != nil {
+		return err
+	}
 
-	e.addUnmuxed(
+	return e.addUnmuxed(
 		&seller,
 		history.EffectTrade,
 		sd,
@@ -1100,8 +1115,7 @@ func (e *effectsWrapper) addClaimLiquidityPoolTradeEffect(claim xdr.ClaimAtom) e
 			"amount": amount.String(claim.LiquidityPool.AmountBought),
 		},
 	}
-	e.addMuxed(e.operation.SourceAccount(), history.EffectLiquidityPoolTrade, details)
-	return nil
+	return e.addMuxed(e.operation.SourceAccount(), history.EffectLiquidityPoolTrade, details)
 }
 
 func (e *effectsWrapper) addClawbackEffects() error {
@@ -1110,20 +1124,26 @@ func (e *effectsWrapper) addClawbackEffects() error {
 		"amount": amount.String(op.Amount),
 	}
 	source := e.operation.SourceAccount()
-	addAssetDetails(details, op.Asset, "")
+	if err := addAssetDetails(details, op.Asset, ""); err != nil {
+		return err
+	}
 
 	// The funds will be burned, but even with that, we generated an account credited effect
-	e.addMuxed(
+	if err := e.addMuxed(
 		source,
 		history.EffectAccountCredited,
 		details,
-	)
+	); err != nil {
+		return err
+	}
 
-	e.addMuxed(
+	if err := e.addMuxed(
 		&op.From,
 		history.EffectAccountDebited,
 		details,
-	)
+	); err != nil {
+		return err
+	}
 
 	return nil
 }
@@ -1138,23 +1158,29 @@ func (e *effectsWrapper) addClawbackClaimableBalanceEffects(changes []ingest.Cha
 		"balance_id": balanceId,
 	}
 	source := e.operation.SourceAccount()
-	e.addMuxed(
+	if err := e.addMuxed(
 		source,
 		history.EffectClaimableBalanceClawedBack,
 		details,
-	)
+	); err != nil {
+		return err
+	}
 
 	// Generate the account credited effect (although the funds will be burned) for the asset issuer
 	for _, c := range changes {
 		if c.Type == xdr.LedgerEntryTypeClaimableBalance && c.Post == nil && c.Pre != nil {
 			cb := c.Pre.Data.ClaimableBalance
 			details = map[string]interface{}{"amount": amount.String(cb.Amount)}
-			addAssetDetails(details, cb.Asset, "")
-			e.addMuxed(
+			if err := addAssetDetails(details, cb.Asset, ""); err != nil {
+				return err
+			}
+			if err := e.addMuxed(
 				source,
 				history.EffectAccountCredited,
 				details,
-			)
+			); err != nil {
+				return err
+			}
 			break
 		}
 	}
@@ -1165,7 +1191,9 @@ func (e *effectsWrapper) addClawbackClaimableBalanceEffects(changes []ingest.Cha
 func (e *effectsWrapper) addSetTrustLineFlagsEffects() error {
 	source := e.operation.SourceAccount()
 	op := e.operation.operation.Body.MustSetTrustLineFlagsOp()
-	e.addTrustLineFlagsEffect(source, &op.Trustor, op.Asset, &op.SetFlags, &op.ClearFlags)
+	if err := e.addTrustLineFlagsEffect(source, &op.Trustor, op.Asset, &op.SetFlags, &op.ClearFlags); err != nil {
+		return err
+	}
 	return e.addLiquidityPoolRevokedEffect()
 }
 
@@ -1174,11 +1202,13 @@ func (e *effectsWrapper) addTrustLineFlagsEffect(
 	trustor *xdr.AccountId,
 	asset xdr.Asset,
 	setFlags *xdr.Uint32,
-	clearFlags *xdr.Uint32) {
+	clearFlags *xdr.Uint32) error {
 	details := map[string]interface{}{
 		"trustor": trustor.Address(),
 	}
-	addAssetDetails(details, asset, "")
+	if err := addAssetDetails(details, asset, ""); err != nil {
+		return err
+	}
 
 	var flagDetailsAdded bool
 	if setFlags != nil {
@@ -1191,8 +1221,11 @@ func (e *effectsWrapper) addTrustLineFlagsEffect(
 	}
 
 	if flagDetailsAdded {
-		e.addMuxed(account, history.EffectTrustlineFlagsUpdated, details)
+		if err := e.addMuxed(account, history.EffectTrustlineFlagsUpdated, details); err != nil {
+			return err
+		}
 	}
+	return nil
 }
 
 func setTrustLineFlagDetails(flagDetails map[string]interface{}, flags xdr.TrustLineFlags, setValue bool) {
@@ -1278,8 +1311,8 @@ func (e *effectsWrapper) addLiquidityPoolRevokedEffect() error {
 		"reserves_revoked": reservesRevoked,
 		"shares_revoked":   amount.String(-delta.TotalPoolShares),
 	}
-	e.addMuxed(source, history.EffectLiquidityPoolRevoked, details)
-	return nil
+
+	return e.addMuxed(source, history.EffectLiquidityPoolRevoked, details)
 }
 
 func setAuthFlagDetails(flagDetails map[string]interface{}, flags xdr.AccountFlags, setValue bool) {
@@ -1297,15 +1330,19 @@ func setAuthFlagDetails(flagDetails map[string]interface{}, flags xdr.AccountFla
 	}
 }
 
-func tradeDetails(buyer xdr.MuxedAccount, seller xdr.AccountId, claim xdr.ClaimAtom) (bd map[string]interface{}, sd map[string]interface{}) {
+func tradeDetails(buyer xdr.MuxedAccount, seller xdr.AccountId, claim xdr.ClaimAtom) (bd map[string]interface{}, sd map[string]interface{}, err error) {
 	bd = map[string]interface{}{
 		"offer_id":      claim.OfferId(),
 		"seller":        seller.Address(),
 		"bought_amount": amount.String(claim.AmountSold()),
 		"sold_amount":   amount.String(claim.AmountBought()),
 	}
-	addAssetDetails(bd, claim.AssetSold(), "bought_")
-	addAssetDetails(bd, claim.AssetBought(), "sold_")
+	if err = addAssetDetails(bd, claim.AssetSold(), "bought_"); err != nil {
+		return
+	}
+	if err = addAssetDetails(bd, claim.AssetBought(), "sold_"); err != nil {
+		return
+	}
 
 	sd = map[string]interface{}{
 		"offer_id":      claim.OfferId(),
@@ -1313,9 +1350,12 @@ func tradeDetails(buyer xdr.MuxedAccount, seller xdr.AccountId, claim xdr.ClaimA
 		"sold_amount":   amount.String(claim.AmountSold()),
 	}
 	addAccountAndMuxedAccountDetails(sd, buyer, "seller")
-	addAssetDetails(sd, claim.AssetBought(), "bought_")
-	addAssetDetails(sd, claim.AssetSold(), "sold_")
-
+	if err = addAssetDetails(sd, claim.AssetBought(), "bought_"); err != nil {
+		return
+	}
+	if err = addAssetDetails(sd, claim.AssetSold(), "sold_"); err != nil {
+		return
+	}
 	return
 }
 
@@ -1359,8 +1399,8 @@ func (e *effectsWrapper) addLiquidityPoolDepositEffect() error {
 		},
 		"shares_received": amount.String(delta.TotalPoolShares),
 	}
-	e.addMuxed(e.operation.SourceAccount(), history.EffectLiquidityPoolDeposited, details)
-	return nil
+
+	return e.addMuxed(e.operation.SourceAccount(), history.EffectLiquidityPoolDeposited, details)
 }
 
 func (e *effectsWrapper) addLiquidityPoolWithdrawEffect() error {
@@ -1383,6 +1423,6 @@ func (e *effectsWrapper) addLiquidityPoolWithdrawEffect() error {
 		},
 		"shares_redeemed": amount.String(-delta.TotalPoolShares),
 	}
-	e.addMuxed(e.operation.SourceAccount(), history.EffectLiquidityPoolWithdrew, details)
-	return nil
+
+	return e.addMuxed(e.operation.SourceAccount(), history.EffectLiquidityPoolWithdrew, details)
 }
diff --git a/services/horizon/internal/ingest/processors/effects_processor_test.go b/services/horizon/internal/ingest/processors/effects_processor_test.go
index 4293fb5b3b..3afd169c30 100644
--- a/services/horizon/internal/ingest/processors/effects_processor_test.go
+++ b/services/horizon/internal/ingest/processors/effects_processor_test.go
@@ -5,15 +5,15 @@ package processors
 import (
 	"context"
 	"encoding/hex"
+	"encoding/json"
 	"testing"
 
 	"github.com/guregu/null"
-	"github.com/stellar/go/protocols/horizon/base"
 	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/suite"
 
 	"github.com/stellar/go/ingest"
+	"github.com/stellar/go/protocols/horizon/base"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	. "github.com/stellar/go/services/horizon/internal/test/transactions"
 	"github.com/stellar/go/support/db"
@@ -27,9 +27,10 @@ type EffectsProcessorTestSuiteLedger struct {
 	ctx                    context.Context
 	processor              *EffectProcessor
 	mockSession            *db.MockSession
-	mockQ                  *history.MockQEffects
+	accountLoader          *history.AccountLoader
 	mockBatchInsertBuilder *history.MockEffectBatchInsertBuilder
 
+	lcm         xdr.LedgerCloseMeta
 	firstTx     ingest.LedgerTransaction
 	secondTx    ingest.LedgerTransaction
 	thirdTx     ingest.LedgerTransaction
@@ -38,7 +39,6 @@ type EffectsProcessorTestSuiteLedger struct {
 	secondTxID  int64
 	thirdTxID   int64
 	failedTxID  int64
-	sequence    uint32
 	addresses   []string
 	addressToID map[string]int64
 	txs         []ingest.LedgerTransaction
@@ -50,11 +50,18 @@ func TestEffectsProcessorTestSuiteLedger(t *testing.T) {
 
 func (s *EffectsProcessorTestSuiteLedger) SetupTest() {
 	s.ctx = context.Background()
-	s.mockQ = &history.MockQEffects{}
+	s.accountLoader = history.NewAccountLoader()
 	s.mockBatchInsertBuilder = &history.MockEffectBatchInsertBuilder{}
 
-	s.sequence = uint32(20)
-
+	s.lcm = xdr.LedgerCloseMeta{
+		V0: &xdr.LedgerCloseMetaV0{
+			LedgerHeader: xdr.LedgerHeaderHistoryEntry{
+				Header: xdr.LedgerHeader{
+					LedgerSeq: xdr.Uint32(20),
+				},
+			},
+		},
+	}
 	s.addresses = []string{
 		"GANFZDRBCNTUXIODCJEYMACPMCSZEVE4WZGZ3CZDZ3P2SXK4KH75IK6Y",
 		"GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H",
@@ -72,7 +79,7 @@ func (s *EffectsProcessorTestSuiteLedger) SetupTest() {
 			Hash:          "829d53f2dceebe10af8007564b0aefde819b95734ad431df84270651e7ed8a90",
 		},
 	)
-	s.firstTxID = toid.New(int32(s.sequence), 1, 0).ToInt64()
+	s.firstTxID = toid.New(int32(s.lcm.LedgerSequence()), 1, 0).ToInt64()
 
 	s.secondTx = BuildLedgerTransaction(
 		s.Suite.T(),
@@ -86,7 +93,7 @@ func (s *EffectsProcessorTestSuiteLedger) SetupTest() {
 		},
 	)
 
-	s.secondTxID = toid.New(int32(s.sequence), 2, 0).ToInt64()
+	s.secondTxID = toid.New(int32(s.lcm.LedgerSequence()), 2, 0).ToInt64()
 
 	s.thirdTx = BuildLedgerTransaction(
 		s.Suite.T(),
@@ -99,7 +106,7 @@ func (s *EffectsProcessorTestSuiteLedger) SetupTest() {
 			Hash:          "2a805712c6d10f9e74bb0ccf54ae92a2b4b1e586451fe8133a2433816f6b567c",
 		},
 	)
-	s.thirdTxID = toid.New(int32(s.sequence), 3, 0).ToInt64()
+	s.thirdTxID = toid.New(int32(s.lcm.LedgerSequence()), 3, 0).ToInt64()
 
 	s.failedTx = BuildLedgerTransaction(
 		s.Suite.T(),
@@ -112,7 +119,7 @@ func (s *EffectsProcessorTestSuiteLedger) SetupTest() {
 			Hash:          "24206737a02f7f855c46e367418e38c223f897792c76bbfb948e1b0dbd695f8b",
 		},
 	)
-	s.failedTxID = toid.New(int32(s.sequence), 4, 0).ToInt64()
+	s.failedTxID = toid.New(int32(s.lcm.LedgerSequence()), 4, 0).ToInt64()
 
 	s.addressToID = map[string]int64{
 		s.addresses[0]: 2,
@@ -121,9 +128,8 @@ func (s *EffectsProcessorTestSuiteLedger) SetupTest() {
 	}
 
 	s.processor = NewEffectProcessor(
-		s.mockSession,
-		s.mockQ,
-		20,
+		s.accountLoader,
+		s.mockBatchInsertBuilder,
 	)
 
 	s.txs = []ingest.LedgerTransaction{
@@ -134,42 +140,42 @@ func (s *EffectsProcessorTestSuiteLedger) SetupTest() {
 }
 
 func (s *EffectsProcessorTestSuiteLedger) TearDownTest() {
-	s.mockQ.AssertExpectations(s.T())
+	s.mockBatchInsertBuilder.AssertExpectations(s.T())
 }
 
 func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulEffectBatchAdds() {
 	s.mockBatchInsertBuilder.On(
 		"Add",
-		s.addressToID[s.addresses[2]],
+		s.accountLoader.GetFuture(s.addresses[2]),
 		null.String{},
-		toid.New(int32(s.sequence), 1, 1).ToInt64(),
+		toid.New(int32(s.lcm.LedgerSequence()), 1, 1).ToInt64(),
 		uint32(1),
 		history.EffectSequenceBumped,
 		[]byte("{\"new_seq\":300000000000}"),
 	).Return(nil).Once()
 	s.mockBatchInsertBuilder.On(
 		"Add",
-		s.addressToID[s.addresses[2]],
+		s.accountLoader.GetFuture(s.addresses[2]),
 		null.String{},
-		toid.New(int32(s.sequence), 2, 1).ToInt64(),
+		toid.New(int32(s.lcm.LedgerSequence()), 2, 1).ToInt64(),
 		uint32(1),
 		history.EffectAccountCreated,
 		[]byte("{\"starting_balance\":\"1000.0000000\"}"),
 	).Return(nil).Once()
 	s.mockBatchInsertBuilder.On(
 		"Add",
-		s.addressToID[s.addresses[1]],
+		s.accountLoader.GetFuture(s.addresses[1]),
 		null.String{},
-		toid.New(int32(s.sequence), 2, 1).ToInt64(),
+		toid.New(int32(s.lcm.LedgerSequence()), 2, 1).ToInt64(),
 		uint32(2),
 		history.EffectAccountDebited,
 		[]byte("{\"amount\":\"1000.0000000\",\"asset_type\":\"native\"}"),
 	).Return(nil).Once()
 	s.mockBatchInsertBuilder.On(
 		"Add",
-		s.addressToID[s.addresses[2]],
+		s.accountLoader.GetFuture(s.addresses[2]),
 		null.String{},
-		toid.New(int32(s.sequence), 2, 1).ToInt64(),
+		toid.New(int32(s.lcm.LedgerSequence()), 2, 1).ToInt64(),
 		uint32(3),
 		history.EffectSignerCreated,
 		[]byte("{\"public_key\":\"GCQZP3IU7XU6EJ63JZXKCQOYT2RNXN3HB5CNHENNUEUHSMA4VUJJJSEN\",\"weight\":1}"),
@@ -177,9 +183,9 @@ func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulEffectBatchAdds() {
 
 	s.mockBatchInsertBuilder.On(
 		"Add",
-		s.addressToID[s.addresses[0]],
+		s.accountLoader.GetFuture(s.addresses[0]),
 		null.String{},
-		toid.New(int32(s.sequence), 3, 1).ToInt64(),
+		toid.New(int32(s.lcm.LedgerSequence()), 3, 1).ToInt64(),
 		uint32(1),
 		history.EffectAccountCredited,
 		[]byte("{\"amount\":\"10.0000000\",\"asset_type\":\"native\"}"),
@@ -187,81 +193,45 @@ func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulEffectBatchAdds() {
 
 	s.mockBatchInsertBuilder.On(
 		"Add",
-		s.addressToID[s.addresses[0]],
+		s.accountLoader.GetFuture(s.addresses[0]),
 		null.String{},
-		toid.New(int32(s.sequence), 3, 1).ToInt64(),
+		toid.New(int32(s.lcm.LedgerSequence()), 3, 1).ToInt64(),
 		uint32(2),
 		history.EffectAccountDebited,
 		[]byte("{\"amount\":\"10.0000000\",\"asset_type\":\"native\"}"),
 	).Return(nil).Once()
 }
 
-func (s *EffectsProcessorTestSuiteLedger) mockSuccessfulCreateAccounts() {
-	s.mockQ.On(
-		"CreateAccounts",
-		s.ctx,
-		mock.AnythingOfType("[]string"),
-		maxBatchSize,
-	).Run(func(args mock.Arguments) {
-		arg := args.Get(1).([]string)
-		s.Assert().ElementsMatch(s.addresses, arg)
-	}).Return(s.addressToID, nil).Once()
-}
-
 func (s *EffectsProcessorTestSuiteLedger) TestEmptyEffects() {
-	err := s.processor.Commit(context.Background())
-	s.Assert().NoError(err)
+	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
+	s.Assert().NoError(s.processor.Flush(s.ctx, s.mockSession))
 }
 
 func (s *EffectsProcessorTestSuiteLedger) TestIngestEffectsSucceeds() {
-	s.mockSuccessfulCreateAccounts()
-	s.mockQ.On("NewEffectBatchInsertBuilder").
-		Return(s.mockBatchInsertBuilder).Once()
-
 	s.mockSuccessfulEffectBatchAdds()
-
-	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
-
 	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(s.ctx, tx)
-		s.Assert().NoError(err)
+		s.Assert().NoError(s.processor.ProcessTransaction(s.lcm, tx))
 	}
-	err := s.processor.Commit(s.ctx)
-	s.Assert().NoError(err)
-}
-
-func (s *EffectsProcessorTestSuiteLedger) TestCreateAccountsFails() {
-	s.mockQ.On("CreateAccounts", s.ctx, mock.AnythingOfType("[]string"), maxBatchSize).
-		Return(s.addressToID, errors.New("transient error")).Once()
 
-	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(s.ctx, tx)
-		s.Assert().NoError(err)
-	}
-	err := s.processor.Commit(s.ctx)
-	s.Assert().EqualError(err, "Could not create account ids: transient error")
+	s.mockBatchInsertBuilder.On("Exec", s.ctx, s.mockSession).Return(nil).Once()
+	s.Assert().NoError(s.processor.Flush(s.ctx, s.mockSession))
 }
 
 func (s *EffectsProcessorTestSuiteLedger) TestBatchAddFails() {
-	s.mockSuccessfulCreateAccounts()
-	s.mockQ.On("NewEffectBatchInsertBuilder").
-		Return(s.mockBatchInsertBuilder).Once()
-
 	s.mockBatchInsertBuilder.On(
 		"Add",
-		s.addressToID[s.addresses[2]],
+		s.accountLoader.GetFuture(s.addresses[2]),
 		null.String{},
-		toid.New(int32(s.sequence), 1, 1).ToInt64(),
+		toid.New(int32(s.lcm.LedgerSequence()), 1, 1).ToInt64(),
 		uint32(1),
 		history.EffectSequenceBumped,
 		[]byte("{\"new_seq\":300000000000}"),
 	).Return(errors.New("transient error")).Once()
-	for _, tx := range s.txs {
-		err := s.processor.ProcessTransaction(s.ctx, tx)
-		s.Assert().NoError(err)
-	}
-	err := s.processor.Commit(s.ctx)
-	s.Assert().EqualError(err, "could not insert operation effect in db: transient error")
+
+	s.Assert().EqualError(
+		s.processor.ProcessTransaction(s.lcm, s.txs[0]),
+		"reading operation 85899350017 effects: could not insert operation effect in db: transient error",
+	)
 }
 
 func getRevokeSponsorshipMeta(t *testing.T) (string, []effect) {
@@ -462,7 +432,7 @@ func TestEffectsCoversAllOperationTypes(t *testing.T) {
 				}
 				assert.True(t, err2 != nil || err == nil, s)
 			}()
-			_, err = operation.effects()
+			err = operation.ingestEffects(history.NewAccountLoader(), &history.MockEffectBatchInsertBuilder{})
 		}()
 	}
 
@@ -484,7 +454,7 @@ func TestEffectsCoversAllOperationTypes(t *testing.T) {
 		ledgerSequence: 1,
 	}
 	// calling effects should error due to the unknown operation
-	_, err := operation.effects()
+	err := operation.ingestEffects(history.NewAccountLoader(), &history.MockEffectBatchInsertBuilder{})
 	assert.Contains(t, err.Error(), "Unknown operation type")
 }
 
@@ -1546,7 +1516,6 @@ func TestOperationEffects(t *testing.T) {
 	}
 	for _, tc := range testCases {
 		t.Run(tc.desc, func(t *testing.T) {
-			tt := assert.New(t)
 			transaction := BuildLedgerTransaction(
 				t,
 				TestTransaction{
@@ -1566,15 +1535,12 @@ func TestOperationEffects(t *testing.T) {
 				ledgerSequence: tc.sequence,
 			}
 
-			effects, err := operation.effects()
-			tt.NoError(err)
-			tt.Equal(tc.expected, effects)
+			assertIngestEffects(t, operation, tc.expected)
 		})
 	}
 }
 
 func TestOperationEffectsSetOptionsSignersOrder(t *testing.T) {
-	tt := assert.New(t)
 	transaction := ingest.LedgerTransaction{
 		UnsafeMeta: createTransactionMeta([]xdr.OperationMeta{
 			{
@@ -1656,8 +1622,6 @@ func TestOperationEffectsSetOptionsSignersOrder(t *testing.T) {
 		ledgerSequence: 46,
 	}
 
-	effects, err := operation.effects()
-	tt.NoError(err)
 	expected := []effect{
 		{
 			address:     "GCBBDQLCTNASZJ3MTKAOYEOWRGSHDFAJVI7VPZUOP7KXNHYR3HP2BUKV",
@@ -1700,12 +1664,11 @@ func TestOperationEffectsSetOptionsSignersOrder(t *testing.T) {
 			order:      uint32(4),
 		},
 	}
-	tt.Equal(expected, effects)
+	assertIngestEffects(t, operation, expected)
 }
 
 // Regression for https://github.com/stellar/go/issues/2136
 func TestOperationEffectsSetOptionsSignersNoUpdated(t *testing.T) {
-	tt := assert.New(t)
 	transaction := ingest.LedgerTransaction{
 		UnsafeMeta: createTransactionMeta([]xdr.OperationMeta{
 			{
@@ -1787,8 +1750,6 @@ func TestOperationEffectsSetOptionsSignersNoUpdated(t *testing.T) {
 		ledgerSequence: 46,
 	}
 
-	effects, err := operation.effects()
-	tt.NoError(err)
 	expected := []effect{
 		{
 			address:     "GCBBDQLCTNASZJ3MTKAOYEOWRGSHDFAJVI7VPZUOP7KXNHYR3HP2BUKV",
@@ -1820,11 +1781,10 @@ func TestOperationEffectsSetOptionsSignersNoUpdated(t *testing.T) {
 			order:      uint32(3),
 		},
 	}
-	tt.Equal(expected, effects)
+	assertIngestEffects(t, operation, expected)
 }
 
 func TestOperationRegressionAccountTrustItself(t *testing.T) {
-	tt := assert.New(t)
 	// NOTE:  when an account trusts itself, the transaction is successful but
 	// no ledger entries are actually modified.
 	transaction := ingest.LedgerTransaction{
@@ -1853,9 +1813,7 @@ func TestOperationRegressionAccountTrustItself(t *testing.T) {
 		ledgerSequence: 46,
 	}
 
-	effects, err := operation.effects()
-	tt.NoError(err)
-	tt.Equal([]effect{}, effects)
+	assertIngestEffects(t, operation, []effect{})
 }
 
 func TestOperationEffectsAllowTrustAuthorizedToMaintainLiabilities(t *testing.T) {
@@ -1889,9 +1847,6 @@ func TestOperationEffectsAllowTrustAuthorizedToMaintainLiabilities(t *testing.T)
 		ledgerSequence: 1,
 	}
 
-	effects, err := operation.effects()
-	tt.NoError(err)
-
 	expected := []effect{
 		{
 			address:     "GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD",
@@ -1919,11 +1874,10 @@ func TestOperationEffectsAllowTrustAuthorizedToMaintainLiabilities(t *testing.T)
 			order:      uint32(2),
 		},
 	}
-	tt.Equal(expected, effects)
+	assertIngestEffects(t, operation, expected)
 }
 
 func TestOperationEffectsClawback(t *testing.T) {
-	tt := assert.New(t)
 	aid := xdr.MustAddress("GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD")
 	source := aid.ToMuxedAccount()
 	op := xdr.Operation{
@@ -1950,9 +1904,6 @@ func TestOperationEffectsClawback(t *testing.T) {
 		ledgerSequence: 1,
 	}
 
-	effects, err := operation.effects()
-	tt.NoError(err)
-
 	expected := []effect{
 		{
 			address:     "GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD",
@@ -1979,11 +1930,10 @@ func TestOperationEffectsClawback(t *testing.T) {
 			order:      uint32(2),
 		},
 	}
-	tt.Equal(expected, effects)
+	assertIngestEffects(t, operation, expected)
 }
 
 func TestOperationEffectsClawbackClaimableBalance(t *testing.T) {
-	tt := assert.New(t)
 	aid := xdr.MustAddress("GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD")
 	source := aid.ToMuxedAccount()
 	var balanceID xdr.ClaimableBalanceId
@@ -2010,9 +1960,6 @@ func TestOperationEffectsClawbackClaimableBalance(t *testing.T) {
 		ledgerSequence: 1,
 	}
 
-	effects, err := operation.effects()
-	tt.NoError(err)
-
 	expected := []effect{
 		{
 			address:     "GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD",
@@ -2024,11 +1971,10 @@ func TestOperationEffectsClawbackClaimableBalance(t *testing.T) {
 			order:      uint32(1),
 		},
 	}
-	tt.Equal(expected, effects)
+	assertIngestEffects(t, operation, expected)
 }
 
 func TestOperationEffectsSetTrustLineFlags(t *testing.T) {
-	tt := assert.New(t)
 	aid := xdr.MustAddress("GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD")
 	source := aid.ToMuxedAccount()
 	trustor := xdr.MustAddress("GAUJETIZVEP2NRYLUESJ3LS66NVCEGMON4UDCBCSBEVPIID773P2W6AY")
@@ -2059,9 +2005,6 @@ func TestOperationEffectsSetTrustLineFlags(t *testing.T) {
 		ledgerSequence: 1,
 	}
 
-	effects, err := operation.effects()
-	tt.NoError(err)
-
 	expected := []effect{
 		{
 			address:     "GDRW375MAYR46ODGF2WGANQC2RRZL7O246DYHHCGWTV2RE7IHE2QUQLD",
@@ -2079,7 +2022,7 @@ func TestOperationEffectsSetTrustLineFlags(t *testing.T) {
 			order:      uint32(1),
 		},
 	}
-	tt.Equal(expected, effects)
+	assertIngestEffects(t, operation, expected)
 }
 
 type CreateClaimableBalanceEffectsTestSuite struct {
@@ -2328,9 +2271,7 @@ func (s *CreateClaimableBalanceEffectsTestSuite) TestEffects() {
 				ledgerSequence: 1,
 			}
 
-			effects, err := operation.effects()
-			s.Assert().NoError(err)
-			s.Assert().Equal(tc.expected, effects)
+			assertIngestEffects(t, operation, tc.expected)
 		})
 	}
 }
@@ -2588,13 +2529,42 @@ func (s *ClaimClaimableBalanceEffectsTestSuite) TestEffects() {
 				ledgerSequence: 1,
 			}
 
-			effects, err := operation.effects()
-			s.Assert().NoError(err)
-			s.Assert().Equal(tc.expected, effects)
+			assertIngestEffects(t, operation, tc.expected)
 		})
 	}
 }
 
+type effect struct {
+	address      string
+	addressMuxed null.String
+	operationID  int64
+	details      map[string]interface{}
+	effectType   history.EffectType
+	order        uint32
+}
+
+func assertIngestEffects(t *testing.T, operation transactionOperationWrapper, expected []effect) {
+	accountLoader := history.NewAccountLoader()
+	mockBatchInsertBuilder := &history.MockEffectBatchInsertBuilder{}
+
+	for _, expectedEffect := range expected {
+		detailsJSON, err := json.Marshal(expectedEffect.details)
+		assert.NoError(t, err)
+		mockBatchInsertBuilder.On(
+			"Add",
+			accountLoader.GetFuture(expectedEffect.address),
+			expectedEffect.addressMuxed,
+			expectedEffect.operationID,
+			expectedEffect.order,
+			expectedEffect.effectType,
+			detailsJSON,
+		).Return(nil).Once()
+	}
+
+	assert.NoError(t, operation.ingestEffects(accountLoader, mockBatchInsertBuilder))
+	mockBatchInsertBuilder.AssertExpectations(t)
+}
+
 func TestClaimClaimableBalanceEffectsTestSuite(t *testing.T) {
 	suite.Run(t, new(ClaimClaimableBalanceEffectsTestSuite))
 }
@@ -2811,10 +2781,7 @@ func TestTrustlineSponsorshipEffects(t *testing.T) {
 		ledgerSequence: 1,
 	}
 
-	effects, err := operation.effects()
-	assert.NoError(t, err)
-	assert.Equal(t, expected, effects)
-
+	assertIngestEffects(t, operation, expected)
 }
 
 func TestLiquidityPoolEffects(t *testing.T) {
@@ -3445,9 +3412,7 @@ func TestLiquidityPoolEffects(t *testing.T) {
 				ledgerSequence: 1,
 			}
 
-			effects, err := operation.effects()
-			assert.NoError(t, err)
-			assert.Equal(t, tc.expected, effects)
+			assertIngestEffects(t, operation, tc.expected)
 		})
 	}
 
diff --git a/services/horizon/internal/ingest/processors/operations_processor.go b/services/horizon/internal/ingest/processors/operations_processor.go
index d22ba2cd6d..5aefdbe0d1 100644
--- a/services/horizon/internal/ingest/processors/operations_processor.go
+++ b/services/horizon/internal/ingest/processors/operations_processor.go
@@ -294,7 +294,9 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{},
 		addAccountAndMuxedAccountDetails(details, *source, "from")
 		addAccountAndMuxedAccountDetails(details, op.Destination, "to")
 		details["amount"] = amount.String(op.Amount)
-		addAssetDetails(details, op.Asset, "")
+		if err := addAssetDetails(details, op.Asset, ""); err != nil {
+			return nil, err
+		}
 	case xdr.OperationTypePathPaymentStrictReceive:
 		op := operation.operation.Body.MustPathPaymentStrictReceiveOp()
 		addAccountAndMuxedAccountDetails(details, *source, "from")
@@ -303,8 +305,12 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{},
 		details["amount"] = amount.String(op.DestAmount)
 		details["source_amount"] = amount.String(0)
 		details["source_max"] = amount.String(op.SendMax)
-		addAssetDetails(details, op.DestAsset, "")
-		addAssetDetails(details, op.SendAsset, "source_")
+		if err := addAssetDetails(details, op.DestAsset, ""); err != nil {
+			return nil, err
+		}
+		if err := addAssetDetails(details, op.SendAsset, "source_"); err != nil {
+			return nil, err
+		}
 
 		if operation.transaction.Result.Successful() {
 			result := operation.OperationResult().MustPathPaymentStrictReceiveResult()
@@ -314,7 +320,9 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{},
 		var path = make([]map[string]interface{}, len(op.Path))
 		for i := range op.Path {
 			path[i] = make(map[string]interface{})
-			addAssetDetails(path[i], op.Path[i], "")
+			if err := addAssetDetails(path[i], op.Path[i], ""); err != nil {
+				return nil, err
+			}
 		}
 		details["path"] = path
 
@@ -326,8 +334,12 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{},
 		details["amount"] = amount.String(0)
 		details["source_amount"] = amount.String(op.SendAmount)
 		details["destination_min"] = amount.String(op.DestMin)
-		addAssetDetails(details, op.DestAsset, "")
-		addAssetDetails(details, op.SendAsset, "source_")
+		if err := addAssetDetails(details, op.DestAsset, ""); err != nil {
+			return nil, err
+		}
+		if err := addAssetDetails(details, op.SendAsset, "source_"); err != nil {
+			return nil, err
+		}
 
 		if operation.transaction.Result.Successful() {
 			result := operation.OperationResult().MustPathPaymentStrictSendResult()
@@ -337,7 +349,9 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{},
 		var path = make([]map[string]interface{}, len(op.Path))
 		for i := range op.Path {
 			path[i] = make(map[string]interface{})
-			addAssetDetails(path[i], op.Path[i], "")
+			if err := addAssetDetails(path[i], op.Path[i], ""); err != nil {
+				return nil, err
+			}
 		}
 		details["path"] = path
 	case xdr.OperationTypeManageBuyOffer:
@@ -349,8 +363,12 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{},
 			"n": op.Price.N,
 			"d": op.Price.D,
 		}
-		addAssetDetails(details, op.Buying, "buying_")
-		addAssetDetails(details, op.Selling, "selling_")
+		if err := addAssetDetails(details, op.Buying, "buying_"); err != nil {
+			return nil, err
+		}
+		if err := addAssetDetails(details, op.Selling, "selling_"); err != nil {
+			return nil, err
+		}
 	case xdr.OperationTypeManageSellOffer:
 		op := operation.operation.Body.MustManageSellOfferOp()
 		details["offer_id"] = op.OfferId
@@ -360,8 +378,12 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{},
 			"n": op.Price.N,
 			"d": op.Price.D,
 		}
-		addAssetDetails(details, op.Buying, "buying_")
-		addAssetDetails(details, op.Selling, "selling_")
+		if err := addAssetDetails(details, op.Buying, "buying_"); err != nil {
+			return nil, err
+		}
+		if err := addAssetDetails(details, op.Selling, "selling_"); err != nil {
+			return nil, err
+		}
 	case xdr.OperationTypeCreatePassiveSellOffer:
 		op := operation.operation.Body.MustCreatePassiveSellOfferOp()
 		details["amount"] = amount.String(op.Amount)
@@ -370,8 +392,12 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{},
 			"n": op.Price.N,
 			"d": op.Price.D,
 		}
-		addAssetDetails(details, op.Buying, "buying_")
-		addAssetDetails(details, op.Selling, "selling_")
+		if err := addAssetDetails(details, op.Buying, "buying_"); err != nil {
+			return nil, err
+		}
+		if err := addAssetDetails(details, op.Selling, "selling_"); err != nil {
+			return nil, err
+		}
 	case xdr.OperationTypeSetOptions:
 		op := operation.operation.Body.MustSetOptionsOp()
 
@@ -418,14 +444,18 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{},
 				return nil, err
 			}
 		} else {
-			addAssetDetails(details, op.Line.ToAsset(), "")
+			if err := addAssetDetails(details, op.Line.ToAsset(), ""); err != nil {
+				return nil, err
+			}
 			details["trustee"] = details["asset_issuer"]
 		}
 		addAccountAndMuxedAccountDetails(details, *source, "trustor")
 		details["limit"] = amount.String(op.Limit)
 	case xdr.OperationTypeAllowTrust:
 		op := operation.operation.Body.MustAllowTrustOp()
-		addAssetDetails(details, op.Asset.ToAsset(source.ToAccountId()), "")
+		if err := addAssetDetails(details, op.Asset.ToAsset(source.ToAccountId()), ""); err != nil {
+			return nil, err
+		}
 		addAccountAndMuxedAccountDetails(details, *source, "trustee")
 		details["trustor"] = op.Trustor.Address()
 		details["authorize"] = xdr.TrustLineFlags(op.Authorize).IsAuthorized()
@@ -496,7 +526,9 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{},
 		}
 	case xdr.OperationTypeClawback:
 		op := operation.operation.Body.MustClawbackOp()
-		addAssetDetails(details, op.Asset, "")
+		if err := addAssetDetails(details, op.Asset, ""); err != nil {
+			return nil, err
+		}
 		addAccountAndMuxedAccountDetails(details, op.From, "from")
 		details["amount"] = amount.String(op.Amount)
 	case xdr.OperationTypeClawbackClaimableBalance:
@@ -509,7 +541,9 @@ func (operation *transactionOperationWrapper) Details() (map[string]interface{},
 	case xdr.OperationTypeSetTrustLineFlags:
 		op := operation.operation.Body.MustSetTrustLineFlagsOp()
 		details["trustor"] = op.Trustor.Address()
-		addAssetDetails(details, op.Asset, "")
+		if err := addAssetDetails(details, op.Asset, ""); err != nil {
+			return nil, err
+		}
 		if op.SetFlags > 0 {
 			addTrustLineFlagDetails(details, xdr.TrustLineFlags(op.SetFlags), "set")
 		}

From 98c135cbf8fe8d2b7247d0cd4c9e6182152a7d35 Mon Sep 17 00:00:00 2001
From: shawn <sreuland@users.noreply.github.com>
Date: Thu, 26 Oct 2023 11:03:55 -0700
Subject: [PATCH 13/17] integrating new loaders and builders into processors
 (#5083)

---
 services/horizon/docker/docker-compose.yml    |   2 +-
 .../horizon/docker/verify-range/Dockerfile    |   4 +-
 .../horizon/docker/verify-range/dependencies  |   6 +-
 .../internal/actions/transaction_test.go      |   1 +
 .../internal/db2/history/account_loader.go    |  15 +-
 .../db2/history/account_loader_test.go        |  24 ++-
 .../internal/db2/history/asset_loader.go      |  44 ++---
 .../internal/db2/history/asset_loader_test.go |  67 +++++---
 .../db2/history/claimable_balance_loader.go   |  12 +-
 .../history/claimable_balance_loader_test.go  |  18 +-
 .../internal/db2/history/fee_bump_scenario.go |   6 +-
 .../db2/history/liquidity_pool_loader.go      |  15 +-
 .../db2/history/liquidity_pool_loader_test.go |  22 +--
 ...n_participant_batch_insert_builder_test.go |   4 +-
 .../internal/db2/history/participants_test.go |   4 +-
 services/horizon/internal/ingest/fsm.go       |   2 +
 .../internal/ingest/group_processors.go       |  28 +++-
 .../internal/ingest/group_processors_test.go  |  42 +++--
 .../internal/ingest/processor_runner.go       |  81 +++++----
 .../internal/ingest/processor_runner_test.go  | 158 +++++++++++-------
 .../ingest/processors/change_processors.go    |  57 -------
 .../ingest/processors/ledgers_processor.go    |   6 +-
 .../internal/ingest/processors/main.go        |  65 +++++++
 .../stats_ledger_transaction_processor.go     |   7 +-
 ...stats_ledger_transaction_processor_test.go |  12 +-
 .../ingest/processors/trades_processor.go     |  31 +++-
 .../transaction_preconditions_test.go         |  71 --------
 .../internal/integration/txsub_test.go        |   6 +-
 support/db/batch_insert_builder_test.go       |   1 +
 support/db/fast_batch_insert_builder_test.go  |  18 +-
 support/db/internal_test.go                   |   1 +
 support/db/main_test.go                       |   9 +-
 32 files changed, 467 insertions(+), 372 deletions(-)

diff --git a/services/horizon/docker/docker-compose.yml b/services/horizon/docker/docker-compose.yml
index 40bced6677..377e26b0b4 100644
--- a/services/horizon/docker/docker-compose.yml
+++ b/services/horizon/docker/docker-compose.yml
@@ -1,7 +1,7 @@
 version: '3'
 services:
   horizon-postgres:
-    image: postgres:9.6.17-alpine
+    image: postgres:postgres:12-bullseye
     restart: on-failure
     environment:
       - POSTGRES_HOST_AUTH_METHOD=trust
diff --git a/services/horizon/docker/verify-range/Dockerfile b/services/horizon/docker/verify-range/Dockerfile
index 499f86881e..60fc6df745 100644
--- a/services/horizon/docker/verify-range/Dockerfile
+++ b/services/horizon/docker/verify-range/Dockerfile
@@ -1,14 +1,12 @@
 FROM ubuntu:20.04
 
-MAINTAINER Bartek Nowotarski <bartek@stellar.org>
-
 ARG STELLAR_CORE_VERSION 
 ENV STELLAR_CORE_VERSION=${STELLAR_CORE_VERSION:-*}
 # to remove tzdata interactive flow
 ENV DEBIAN_FRONTEND=noninteractive
 
 ADD dependencies /
-RUN ["chmod", "+x", "dependencies"]
+RUN ["chmod", "+x", "/dependencies"]
 RUN /dependencies
 
 ADD stellar-core.cfg /
diff --git a/services/horizon/docker/verify-range/dependencies b/services/horizon/docker/verify-range/dependencies
index fa622f9d2e..e17c6f4b5f 100644
--- a/services/horizon/docker/verify-range/dependencies
+++ b/services/horizon/docker/verify-range/dependencies
@@ -11,8 +11,8 @@ echo "deb https://apt.stellar.org $(lsb_release -cs) stable" | sudo tee -a /etc/
 apt-get update
 apt-get install -y stellar-core=${STELLAR_CORE_VERSION}
 
-wget -q https://dl.google.com/go/go1.18.linux-amd64.tar.gz
-tar -C /usr/local -xzf go1.18.linux-amd64.tar.gz
+wget -q https://dl.google.com/go/go1.20.linux-amd64.tar.gz
+tar -C /usr/local -xzf go1.20.linux-amd64.tar.gz
 
 git clone https://github.com/stellar/go.git stellar-go
 cd stellar-go
@@ -20,4 +20,4 @@ cd stellar-go
 # Below ensures we also fetch PR refs
 git config --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pull/*"
 git fetch --force --quiet origin
-/usr/local/go/bin/go build -v ./services/horizon
+/usr/local/go/bin/go build -v ./services/horizon/.
\ No newline at end of file
diff --git a/services/horizon/internal/actions/transaction_test.go b/services/horizon/internal/actions/transaction_test.go
index e029edef3a..b76cf1b0bf 100644
--- a/services/horizon/internal/actions/transaction_test.go
+++ b/services/horizon/internal/actions/transaction_test.go
@@ -149,6 +149,7 @@ func checkOuterHashResponse(
 }
 
 func TestFeeBumpTransactionPage(t *testing.T) {
+
 	tt := test.Start(t)
 	defer tt.Finish()
 	test.ResetHorizonDB(t, tt.HorizonDB)
diff --git a/services/horizon/internal/db2/history/account_loader.go b/services/horizon/internal/db2/history/account_loader.go
index e9fd9bedea..f3946b0448 100644
--- a/services/horizon/internal/db2/history/account_loader.go
+++ b/services/horizon/internal/db2/history/account_loader.go
@@ -28,7 +28,7 @@ const loaderLookupBatchSize = 50000
 
 // Value implements the database/sql/driver Valuer interface.
 func (a FutureAccountID) Value() (driver.Value, error) {
-	return a.loader.GetNow(a.address), nil
+	return a.loader.GetNow(a.address)
 }
 
 // AccountLoader will map account addresses to their history
@@ -71,11 +71,15 @@ func (a *AccountLoader) GetFuture(address string) FutureAccountID {
 // GetNow should only be called on values which were registered by
 // GetFuture() calls. Also, Exec() must be called before any GetNow
 // call can succeed.
-func (a *AccountLoader) GetNow(address string) int64 {
-	if id, ok := a.ids[address]; !ok {
-		panic(fmt.Errorf("address %v not present", address))
+func (a *AccountLoader) GetNow(address string) (int64, error) {
+	if !a.sealed {
+		return 0, fmt.Errorf(`invalid account loader state,  
+		Exec was not called yet to properly seal and resolve %v id`, address)
+	}
+	if internalID, ok := a.ids[address]; !ok {
+		return 0, fmt.Errorf(`account loader address %q was not found`, address)
 	} else {
-		return id
+		return internalID, nil
 	}
 }
 
@@ -205,5 +209,6 @@ func NewAccountLoaderStub() AccountLoaderStub {
 // Insert updates the wrapped AccountLoader so that the given account
 // address is mapped to the provided history account id
 func (a AccountLoaderStub) Insert(address string, id int64) {
+	a.Loader.sealed = true
 	a.Loader.ids[address] = id
 }
diff --git a/services/horizon/internal/db2/history/account_loader_test.go b/services/horizon/internal/db2/history/account_loader_test.go
index 11047f3be2..54d2c7a143 100644
--- a/services/horizon/internal/db2/history/account_loader_test.go
+++ b/services/horizon/internal/db2/history/account_loader_test.go
@@ -22,16 +22,11 @@ func TestAccountLoader(t *testing.T) {
 	}
 
 	loader := NewAccountLoader()
-	var futures []FutureAccountID
 	for _, address := range addresses {
 		future := loader.GetFuture(address)
-		futures = append(futures, future)
-		assert.Panics(t, func() {
-			loader.GetNow(address)
-		})
-		assert.Panics(t, func() {
-			future.Value()
-		})
+		_, err := future.Value()
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), `invalid account loader state,`)
 		duplicateFuture := loader.GetFuture(address)
 		assert.Equal(t, future, duplicateFuture)
 	}
@@ -42,15 +37,16 @@ func TestAccountLoader(t *testing.T) {
 	})
 
 	q := &Q{session}
-	for i, address := range addresses {
-		future := futures[i]
-		id := loader.GetNow(address)
-		val, err := future.Value()
+	for _, address := range addresses {
+		internalId, err := loader.GetNow(address)
 		assert.NoError(t, err)
-		assert.Equal(t, id, val)
 		var account Account
 		assert.NoError(t, q.AccountByAddress(context.Background(), &account, address))
-		assert.Equal(t, account.ID, id)
+		assert.Equal(t, account.ID, internalId)
 		assert.Equal(t, account.Address, address)
 	}
+
+	_, err := loader.GetNow("not present")
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), `was not found`)
 }
diff --git a/services/horizon/internal/db2/history/asset_loader.go b/services/horizon/internal/db2/history/asset_loader.go
index 6ef3d7a350..b5ee9a8326 100644
--- a/services/horizon/internal/db2/history/asset_loader.go
+++ b/services/horizon/internal/db2/history/asset_loader.go
@@ -5,6 +5,7 @@ import (
 	"database/sql/driver"
 	"fmt"
 	"sort"
+	"strings"
 
 	sq "github.com/Masterminds/squirrel"
 
@@ -21,11 +22,18 @@ type AssetKey struct {
 	Issuer string
 }
 
+func (key AssetKey) String() string {
+	if key.Type == xdr.AssetTypeToString[xdr.AssetTypeAssetTypeNative] {
+		return key.Type
+	}
+	return key.Type + "/" + key.Code + "/" + key.Issuer
+}
+
 // AssetKeyFromXDR constructs an AssetKey from an xdr asset
 func AssetKeyFromXDR(asset xdr.Asset) AssetKey {
 	return AssetKey{
 		Type:   xdr.AssetTypeToString[asset.Type],
-		Code:   asset.GetCode(),
+		Code:   strings.TrimRight(asset.GetCode(), "\x00"),
 		Issuer: asset.GetIssuer(),
 	}
 }
@@ -41,7 +49,7 @@ type FutureAssetID struct {
 
 // Value implements the database/sql/driver Valuer interface.
 func (a FutureAssetID) Value() (driver.Value, error) {
-	return a.loader.GetNow(a.asset), nil
+	return a.loader.GetNow(a.asset)
 }
 
 // AssetLoader will map assets to their history
@@ -81,11 +89,15 @@ func (a *AssetLoader) GetFuture(asset AssetKey) FutureAssetID {
 // GetNow should only be called on values which were registered by
 // GetFuture() calls. Also, Exec() must be called before any GetNow
 // call can succeed.
-func (a *AssetLoader) GetNow(asset AssetKey) int64 {
-	if id, ok := a.ids[asset]; !ok {
-		panic(fmt.Errorf("asset %v not present", asset))
+func (a *AssetLoader) GetNow(asset AssetKey) (int64, error) {
+	if !a.sealed {
+		return 0, fmt.Errorf(`invalid asset loader state,  
+		Exec was not called yet to properly seal and resolve %v id`, asset)
+	}
+	if internalID, ok := a.ids[asset]; !ok {
+		return 0, fmt.Errorf(`asset loader id %v was not found`, asset)
 	} else {
-		return id
+		return internalID, nil
 	}
 }
 
@@ -137,6 +149,11 @@ func (a *AssetLoader) Exec(ctx context.Context, session db.SessionInterface) err
 	assetTypes := make([]string, 0, len(a.set)-len(a.ids))
 	assetCodes := make([]string, 0, len(a.set)-len(a.ids))
 	assetIssuers := make([]string, 0, len(a.set)-len(a.ids))
+	// sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock
+	// https://github.com/stellar/go/issues/2370
+	sort.Slice(keys, func(i, j int) bool {
+		return keys[i].String() < keys[j].String()
+	})
 	insert := 0
 	for _, key := range keys {
 		if _, ok := a.ids[key]; ok {
@@ -152,20 +169,6 @@ func (a *AssetLoader) Exec(ctx context.Context, session db.SessionInterface) err
 		return nil
 	}
 	keys = keys[:insert]
-	// sort entries before inserting rows to prevent deadlocks on acquiring a ShareLock
-	// https://github.com/stellar/go/issues/2370
-	sort.Slice(keys, func(i, j int) bool {
-		if keys[i].Type < keys[j].Type {
-			return true
-		}
-		if keys[i].Code < keys[j].Code {
-			return true
-		}
-		if keys[i].Issuer < keys[j].Issuer {
-			return true
-		}
-		return false
-	})
 
 	err := bulkInsert(
 		ctx,
@@ -211,5 +214,6 @@ func NewAssetLoaderStub() AssetLoaderStub {
 // Insert updates the wrapped AssetLoaderStub so that the given asset
 // address is mapped to the provided history asset id
 func (a AssetLoaderStub) Insert(asset AssetKey, id int64) {
+	a.Loader.sealed = true
 	a.Loader.ids[asset] = id
 }
diff --git a/services/horizon/internal/db2/history/asset_loader_test.go b/services/horizon/internal/db2/history/asset_loader_test.go
index 99f510266c..d67163d764 100644
--- a/services/horizon/internal/db2/history/asset_loader_test.go
+++ b/services/horizon/internal/db2/history/asset_loader_test.go
@@ -12,6 +12,28 @@ import (
 	"github.com/stellar/go/xdr"
 )
 
+func TestAssetKeyToString(t *testing.T) {
+	num4key := AssetKey{
+		Type:   "credit_alphanum4",
+		Code:   "USD",
+		Issuer: "A1B2C3",
+	}
+
+	num12key := AssetKey{
+		Type:   "credit_alphanum12",
+		Code:   "USDABC",
+		Issuer: "A1B2C3",
+	}
+
+	nativekey := AssetKey{
+		Type: "native",
+	}
+
+	assert.Equal(t, num4key.String(), "credit_alphanum4/USD/A1B2C3")
+	assert.Equal(t, num12key.String(), "credit_alphanum12/USDABC/A1B2C3")
+	assert.Equal(t, nativekey.String(), "native")
+}
+
 func TestAssetLoader(t *testing.T) {
 	tt := test.Start(t)
 	defer tt.Finish()
@@ -22,30 +44,34 @@ func TestAssetLoader(t *testing.T) {
 	for i := 0; i < 100; i++ {
 		var key AssetKey
 		if i == 0 {
-			key.Type = "native"
+			key = AssetKeyFromXDR(xdr.Asset{Type: xdr.AssetTypeAssetTypeNative})
 		} else if i%2 == 0 {
-			key.Type = "credit_alphanum4"
-			key.Code = fmt.Sprintf("ab%d", i)
-			key.Issuer = keypair.MustRandom().Address()
+			code := [4]byte{0, 0, 0, 0}
+			copy(code[:], fmt.Sprintf("ab%d", i))
+			key = AssetKeyFromXDR(xdr.Asset{
+				Type: xdr.AssetTypeAssetTypeCreditAlphanum4,
+				AlphaNum4: &xdr.AlphaNum4{
+					AssetCode: code,
+					Issuer:    xdr.MustAddress(keypair.MustRandom().Address())}})
 		} else {
-			key.Type = "credit_alphanum12"
-			key.Code = fmt.Sprintf("abcdef%d", i)
-			key.Issuer = keypair.MustRandom().Address()
+			code := [12]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+			copy(code[:], fmt.Sprintf("abcdef%d", i))
+			key = AssetKeyFromXDR(xdr.Asset{
+				Type: xdr.AssetTypeAssetTypeCreditAlphanum12,
+				AlphaNum12: &xdr.AlphaNum12{
+					AssetCode: code,
+					Issuer:    xdr.MustAddress(keypair.MustRandom().Address())}})
+
 		}
 		keys = append(keys, key)
 	}
 
 	loader := NewAssetLoader()
-	var futures []FutureAssetID
 	for _, key := range keys {
 		future := loader.GetFuture(key)
-		futures = append(futures, future)
-		assert.Panics(t, func() {
-			loader.GetNow(key)
-		})
-		assert.Panics(t, func() {
-			future.Value()
-		})
+		_, err := future.Value()
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), `invalid asset loader state,`)
 		duplicateFuture := loader.GetFuture(key)
 		assert.Equal(t, future, duplicateFuture)
 	}
@@ -56,12 +82,9 @@ func TestAssetLoader(t *testing.T) {
 	})
 
 	q := &Q{session}
-	for i, key := range keys {
-		future := futures[i]
-		internalID := loader.GetNow(key)
-		val, err := future.Value()
+	for _, key := range keys {
+		internalID, err := loader.GetNow(key)
 		assert.NoError(t, err)
-		assert.Equal(t, internalID, val)
 		var assetXDR xdr.Asset
 		if key.Type == "native" {
 			assetXDR = xdr.MustNewNativeAsset()
@@ -72,4 +95,8 @@ func TestAssetLoader(t *testing.T) {
 		assert.NoError(t, err)
 		assert.Equal(t, assetID, internalID)
 	}
+
+	_, err := loader.GetNow(AssetKey{})
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), `was not found`)
 }
diff --git a/services/horizon/internal/db2/history/claimable_balance_loader.go b/services/horizon/internal/db2/history/claimable_balance_loader.go
index a077eb683e..dd7dee4ea5 100644
--- a/services/horizon/internal/db2/history/claimable_balance_loader.go
+++ b/services/horizon/internal/db2/history/claimable_balance_loader.go
@@ -23,7 +23,7 @@ type FutureClaimableBalanceID struct {
 
 // Value implements the database/sql/driver Valuer interface.
 func (a FutureClaimableBalanceID) Value() (driver.Value, error) {
-	return a.loader.getNow(a.id), nil
+	return a.loader.getNow(a.id)
 }
 
 // ClaimableBalanceLoader will map claimable balance ids to their internal
@@ -64,11 +64,15 @@ func (a *ClaimableBalanceLoader) GetFuture(id string) FutureClaimableBalanceID {
 // getNow should only be called on values which were registered by
 // GetFuture() calls. Also, Exec() must be called before any getNow
 // call can succeed.
-func (a *ClaimableBalanceLoader) getNow(id string) int64 {
+func (a *ClaimableBalanceLoader) getNow(id string) (int64, error) {
+	if !a.sealed {
+		return 0, fmt.Errorf(`invalid claimable balance loader state, 
+		Exec was not called yet to properly seal and resolve %v id`, id)
+	}
 	if internalID, ok := a.ids[id]; !ok {
-		panic(fmt.Errorf("id %v not present", id))
+		return 0, fmt.Errorf(`claimable balance loader id %q was not found`, id)
 	} else {
-		return internalID
+		return internalID, nil
 	}
 }
 
diff --git a/services/horizon/internal/db2/history/claimable_balance_loader_test.go b/services/horizon/internal/db2/history/claimable_balance_loader_test.go
index b119daa674..4dd7324521 100644
--- a/services/horizon/internal/db2/history/claimable_balance_loader_test.go
+++ b/services/horizon/internal/db2/history/claimable_balance_loader_test.go
@@ -32,12 +32,9 @@ func TestClaimableBalanceLoader(t *testing.T) {
 	for _, id := range ids {
 		future := loader.GetFuture(id)
 		futures = append(futures, future)
-		assert.Panics(t, func() {
-			loader.getNow(id)
-		})
-		assert.Panics(t, func() {
-			future.Value()
-		})
+		_, err := future.Value()
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), `invalid claimable balance loader state,`)
 		duplicateFuture := loader.GetFuture(id)
 		assert.Equal(t, future, duplicateFuture)
 	}
@@ -50,13 +47,16 @@ func TestClaimableBalanceLoader(t *testing.T) {
 	q := &Q{session}
 	for i, id := range ids {
 		future := futures[i]
-		internalID := loader.getNow(id)
-		val, err := future.Value()
+		internalID, err := future.Value()
 		assert.NoError(t, err)
-		assert.Equal(t, internalID, val)
 		cb, err := q.ClaimableBalanceByID(context.Background(), id)
 		assert.NoError(t, err)
 		assert.Equal(t, cb.BalanceID, id)
 		assert.Equal(t, cb.InternalID, internalID)
 	}
+
+	futureCb := &FutureClaimableBalanceID{id: "not-present", loader: loader}
+	_, err := futureCb.Value()
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), `was not found`)
 }
diff --git a/services/horizon/internal/db2/history/fee_bump_scenario.go b/services/horizon/internal/db2/history/fee_bump_scenario.go
index 5d155ac5e8..75dcc20d61 100644
--- a/services/horizon/internal/db2/history/fee_bump_scenario.go
+++ b/services/horizon/internal/db2/history/fee_bump_scenario.go
@@ -269,6 +269,7 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 	details, err := json.Marshal(map[string]string{
 		"bump_to": "98",
 	})
+
 	tt.Assert.NoError(err)
 
 	tt.Assert.NoError(opBuilder.Add(
@@ -296,9 +297,10 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 		EffectSequenceBumped,
 		details,
 	)
+
 	tt.Assert.NoError(err)
-	tt.Assert.NoError(accountLoader.Exec(ctx, q))
-	tt.Assert.NoError(effectBuilder.Exec(ctx, q))
+	tt.Assert.NoError(accountLoader.Exec(ctx, q.SessionInterface))
+	tt.Assert.NoError(effectBuilder.Exec(ctx, q.SessionInterface))
 
 	tt.Assert.NoError(q.Commit())
 
diff --git a/services/horizon/internal/db2/history/liquidity_pool_loader.go b/services/horizon/internal/db2/history/liquidity_pool_loader.go
index 7c2fe6fd4d..cf89ae67b4 100644
--- a/services/horizon/internal/db2/history/liquidity_pool_loader.go
+++ b/services/horizon/internal/db2/history/liquidity_pool_loader.go
@@ -23,7 +23,7 @@ type FutureLiquidityPoolID struct {
 
 // Value implements the database/sql/driver Valuer interface.
 func (a FutureLiquidityPoolID) Value() (driver.Value, error) {
-	return a.loader.GetNow(a.id), nil
+	return a.loader.GetNow(a.id)
 }
 
 // LiquidityPoolLoader will map liquidity pools to their internal
@@ -64,11 +64,15 @@ func (a *LiquidityPoolLoader) GetFuture(id string) FutureLiquidityPoolID {
 // GetNow should only be called on values which were registered by
 // GetFuture() calls. Also, Exec() must be called before any GetNow
 // call can succeed.
-func (a *LiquidityPoolLoader) GetNow(id string) int64 {
-	if id, ok := a.ids[id]; !ok {
-		panic(fmt.Errorf("id %v not present", id))
+func (a *LiquidityPoolLoader) GetNow(id string) (int64, error) {
+	if !a.sealed {
+		return 0, fmt.Errorf(`invalid liquidity pool loader state,  
+		Exec was not called yet to properly seal and resolve %v id`, id)
+	}
+	if internalID, ok := a.ids[id]; !ok {
+		return 0, fmt.Errorf(`liquidity pool loader id %q was not found`, id)
 	} else {
-		return id
+		return internalID, nil
 	}
 }
 
@@ -156,5 +160,6 @@ func NewLiquidityPoolLoaderStub() LiquidityPoolLoaderStub {
 // Insert updates the wrapped LiquidityPoolLoader so that the given liquidity pool
 // is mapped to the provided history liquidity pool id
 func (a LiquidityPoolLoaderStub) Insert(lp string, id int64) {
+	a.Loader.sealed = true
 	a.Loader.ids[lp] = id
 }
diff --git a/services/horizon/internal/db2/history/liquidity_pool_loader_test.go b/services/horizon/internal/db2/history/liquidity_pool_loader_test.go
index e2b1e05beb..6e5b4addf7 100644
--- a/services/horizon/internal/db2/history/liquidity_pool_loader_test.go
+++ b/services/horizon/internal/db2/history/liquidity_pool_loader_test.go
@@ -25,16 +25,11 @@ func TestLiquidityPoolLoader(t *testing.T) {
 	}
 
 	loader := NewLiquidityPoolLoader()
-	var futures []FutureLiquidityPoolID
 	for _, id := range ids {
 		future := loader.GetFuture(id)
-		futures = append(futures, future)
-		assert.Panics(t, func() {
-			loader.GetNow(id)
-		})
-		assert.Panics(t, func() {
-			future.Value()
-		})
+		_, err := future.Value()
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), `invalid liquidity pool loader state,`)
 		duplicateFuture := loader.GetFuture(id)
 		assert.Equal(t, future, duplicateFuture)
 	}
@@ -45,15 +40,16 @@ func TestLiquidityPoolLoader(t *testing.T) {
 	})
 
 	q := &Q{session}
-	for i, id := range ids {
-		future := futures[i]
-		internalID := loader.GetNow(id)
-		val, err := future.Value()
+	for _, id := range ids {
+		internalID, err := loader.GetNow(id)
 		assert.NoError(t, err)
-		assert.Equal(t, internalID, val)
 		lp, err := q.LiquidityPoolByID(context.Background(), id)
 		assert.NoError(t, err)
 		assert.Equal(t, lp.PoolID, id)
 		assert.Equal(t, lp.InternalID, internalID)
 	}
+
+	_, err := loader.GetNow("not present")
+	assert.Error(t, err)
+	assert.Contains(t, err.Error(), `was not found`)
 }
diff --git a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
index fc2ca9c831..508bfa22cf 100644
--- a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
+++ b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
@@ -43,6 +43,8 @@ func TestAddOperationParticipants(t *testing.T) {
 
 		op := ops[0]
 		tt.Assert.Equal(int64(240518172673), op.OperationID)
-		tt.Assert.Equal(accountLoader.GetNow(address), op.AccountID)
+		val, err := accountLoader.GetNow(address)
+		tt.Assert.NoError(err)
+		tt.Assert.Equal(val, op.AccountID)
 	}
 }
diff --git a/services/horizon/internal/db2/history/participants_test.go b/services/horizon/internal/db2/history/participants_test.go
index 16671098bf..07f6d59c3e 100644
--- a/services/horizon/internal/db2/history/participants_test.go
+++ b/services/horizon/internal/db2/history/participants_test.go
@@ -63,7 +63,9 @@ func TestTransactionParticipantsBatch(t *testing.T) {
 		{TransactionID: 2},
 	}
 	for i := range expected {
-		expected[i].AccountID = accountLoader.GetNow(addresses[i])
+		val, err := accountLoader.GetNow(addresses[i])
+		tt.Assert.NoError(err)
+		expected[i].AccountID = val
 	}
 	tt.Assert.ElementsMatch(expected, participants)
 }
diff --git a/services/horizon/internal/ingest/fsm.go b/services/horizon/internal/ingest/fsm.go
index 38e3fe9ed7..284ea7127b 100644
--- a/services/horizon/internal/ingest/fsm.go
+++ b/services/horizon/internal/ingest/fsm.go
@@ -459,6 +459,7 @@ func (r resumeState) run(s *system) (transition, error) {
 
 		// Update cursor if there's more than one ingesting instance: either
 		// Captive-Core or DB ingestion connected to another Stellar-Core.
+		// remove now?
 		if err = s.updateCursor(lastIngestedLedger); err != nil {
 			// Don't return updateCursor error.
 			log.WithError(err).Warn("error updating stellar-core cursor")
@@ -524,6 +525,7 @@ func (r resumeState) run(s *system) (transition, error) {
 		return retryResume(r), err
 	}
 
+	//TODO remove now? stellar-core-db-url is removed
 	if err = s.updateCursor(ingestLedger); err != nil {
 		// Don't return updateCursor error.
 		log.WithError(err).Warn("error updating stellar-core cursor")
diff --git a/services/horizon/internal/ingest/group_processors.go b/services/horizon/internal/ingest/group_processors.go
index 86622810b5..af486a35cf 100644
--- a/services/horizon/internal/ingest/group_processors.go
+++ b/services/horizon/internal/ingest/group_processors.go
@@ -7,7 +7,9 @@ import (
 
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/ingest/processors"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
+	"github.com/stellar/go/xdr"
 )
 
 type processorsRunDurations map[string]time.Duration
@@ -51,21 +53,23 @@ func (g groupChangeProcessors) Commit(ctx context.Context) error {
 }
 
 type groupTransactionProcessors struct {
-	processors []horizonTransactionProcessor
+	processors  []horizonTransactionProcessor
+	lazyLoaders []horizonLazyLoader
 	processorsRunDurations
 }
 
-func newGroupTransactionProcessors(processors []horizonTransactionProcessor) *groupTransactionProcessors {
+func newGroupTransactionProcessors(processors []horizonTransactionProcessor, lazyLoaders []horizonLazyLoader) *groupTransactionProcessors {
 	return &groupTransactionProcessors{
 		processors:             processors,
 		processorsRunDurations: make(map[string]time.Duration),
+		lazyLoaders:            lazyLoaders,
 	}
 }
 
-func (g groupTransactionProcessors) ProcessTransaction(ctx context.Context, tx ingest.LedgerTransaction) error {
+func (g groupTransactionProcessors) ProcessTransaction(lcm xdr.LedgerCloseMeta, tx ingest.LedgerTransaction) error {
 	for _, p := range g.processors {
 		startTime := time.Now()
-		if err := p.ProcessTransaction(ctx, tx); err != nil {
+		if err := p.ProcessTransaction(lcm, tx); err != nil {
 			return errors.Wrapf(err, "error in %T.ProcessTransaction", p)
 		}
 		g.AddRunDuration(fmt.Sprintf("%T", p), startTime)
@@ -73,11 +77,21 @@ func (g groupTransactionProcessors) ProcessTransaction(ctx context.Context, tx i
 	return nil
 }
 
-func (g groupTransactionProcessors) Commit(ctx context.Context) error {
+func (g groupTransactionProcessors) Flush(ctx context.Context, session db.SessionInterface) error {
+	// need to trigger all lazy loaders to now resolve their future placeholders
+	// with real db values first
+	for _, loader := range g.lazyLoaders {
+		if err := loader.Exec(ctx, session); err != nil {
+			return errors.Wrapf(err, "error during lazy loader resolution, %T.Exec", loader)
+		}
+	}
+
+	// now flush each processor which may call loader.GetNow(), which
+	// required the prior loader.Exec() to have been called.
 	for _, p := range g.processors {
 		startTime := time.Now()
-		if err := p.Commit(ctx); err != nil {
-			return errors.Wrapf(err, "error in %T.Commit", p)
+		if err := p.Flush(ctx, session); err != nil {
+			return errors.Wrapf(err, "error in %T.Flush", p)
 		}
 		g.AddRunDuration(fmt.Sprintf("%T", p), startTime)
 	}
diff --git a/services/horizon/internal/ingest/group_processors_test.go b/services/horizon/internal/ingest/group_processors_test.go
index 6848c24a66..73d4f56f3f 100644
--- a/services/horizon/internal/ingest/group_processors_test.go
+++ b/services/horizon/internal/ingest/group_processors_test.go
@@ -11,6 +11,8 @@ import (
 	"github.com/stretchr/testify/suite"
 
 	"github.com/stellar/go/ingest"
+	"github.com/stellar/go/support/db"
+	"github.com/stellar/go/xdr"
 )
 
 var _ horizonChangeProcessor = (*mockHorizonChangeProcessor)(nil)
@@ -35,13 +37,13 @@ type mockHorizonTransactionProcessor struct {
 	mock.Mock
 }
 
-func (m *mockHorizonTransactionProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error {
-	args := m.Called(ctx, transaction)
+func (m *mockHorizonTransactionProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error {
+	args := m.Called(lcm, transaction)
 	return args.Error(0)
 }
 
-func (m *mockHorizonTransactionProcessor) Commit(ctx context.Context) error {
-	args := m.Called(ctx)
+func (m *mockHorizonTransactionProcessor) Flush(ctx context.Context, session db.SessionInterface) error {
+	args := m.Called(ctx, session)
 	return args.Error(0)
 }
 
@@ -124,6 +126,7 @@ type GroupTransactionProcessorsTestSuiteLedger struct {
 	processors *groupTransactionProcessors
 	processorA *mockHorizonTransactionProcessor
 	processorB *mockHorizonTransactionProcessor
+	session    db.SessionInterface
 }
 
 func TestGroupTransactionProcessorsTestSuiteLedger(t *testing.T) {
@@ -137,7 +140,8 @@ func (s *GroupTransactionProcessorsTestSuiteLedger) SetupTest() {
 	s.processors = newGroupTransactionProcessors([]horizonTransactionProcessor{
 		s.processorA,
 		s.processorB,
-	})
+	}, nil)
+	s.session = &db.MockSession{}
 }
 
 func (s *GroupTransactionProcessorsTestSuiteLedger) TearDownTest() {
@@ -147,46 +151,48 @@ func (s *GroupTransactionProcessorsTestSuiteLedger) TearDownTest() {
 
 func (s *GroupTransactionProcessorsTestSuiteLedger) TestProcessTransactionFails() {
 	transaction := ingest.LedgerTransaction{}
+	closeMeta := xdr.LedgerCloseMeta{}
 	s.processorA.
-		On("ProcessTransaction", s.ctx, transaction).
+		On("ProcessTransaction", closeMeta, transaction).
 		Return(errors.New("transient error")).Once()
 
-	err := s.processors.ProcessTransaction(s.ctx, transaction)
+	err := s.processors.ProcessTransaction(closeMeta, transaction)
 	s.Assert().Error(err)
 	s.Assert().EqualError(err, "error in *ingest.mockHorizonTransactionProcessor.ProcessTransaction: transient error")
 }
 
 func (s *GroupTransactionProcessorsTestSuiteLedger) TestProcessTransactionSucceeds() {
 	transaction := ingest.LedgerTransaction{}
+	closeMeta := xdr.LedgerCloseMeta{}
 	s.processorA.
-		On("ProcessTransaction", s.ctx, transaction).
+		On("ProcessTransaction", closeMeta, transaction).
 		Return(nil).Once()
 	s.processorB.
-		On("ProcessTransaction", s.ctx, transaction).
+		On("ProcessTransaction", closeMeta, transaction).
 		Return(nil).Once()
 
-	err := s.processors.ProcessTransaction(s.ctx, transaction)
+	err := s.processors.ProcessTransaction(closeMeta, transaction)
 	s.Assert().NoError(err)
 }
 
-func (s *GroupTransactionProcessorsTestSuiteLedger) TestCommitFails() {
+func (s *GroupTransactionProcessorsTestSuiteLedger) TestFlushFails() {
 	s.processorA.
-		On("Commit", s.ctx).
+		On("Flush", s.ctx, s.session).
 		Return(errors.New("transient error")).Once()
 
-	err := s.processors.Commit(s.ctx)
+	err := s.processors.Flush(s.ctx, s.session)
 	s.Assert().Error(err)
-	s.Assert().EqualError(err, "error in *ingest.mockHorizonTransactionProcessor.Commit: transient error")
+	s.Assert().EqualError(err, "error in *ingest.mockHorizonTransactionProcessor.Flush: transient error")
 }
 
-func (s *GroupTransactionProcessorsTestSuiteLedger) TestCommitSucceeds() {
+func (s *GroupTransactionProcessorsTestSuiteLedger) TestFlushSucceeds() {
 	s.processorA.
-		On("Commit", s.ctx).
+		On("Flush", s.ctx, s.session).
 		Return(nil).Once()
 	s.processorB.
-		On("Commit", s.ctx).
+		On("Flush", s.ctx, s.session).
 		Return(nil).Once()
 
-	err := s.processors.Commit(s.ctx)
+	err := s.processors.Flush(s.ctx, s.session)
 	s.Assert().NoError(err)
 }
diff --git a/services/horizon/internal/ingest/processor_runner.go b/services/horizon/internal/ingest/processor_runner.go
index 481a4e7d52..f66b10c1e3 100644
--- a/services/horizon/internal/ingest/processor_runner.go
+++ b/services/horizon/internal/ingest/processor_runner.go
@@ -32,7 +32,10 @@ type horizonChangeProcessor interface {
 
 type horizonTransactionProcessor interface {
 	processors.LedgerTransactionProcessor
-	Commit(context.Context) error
+}
+
+type horizonLazyLoader interface {
+	Exec(ctx context.Context, session db.SessionInterface) error
 }
 
 type statsChangeProcessor struct {
@@ -47,10 +50,6 @@ type statsLedgerTransactionProcessor struct {
 	*processors.StatsLedgerTransactionProcessor
 }
 
-func (statsLedgerTransactionProcessor) Commit(ctx context.Context) error {
-	return nil
-}
-
 type ledgerStats struct {
 	changeStats          ingest.StatsChangeProcessorResults
 	changeDurations      processorsRunDurations
@@ -135,24 +134,36 @@ func buildChangeProcessor(
 func (s *ProcessorRunner) buildTransactionProcessor(
 	ledgerTransactionStats *processors.StatsLedgerTransactionProcessor,
 	tradeProcessor *processors.TradeProcessor,
-	ledger xdr.LedgerHeaderHistoryEntry,
+	ledgersProcessor *processors.LedgersProcessor,
 ) *groupTransactionProcessors {
+	accountLoader := history.NewAccountLoader()
+	assetLoader := history.NewAssetLoader()
+	lpLoader := history.NewLiquidityPoolLoader()
+	cbLoader := history.NewClaimableBalanceLoader()
+
+	lazyLoaders := []horizonLazyLoader{accountLoader, assetLoader, lpLoader, cbLoader}
+
 	statsLedgerTransactionProcessor := &statsLedgerTransactionProcessor{
 		StatsLedgerTransactionProcessor: ledgerTransactionStats,
 	}
-	*tradeProcessor = *processors.NewTradeProcessor(s.session, s.historyQ, ledger)
-	sequence := uint32(ledger.Header.LedgerSeq)
-	return newGroupTransactionProcessors([]horizonTransactionProcessor{
+	*tradeProcessor = *processors.NewTradeProcessor(accountLoader,
+		lpLoader, assetLoader, s.historyQ.NewTradeBatchInsertBuilder())
+
+	processors := []horizonTransactionProcessor{
 		statsLedgerTransactionProcessor,
-		processors.NewEffectProcessor(s.session, s.historyQ, sequence),
-		processors.NewLedgerProcessor(s.session, s.historyQ, ledger, CurrentVersion),
-		processors.NewOperationProcessor(s.session, s.historyQ, sequence),
+		processors.NewEffectProcessor(accountLoader, s.historyQ.NewEffectBatchInsertBuilder()),
+		ledgersProcessor,
+		processors.NewOperationProcessor(s.historyQ.NewOperationBatchInsertBuilder()),
 		tradeProcessor,
-		processors.NewParticipantsProcessor(s.session, s.historyQ, sequence),
-		processors.NewTransactionProcessor(s.session, s.historyQ, sequence),
-		processors.NewClaimableBalancesTransactionProcessor(s.session, s.historyQ, sequence),
-		processors.NewLiquidityPoolsTransactionProcessor(s.session, s.historyQ, sequence),
-	})
+		processors.NewParticipantsProcessor(accountLoader,
+			s.historyQ.NewTransactionParticipantsBatchInsertBuilder(), s.historyQ.NewOperationParticipantBatchInsertBuilder()),
+		processors.NewTransactionProcessor(s.historyQ.NewTransactionBatchInsertBuilder()),
+		processors.NewClaimableBalancesTransactionProcessor(cbLoader,
+			s.historyQ.NewTransactionClaimableBalanceBatchInsertBuilder(), s.historyQ.NewOperationClaimableBalanceBatchInsertBuilder()),
+		processors.NewLiquidityPoolsTransactionProcessor(lpLoader,
+			s.historyQ.NewTransactionLiquidityPoolBatchInsertBuilder(), s.historyQ.NewOperationLiquidityPoolBatchInsertBuilder())}
+
+	return newGroupTransactionProcessors(processors, lazyLoaders)
 }
 
 func (s *ProcessorRunner) buildTransactionFilterer() *groupTransactionFilterers {
@@ -164,15 +175,15 @@ func (s *ProcessorRunner) buildTransactionFilterer() *groupTransactionFilterers
 	return newGroupTransactionFilterers(f)
 }
 
-func (s *ProcessorRunner) buildFilteredOutProcessor(ledger xdr.LedgerHeaderHistoryEntry) *groupTransactionProcessors {
+func (s *ProcessorRunner) buildFilteredOutProcessor() *groupTransactionProcessors {
 	// when in online mode, the submission result processor must always run (regardless of filtering)
 	var p []horizonTransactionProcessor
 	if s.config.EnableIngestionFiltering {
-		txSubProc := processors.NewTransactionFilteredTmpProcessor(s.session, s.historyQ, uint32(ledger.Header.LedgerSeq))
+		txSubProc := processors.NewTransactionFilteredTmpProcessor(s.historyQ.NewTransactionFilteredTmpBatchInsertBuilder())
 		p = append(p, txSubProc)
 	}
 
-	return newGroupTransactionProcessors(p)
+	return newGroupTransactionProcessors(p, nil)
 }
 
 // checkIfProtocolVersionSupported checks if this Horizon version supports the
@@ -311,26 +322,31 @@ func (s *ProcessorRunner) RunTransactionProcessorsOnLedger(ledger xdr.LedgerClos
 		transactionReader      *ingest.LedgerTransactionReader
 	)
 
+	if err = s.checkIfProtocolVersionSupported(ledger.ProtocolVersion()); err != nil {
+		err = errors.Wrap(err, "Error while checking for supported protocol version")
+		return
+	}
+
+	// ensure capture of the ledger to history regardless of whether it has transactions.
+	ledgersProcessor := processors.NewLedgerProcessor(s.historyQ.NewLedgerBatchInsertBuilder(), CurrentVersion)
+	ledgersProcessor.ProcessLedger(ledger)
+
 	transactionReader, err = ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(s.config.NetworkPassphrase, ledger)
 	if err != nil {
 		err = errors.Wrap(err, "Error creating ledger reader")
 		return
 	}
 
-	if err = s.checkIfProtocolVersionSupported(ledger.ProtocolVersion()); err != nil {
-		err = errors.Wrap(err, "Error while checking for supported protocol version")
-		return
-	}
-	header := transactionReader.GetHeader()
 	groupTransactionFilterers := s.buildTransactionFilterer()
-	groupFilteredOutProcessors := s.buildFilteredOutProcessor(header)
+	groupFilteredOutProcessors := s.buildFilteredOutProcessor()
 	groupTransactionProcessors := s.buildTransactionProcessor(
-		&ledgerTransactionStats, &tradeProcessor, header)
+		&ledgerTransactionStats, &tradeProcessor, ledgersProcessor)
 	err = processors.StreamLedgerTransactions(s.ctx,
 		groupTransactionFilterers,
 		groupFilteredOutProcessors,
 		groupTransactionProcessors,
 		transactionReader,
+		ledger,
 	)
 	if err != nil {
 		err = errors.Wrap(err, "Error streaming changes from ledger")
@@ -338,9 +354,9 @@ func (s *ProcessorRunner) RunTransactionProcessorsOnLedger(ledger xdr.LedgerClos
 	}
 
 	if s.config.EnableIngestionFiltering {
-		err = groupFilteredOutProcessors.Commit(s.ctx)
+		err = groupFilteredOutProcessors.Flush(s.ctx, s.session)
 		if err != nil {
-			err = errors.Wrap(err, "Error committing filtered changes from processor")
+			err = errors.Wrap(err, "Error flushing temp filtered tx from processor")
 			return
 		}
 		if time.Since(s.lastTransactionsTmpGC) > transactionsFilteredTmpGCPeriod {
@@ -348,9 +364,9 @@ func (s *ProcessorRunner) RunTransactionProcessorsOnLedger(ledger xdr.LedgerClos
 		}
 	}
 
-	err = groupTransactionProcessors.Commit(s.ctx)
+	err = groupTransactionProcessors.Flush(s.ctx, s.session)
 	if err != nil {
-		err = errors.Wrap(err, "Error committing changes from processor")
+		err = errors.Wrap(err, "Error flushing changes from processor")
 		return
 	}
 
@@ -390,9 +406,6 @@ func (s *ProcessorRunner) RunAllProcessorsOnLedger(ledger xdr.LedgerCloseMeta) (
 
 	stats.transactionStats, stats.transactionDurations, stats.tradeStats, err =
 		s.RunTransactionProcessorsOnLedger(ledger)
-	if err != nil {
-		return
-	}
 
 	return
 }
diff --git a/services/horizon/internal/ingest/processor_runner_test.go b/services/horizon/internal/ingest/processor_runner_test.go
index c01ee53730..46796807fc 100644
--- a/services/horizon/internal/ingest/processor_runner_test.go
+++ b/services/horizon/internal/ingest/processor_runner_test.go
@@ -234,17 +234,31 @@ func TestProcessorRunnerBuildChangeProcessor(t *testing.T) {
 
 func TestProcessorRunnerBuildTransactionProcessor(t *testing.T) {
 	ctx := context.Background()
-	maxBatchSize := 100000
 
 	q := &mockDBQ{}
 	defer mock.AssertExpectationsForObjects(t, q)
 
-	q.MockQOperations.On("NewOperationBatchInsertBuilder").
-		Return(&history.MockOperationsBatchInsertBuilder{}).Twice() // Twice = with/without failed
 	q.MockQTransactions.On("NewTransactionBatchInsertBuilder").
-		Return(&history.MockTransactionsBatchInsertBuilder{}).Twice()
-	q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize).
-		Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Twice()
+		Return(&history.MockTransactionsBatchInsertBuilder{})
+	q.On("NewTradeBatchInsertBuilder").Return(&history.MockTradeBatchInsertBuilder{})
+	q.MockQLedgers.On("NewLedgerBatchInsertBuilder").
+		Return(&history.MockLedgersBatchInsertBuilder{})
+	q.MockQEffects.On("NewEffectBatchInsertBuilder").
+		Return(&history.MockEffectBatchInsertBuilder{})
+	q.MockQOperations.On("NewOperationBatchInsertBuilder").
+		Return(&history.MockOperationsBatchInsertBuilder{})
+	q.On("NewTransactionParticipantsBatchInsertBuilder").
+		Return(&history.MockTransactionParticipantsBatchInsertBuilder{})
+	q.On("NewOperationParticipantBatchInsertBuilder").
+		Return(&history.MockOperationParticipantBatchInsertBuilder{})
+	q.MockQHistoryClaimableBalances.On("NewTransactionClaimableBalanceBatchInsertBuilder").
+		Return(&history.MockTransactionClaimableBalanceBatchInsertBuilder{})
+	q.MockQHistoryClaimableBalances.On("NewOperationClaimableBalanceBatchInsertBuilder").
+		Return(&history.MockOperationClaimableBalanceBatchInsertBuilder{})
+	q.MockQHistoryLiquidityPools.On("NewTransactionLiquidityPoolBatchInsertBuilder").
+		Return(&history.MockTransactionLiquidityPoolBatchInsertBuilder{})
+	q.MockQHistoryLiquidityPools.On("NewOperationLiquidityPoolBatchInsertBuilder").
+		Return(&history.MockOperationLiquidityPoolBatchInsertBuilder{})
 
 	runner := ProcessorRunner{
 		ctx:      ctx,
@@ -254,17 +268,19 @@ func TestProcessorRunnerBuildTransactionProcessor(t *testing.T) {
 
 	stats := &processors.StatsLedgerTransactionProcessor{}
 	trades := &processors.TradeProcessor{}
-	ledger := xdr.LedgerHeaderHistoryEntry{}
-	processor := runner.buildTransactionProcessor(stats, trades, ledger)
-	assert.IsType(t, &groupTransactionProcessors{}, processor)
 
+	ledgersProcessor := &processors.LedgersProcessor{}
+
+	processor := runner.buildTransactionProcessor(stats, trades, ledgersProcessor)
+	assert.IsType(t, &groupTransactionProcessors{}, processor)
 	assert.IsType(t, &statsLedgerTransactionProcessor{}, processor.processors[0])
 	assert.IsType(t, &processors.EffectProcessor{}, processor.processors[1])
 	assert.IsType(t, &processors.LedgersProcessor{}, processor.processors[2])
 	assert.IsType(t, &processors.OperationProcessor{}, processor.processors[3])
 	assert.IsType(t, &processors.TradeProcessor{}, processor.processors[4])
 	assert.IsType(t, &processors.ParticipantsProcessor{}, processor.processors[5])
-	assert.IsType(t, &processors.TransactionProcessor{}, processor.processors[6])
+	assert.IsType(t, &processors.ClaimableBalancesTransactionProcessor{}, processor.processors[7])
+	assert.IsType(t, &processors.LiquidityPoolsTransactionProcessor{}, processor.processors[8])
 }
 
 func TestProcessorRunnerWithFilterEnabled(t *testing.T) {
@@ -291,33 +307,17 @@ func TestProcessorRunnerWithFilterEnabled(t *testing.T) {
 	}
 
 	// Batches
-	mockAccountSignersBatchInsertBuilder := &history.MockAccountSignersBatchInsertBuilder{}
-	defer mock.AssertExpectationsForObjects(t, mockAccountSignersBatchInsertBuilder)
-	q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize).
-		Return(mockAccountSignersBatchInsertBuilder).Once()
-
-	mockOperationsBatchInsertBuilder := &history.MockOperationsBatchInsertBuilder{}
-	defer mock.AssertExpectationsForObjects(t, mockOperationsBatchInsertBuilder)
-	mockOperationsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once()
-	q.MockQOperations.On("NewOperationBatchInsertBuilder").
-		Return(mockOperationsBatchInsertBuilder).Twice()
-
-	mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{}
-	defer mock.AssertExpectationsForObjects(t, mockTransactionsBatchInsertBuilder)
-	mockTransactionsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Twice()
-
-	q.MockQTransactions.On("NewTransactionBatchInsertBuilder").
-		Return(mockTransactionsBatchInsertBuilder)
-
+	mockTransactionsFilteredTmpBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{}
+	defer mock.AssertExpectationsForObjects(t, mockTransactionsFilteredTmpBatchInsertBuilder)
+	mockTransactionsFilteredTmpBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once()
 	q.MockQTransactions.On("NewTransactionFilteredTmpBatchInsertBuilder").
-		Return(mockTransactionsBatchInsertBuilder)
-
-	q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize).
-		Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once()
+		Return(mockTransactionsFilteredTmpBatchInsertBuilder)
 
 	q.On("DeleteTransactionsFilteredTmpOlderThan", ctx, mock.AnythingOfType("uint64")).
 		Return(int64(0), nil)
 
+	defer mock.AssertExpectationsForObjects(t, mockBatchBuilders(q, mockSession, ctx, maxBatchSize)...)
+
 	mockBatchInsertBuilder := &history.MockLedgersBatchInsertBuilder{}
 	q.MockQLedgers.On("NewLedgerBatchInsertBuilder").Return(mockBatchInsertBuilder)
 	mockBatchInsertBuilder.On(
@@ -364,25 +364,7 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) {
 	}
 
 	// Batches
-	mockAccountSignersBatchInsertBuilder := &history.MockAccountSignersBatchInsertBuilder{}
-	defer mock.AssertExpectationsForObjects(t, mockAccountSignersBatchInsertBuilder)
-	q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize).
-		Return(mockAccountSignersBatchInsertBuilder).Once()
-
-	mockOperationsBatchInsertBuilder := &history.MockOperationsBatchInsertBuilder{}
-	defer mock.AssertExpectationsForObjects(t, mockOperationsBatchInsertBuilder)
-	mockOperationsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once()
-	q.MockQOperations.On("NewOperationBatchInsertBuilder").
-		Return(mockOperationsBatchInsertBuilder).Twice()
-
-	mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{}
-	defer mock.AssertExpectationsForObjects(t, mockTransactionsBatchInsertBuilder)
-	mockTransactionsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once()
-	q.MockQTransactions.On("NewTransactionBatchInsertBuilder").
-		Return(mockTransactionsBatchInsertBuilder).Twice()
-
-	q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize).
-		Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once()
+	defer mock.AssertExpectationsForObjects(t, mockBatchBuilders(q, mockSession, ctx, maxBatchSize)...)
 
 	mockBatchInsertBuilder := &history.MockLedgersBatchInsertBuilder{}
 	q.MockQLedgers.On("NewLedgerBatchInsertBuilder").Return(mockBatchInsertBuilder)
@@ -429,21 +411,21 @@ func TestProcessorRunnerRunAllProcessorsOnLedgerProtocolVersionNotSupported(t *t
 	}
 
 	// Batches
+	mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{}
+	q.MockQTransactions.On("NewTransactionBatchInsertBuilder", maxBatchSize).
+		Return(mockTransactionsBatchInsertBuilder).Twice()
 
 	mockAccountSignersBatchInsertBuilder := &history.MockAccountSignersBatchInsertBuilder{}
-	defer mock.AssertExpectationsForObjects(t, mockAccountSignersBatchInsertBuilder)
 	q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize).
 		Return(mockAccountSignersBatchInsertBuilder).Once()
 
 	mockOperationsBatchInsertBuilder := &history.MockOperationsBatchInsertBuilder{}
-	defer mock.AssertExpectationsForObjects(t, mockOperationsBatchInsertBuilder)
-	q.MockQOperations.On("NewOperationBatchInsertBuilder", maxBatchSize).
+	q.MockQOperations.On("NewOperationBatchInsertBuilder").
 		Return(mockOperationsBatchInsertBuilder).Twice()
 
-	mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{}
-	defer mock.AssertExpectationsForObjects(t, mockTransactionsBatchInsertBuilder)
-	q.MockQTransactions.On("NewTransactionBatchInsertBuilder", maxBatchSize).
-		Return(mockTransactionsBatchInsertBuilder).Twice()
+	defer mock.AssertExpectationsForObjects(t, mockTransactionsBatchInsertBuilder,
+		mockAccountSignersBatchInsertBuilder,
+		mockOperationsBatchInsertBuilder)
 
 	runner := ProcessorRunner{
 		ctx:      ctx,
@@ -460,3 +442,63 @@ func TestProcessorRunnerRunAllProcessorsOnLedgerProtocolVersionNotSupported(t *t
 		),
 	)
 }
+
+func mockBatchBuilders(q *mockDBQ, mockSession *db.MockSession, ctx context.Context, maxBatchSize int) []interface{} {
+	mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{}
+	mockTransactionsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once()
+	q.MockQTransactions.On("NewTransactionBatchInsertBuilder").
+		Return(mockTransactionsBatchInsertBuilder)
+
+	mockAccountSignersBatchInsertBuilder := &history.MockAccountSignersBatchInsertBuilder{}
+	q.MockQSigners.On("NewAccountSignersBatchInsertBuilder", maxBatchSize).
+		Return(mockAccountSignersBatchInsertBuilder).Once()
+
+	mockOperationsBatchInsertBuilder := &history.MockOperationsBatchInsertBuilder{}
+	mockOperationsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once()
+	q.MockQOperations.On("NewOperationBatchInsertBuilder").
+		Return(mockOperationsBatchInsertBuilder).Twice()
+
+	mockEffectBatchInsertBuilder := &history.MockEffectBatchInsertBuilder{}
+	mockEffectBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil).Once()
+	q.MockQEffects.On("NewEffectBatchInsertBuilder").
+		Return(mockEffectBatchInsertBuilder)
+
+	mockTransactionsParticipantsBatchInsertBuilder := &history.MockTransactionParticipantsBatchInsertBuilder{}
+	mockTransactionsParticipantsBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil)
+	q.On("NewTransactionParticipantsBatchInsertBuilder").
+		Return(mockTransactionsParticipantsBatchInsertBuilder)
+
+	mockOperationParticipantBatchInsertBuilder := &history.MockOperationParticipantBatchInsertBuilder{}
+	mockOperationParticipantBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil)
+	q.On("NewOperationParticipantBatchInsertBuilder").
+		Return(mockOperationParticipantBatchInsertBuilder)
+
+	mockTransactionClaimableBalanceBatchInsertBuilder := &history.MockTransactionClaimableBalanceBatchInsertBuilder{}
+	mockTransactionClaimableBalanceBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil)
+	q.MockQHistoryClaimableBalances.On("NewTransactionClaimableBalanceBatchInsertBuilder").
+		Return(mockTransactionClaimableBalanceBatchInsertBuilder)
+
+	mockOperationClaimableBalanceBatchInsertBuilder := &history.MockOperationClaimableBalanceBatchInsertBuilder{}
+	mockOperationClaimableBalanceBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil)
+	q.MockQHistoryClaimableBalances.On("NewOperationClaimableBalanceBatchInsertBuilder").
+		Return(mockOperationClaimableBalanceBatchInsertBuilder)
+
+	mockTransactionLiquidityPoolBatchInsertBuilder := &history.MockTransactionLiquidityPoolBatchInsertBuilder{}
+	mockTransactionLiquidityPoolBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil)
+	q.MockQHistoryLiquidityPools.On("NewTransactionLiquidityPoolBatchInsertBuilder").
+		Return(mockTransactionLiquidityPoolBatchInsertBuilder)
+
+	mockOperationLiquidityPoolBatchInsertBuilder := &history.MockOperationLiquidityPoolBatchInsertBuilder{}
+	mockOperationLiquidityPoolBatchInsertBuilder.On("Exec", ctx, mockSession).Return(nil)
+	q.MockQHistoryLiquidityPools.On("NewOperationLiquidityPoolBatchInsertBuilder").
+		Return(mockOperationLiquidityPoolBatchInsertBuilder)
+
+	q.MockQClaimableBalances.On("NewClaimableBalanceClaimantBatchInsertBuilder", maxBatchSize).
+		Return(&history.MockClaimableBalanceClaimantBatchInsertBuilder{}).Once()
+
+	q.On("NewTradeBatchInsertBuilder").Return(&history.MockTradeBatchInsertBuilder{})
+
+	return []interface{}{mockAccountSignersBatchInsertBuilder,
+		mockOperationsBatchInsertBuilder,
+		mockTransactionsBatchInsertBuilder}
+}
diff --git a/services/horizon/internal/ingest/processors/change_processors.go b/services/horizon/internal/ingest/processors/change_processors.go
index 2e5b126d8f..ee9eb127f1 100644
--- a/services/horizon/internal/ingest/processors/change_processors.go
+++ b/services/horizon/internal/ingest/processors/change_processors.go
@@ -8,63 +8,6 @@ import (
 	"github.com/stellar/go/support/errors"
 )
 
-type ChangeProcessor interface {
-	ProcessChange(ctx context.Context, change ingest.Change) error
-}
-
-type LedgerTransactionProcessor interface {
-	ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error
-}
-
-type LedgerTransactionFilterer interface {
-	FilterTransaction(ctx context.Context, transaction ingest.LedgerTransaction) (bool, error)
-}
-
-func StreamLedgerTransactions(
-	ctx context.Context,
-	txFilterer LedgerTransactionFilterer,
-	filteredTxProcessor LedgerTransactionProcessor,
-	txProcessor LedgerTransactionProcessor,
-	reader *ingest.LedgerTransactionReader,
-) error {
-	for {
-		tx, err := reader.Read()
-		if err == io.EOF {
-			return nil
-		}
-		if err != nil {
-			return errors.Wrap(err, "could not read transaction")
-		}
-		include, err := txFilterer.FilterTransaction(ctx, tx)
-		if err != nil {
-			return errors.Wrapf(
-				err,
-				"could not filter transaction %v",
-				tx.Index,
-			)
-		}
-		if !include {
-			if err = filteredTxProcessor.ProcessTransaction(ctx, tx); err != nil {
-				return errors.Wrapf(
-					err,
-					"could not process transaction %v",
-					tx.Index,
-				)
-			}
-			log.Debugf("Filters did not find match on transaction, dropping this tx with hash %v", tx.Result.TransactionHash.HexString())
-			continue
-		}
-
-		if err = txProcessor.ProcessTransaction(ctx, tx); err != nil {
-			return errors.Wrapf(
-				err,
-				"could not process transaction %v",
-				tx.Index,
-			)
-		}
-	}
-}
-
 func StreamChanges(
 	ctx context.Context,
 	changeProcessor ChangeProcessor,
diff --git a/services/horizon/internal/ingest/processors/ledgers_processor.go b/services/horizon/internal/ingest/processors/ledgers_processor.go
index 1f14cc5518..942a5f8522 100644
--- a/services/horizon/internal/ingest/processors/ledgers_processor.go
+++ b/services/horizon/internal/ingest/processors/ledgers_processor.go
@@ -32,14 +32,18 @@ func NewLedgerProcessor(batch history.LedgerBatchInsertBuilder, ingestVersion in
 	}
 }
 
-func (p *LedgersProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error {
+func (p *LedgersProcessor) ProcessLedger(lcm xdr.LedgerCloseMeta) *ledgerInfo {
 	sequence := lcm.LedgerSequence()
 	entry, ok := p.ledgers[sequence]
 	if !ok {
 		entry = &ledgerInfo{header: lcm.LedgerHeaderHistoryEntry()}
 		p.ledgers[sequence] = entry
 	}
+	return entry
+}
 
+func (p *LedgersProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error {
+	entry := p.ProcessLedger(lcm)
 	opCount := len(transaction.Envelope.Operations())
 	entry.txSetOpCount += opCount
 	if transaction.Result.Successful() {
diff --git a/services/horizon/internal/ingest/processors/main.go b/services/horizon/internal/ingest/processors/main.go
index 5088dd97aa..94f83f3fa9 100644
--- a/services/horizon/internal/ingest/processors/main.go
+++ b/services/horizon/internal/ingest/processors/main.go
@@ -1,7 +1,13 @@
 package processors
 
 import (
+	"context"
+	"io"
+
 	"github.com/guregu/null"
+	"github.com/stellar/go/ingest"
+	"github.com/stellar/go/support/db"
+	"github.com/stellar/go/support/errors"
 	logpkg "github.com/stellar/go/support/log"
 	"github.com/stellar/go/xdr"
 )
@@ -10,6 +16,65 @@ var log = logpkg.DefaultLogger.WithField("service", "ingest")
 
 const maxBatchSize = 100000
 
+type ChangeProcessor interface {
+	ProcessChange(ctx context.Context, change ingest.Change) error
+}
+
+type LedgerTransactionProcessor interface {
+	ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error
+	Flush(ctx context.Context, session db.SessionInterface) error
+}
+
+type LedgerTransactionFilterer interface {
+	FilterTransaction(ctx context.Context, transaction ingest.LedgerTransaction) (bool, error)
+}
+
+func StreamLedgerTransactions(
+	ctx context.Context,
+	txFilterer LedgerTransactionFilterer,
+	filteredTxProcessor LedgerTransactionProcessor,
+	txProcessor LedgerTransactionProcessor,
+	reader *ingest.LedgerTransactionReader,
+	ledger xdr.LedgerCloseMeta,
+) error {
+	for {
+		tx, err := reader.Read()
+		if err == io.EOF {
+			return nil
+		}
+		if err != nil {
+			return errors.Wrap(err, "could not read transaction")
+		}
+		include, err := txFilterer.FilterTransaction(ctx, tx)
+		if err != nil {
+			return errors.Wrapf(
+				err,
+				"could not filter transaction %v",
+				tx.Index,
+			)
+		}
+		if !include {
+			if err = filteredTxProcessor.ProcessTransaction(ledger, tx); err != nil {
+				return errors.Wrapf(
+					err,
+					"could not process transaction %v",
+					tx.Index,
+				)
+			}
+			log.Debugf("Filters did not find match on transaction, dropping this tx with hash %v", tx.Result.TransactionHash.HexString())
+			continue
+		}
+
+		if err = txProcessor.ProcessTransaction(ledger, tx); err != nil {
+			return errors.Wrapf(
+				err,
+				"could not process transaction %v",
+				tx.Index,
+			)
+		}
+	}
+}
+
 func ledgerEntrySponsorToNullString(entry xdr.LedgerEntry) null.String {
 	sponsoringID := entry.SponsoringID()
 
diff --git a/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor.go b/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor.go
index b9585d4802..26118b11c4 100644
--- a/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor.go
+++ b/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor.go
@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"github.com/stellar/go/ingest"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/xdr"
 )
 
@@ -51,7 +52,11 @@ type StatsLedgerTransactionProcessorResults struct {
 	OperationsLiquidityPoolWithdraw         int64
 }
 
-func (p *StatsLedgerTransactionProcessor) ProcessTransaction(ctx context.Context, transaction ingest.LedgerTransaction) error {
+func (p *StatsLedgerTransactionProcessor) Flush(ctx context.Context, session db.SessionInterface) error {
+	return nil
+}
+
+func (p *StatsLedgerTransactionProcessor) ProcessTransaction(lcm xdr.LedgerCloseMeta, transaction ingest.LedgerTransaction) error {
 	p.results.Transactions++
 	ops := int64(len(transaction.Envelope.Operations()))
 	p.results.Operations += ops
diff --git a/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor_test.go b/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor_test.go
index f2bc2a5040..c7fc6d7967 100644
--- a/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor_test.go
+++ b/services/horizon/internal/ingest/processors/stats_ledger_transaction_processor_test.go
@@ -1,7 +1,6 @@
 package processors
 
 import (
-	"context"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -23,12 +22,14 @@ func TestStatsLedgerTransactionProcessoAllOpTypesCovered(t *testing.T) {
 			},
 		},
 	}
+	lcm := xdr.LedgerCloseMeta{}
+
 	for typ, s := range xdr.OperationTypeToStringMap {
 		tx := txTemplate
 		txTemplate.Envelope.V1.Tx.Operations[0].Body.Type = xdr.OperationType(typ)
 		f := func() {
 			var p StatsLedgerTransactionProcessor
-			p.ProcessTransaction(context.Background(), tx)
+			p.ProcessTransaction(lcm, tx)
 		}
 		assert.NotPanics(t, f, s)
 	}
@@ -38,16 +39,17 @@ func TestStatsLedgerTransactionProcessoAllOpTypesCovered(t *testing.T) {
 	txTemplate.Envelope.V1.Tx.Operations[0].Body.Type = 20000
 	f := func() {
 		var p StatsLedgerTransactionProcessor
-		p.ProcessTransaction(context.Background(), tx)
+		p.ProcessTransaction(lcm, tx)
 	}
 	assert.Panics(t, f)
 }
 
 func TestStatsLedgerTransactionProcessor(t *testing.T) {
 	processor := &StatsLedgerTransactionProcessor{}
+	lcm := xdr.LedgerCloseMeta{}
 
 	// Successful
-	assert.NoError(t, processor.ProcessTransaction(context.Background(), ingest.LedgerTransaction{
+	assert.NoError(t, processor.ProcessTransaction(lcm, ingest.LedgerTransaction{
 		Result: xdr.TransactionResultPair{
 			Result: xdr.TransactionResult{
 				Result: xdr.TransactionResultResult{
@@ -88,7 +90,7 @@ func TestStatsLedgerTransactionProcessor(t *testing.T) {
 	}))
 
 	// Failed
-	assert.NoError(t, processor.ProcessTransaction(context.Background(), ingest.LedgerTransaction{
+	assert.NoError(t, processor.ProcessTransaction(lcm, ingest.LedgerTransaction{
 		Result: xdr.TransactionResultPair{
 			Result: xdr.TransactionResult{
 				Result: xdr.TransactionResultResult{
diff --git a/services/horizon/internal/ingest/processors/trades_processor.go b/services/horizon/internal/ingest/processors/trades_processor.go
index b1084c6e08..d5ee89f51e 100644
--- a/services/horizon/internal/ingest/processors/trades_processor.go
+++ b/services/horizon/internal/ingest/processors/trades_processor.go
@@ -92,17 +92,38 @@ func (p *TradeProcessor) Flush(ctx context.Context, session db.SessionInterface)
 	for _, trade := range p.trades {
 		row := trade.row
 		if trade.sellerAccount != "" {
-			row.BaseAccountID = null.IntFrom(p.accountLoader.GetNow(trade.sellerAccount))
+			val, err := p.accountLoader.GetNow(trade.sellerAccount)
+			if err != nil {
+				return err
+			}
+			row.BaseAccountID = null.IntFrom(val)
 		}
 		if trade.buyerAccount != "" {
-			row.CounterAccountID = null.IntFrom(p.accountLoader.GetNow(trade.buyerAccount))
+			val, err := p.accountLoader.GetNow(trade.buyerAccount)
+			if err != nil {
+				return err
+			}
+			row.CounterAccountID = null.IntFrom(val)
 		}
 		if trade.liquidityPoolID != "" {
-			row.BaseLiquidityPoolID = null.IntFrom(p.lpLoader.GetNow(trade.liquidityPoolID))
+			val, err := p.lpLoader.GetNow(trade.liquidityPoolID)
+			if err != nil {
+				return err
+			}
+			row.BaseLiquidityPoolID = null.IntFrom(val)
+		}
+
+		val, err := p.assetLoader.GetNow(history.AssetKeyFromXDR(trade.soldAsset))
+		if err != nil {
+			return err
 		}
+		row.BaseAssetID = val
 
-		row.BaseAssetID = p.assetLoader.GetNow(history.AssetKeyFromXDR(trade.soldAsset))
-		row.CounterAssetID = p.assetLoader.GetNow(history.AssetKeyFromXDR(trade.boughtAsset))
+		val, err = p.assetLoader.GetNow(history.AssetKeyFromXDR(trade.boughtAsset))
+		if err != nil {
+			return err
+		}
+		row.CounterAssetID = val
 
 		if row.BaseAssetID > row.CounterAssetID {
 			row.BaseIsSeller = false
diff --git a/services/horizon/internal/integration/transaction_preconditions_test.go b/services/horizon/internal/integration/transaction_preconditions_test.go
index 44a6baac82..94c0a09c35 100644
--- a/services/horizon/internal/integration/transaction_preconditions_test.go
+++ b/services/horizon/internal/integration/transaction_preconditions_test.go
@@ -5,89 +5,18 @@ import (
 	"encoding/base64"
 	"math"
 	"strconv"
-	"sync"
 	"testing"
 	"time"
 
 	sdk "github.com/stellar/go/clients/horizonclient"
 	"github.com/stellar/go/keypair"
 	"github.com/stellar/go/network"
-	"github.com/stellar/go/protocols/horizon"
 	"github.com/stellar/go/services/horizon/internal/test/integration"
 	"github.com/stellar/go/txnbuild"
 	"github.com/stellar/go/xdr"
 	"github.com/stretchr/testify/assert"
 )
 
-func TestTransactionPreconditionsMinSeq(t *testing.T) {
-	tt := assert.New(t)
-	itest := integration.NewTest(t, integration.Config{})
-	if itest.GetEffectiveProtocolVersion() < 19 {
-		t.Skip("Can't run with protocol < 19")
-	}
-	master := itest.Master()
-	masterAccount := itest.MasterAccount()
-	currentAccountSeq, err := masterAccount.GetSequenceNumber()
-	tt.NoError(err)
-
-	// Ensure that the minSequence of the transaction is enough
-	// but the sequence isn't
-	txParams := buildTXParams(master, masterAccount, currentAccountSeq+100)
-
-	// this errors because the tx.seqNum is more than +1 from sourceAccoubnt.seqNum
-	_, err = itest.SubmitTransaction(master, txParams)
-	tt.Error(err)
-
-	// Now the transaction should be submitted without problems
-	txParams.Preconditions.MinSequenceNumber = &currentAccountSeq
-	tx := itest.MustSubmitTransaction(master, txParams)
-
-	txHistory, err := itest.Client().TransactionDetail(tx.Hash)
-	assert.NoError(t, err)
-	assert.Equal(t, txHistory.Preconditions.MinAccountSequence, strconv.FormatInt(*txParams.Preconditions.MinSequenceNumber, 10))
-
-	// Test the transaction submission queue by sending transactions out of order
-	// and making sure they are all executed properly
-	masterAccount = itest.MasterAccount()
-	currentAccountSeq, err = masterAccount.GetSequenceNumber()
-	tt.NoError(err)
-
-	seqs := []struct {
-		minSeq int64
-		seq    int64
-	}{
-		{0, currentAccountSeq + 9},                 // sent first, executed second
-		{0, currentAccountSeq + 10},                // sent second, executed third
-		{currentAccountSeq, currentAccountSeq + 8}, // sent third, executed first
-	}
-
-	// Send the transactions in parallel since otherwise they are admitted sequentially
-	var results []horizon.Transaction
-	var resultsMx sync.Mutex
-	var wg sync.WaitGroup
-	wg.Add(len(seqs))
-	for _, s := range seqs {
-		sLocal := s
-		go func() {
-			params := buildTXParams(master, masterAccount, sLocal.seq)
-			if sLocal.minSeq > 0 {
-				params.Preconditions.MinSequenceNumber = &sLocal.minSeq
-			}
-			result := itest.MustSubmitTransaction(master, params)
-			resultsMx.Lock()
-			results = append(results, result)
-			resultsMx.Unlock()
-			wg.Done()
-		}()
-		// Space out requests to ensure the queue receives the transactions
-		// in the planned order
-		time.Sleep(time.Millisecond * 50)
-	}
-	wg.Wait()
-
-	tt.Len(results, len(seqs))
-}
-
 func TestTransactionPreconditionsTimeBounds(t *testing.T) {
 	tt := assert.New(t)
 	itest := integration.NewTest(t, integration.Config{})
diff --git a/services/horizon/internal/integration/txsub_test.go b/services/horizon/internal/integration/txsub_test.go
index 60b8717b18..069aa8be1b 100644
--- a/services/horizon/internal/integration/txsub_test.go
+++ b/services/horizon/internal/integration/txsub_test.go
@@ -14,11 +14,11 @@ func TestTxsub(t *testing.T) {
 	itest := integration.NewTest(t, integration.Config{})
 	master := itest.Master()
 
-	// Sanity check: create 20 accounts and submit 2 txs from each of them as
-	// a source at the same time. Then check if the results are correct.
 	t.Run("Sanity", func(t *testing.T) {
+		// simplify this to one tx per account, to align with core capabilities of one
+		// tx per account per ledger.
 		testAccounts := 20
-		subsPerAccont := 2
+		subsPerAccont := 1
 		keys, accounts := itest.CreateAccounts(testAccounts, "1000")
 
 		var wg sync.WaitGroup
diff --git a/support/db/batch_insert_builder_test.go b/support/db/batch_insert_builder_test.go
index e283e8bf57..e0d28e145d 100644
--- a/support/db/batch_insert_builder_test.go
+++ b/support/db/batch_insert_builder_test.go
@@ -13,6 +13,7 @@ import (
 type hungerRow struct {
 	Name        string `db:"name"`
 	HungerLevel string `db:"hunger_level"`
+	JsonValue   []byte `db:"json_value"`
 }
 
 type invalidHungerRow struct {
diff --git a/support/db/fast_batch_insert_builder_test.go b/support/db/fast_batch_insert_builder_test.go
index c31f502735..bfd2f8407b 100644
--- a/support/db/fast_batch_insert_builder_test.go
+++ b/support/db/fast_batch_insert_builder_test.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"testing"
 
+	"github.com/guregu/null"
 	"github.com/stretchr/testify/assert"
 
 	"github.com/stellar/go/support/db/dbtest"
@@ -21,6 +22,7 @@ func TestFastBatchInsertBuilder(t *testing.T) {
 		insertBuilder.Row(map[string]interface{}{
 			"name":         "bubba",
 			"hunger_level": "1",
+			"json_value":   []byte(`{"bump_to": "97"}`),
 		}),
 	)
 
@@ -28,13 +30,14 @@ func TestFastBatchInsertBuilder(t *testing.T) {
 		insertBuilder.Row(map[string]interface{}{
 			"name": "bubba",
 		}),
-		"invalid number of columns (expected=2, actual=1)",
+		"invalid number of columns (expected=3, actual=1)",
 	)
 
 	assert.EqualError(t,
 		insertBuilder.Row(map[string]interface{}{
-			"name": "bubba",
-			"city": "London",
+			"name":       "bubba",
+			"city":       "London",
+			"json_value": []byte(`{"bump_to": "98"}`),
 		}),
 		"column \"hunger_level\" does not exist",
 	)
@@ -43,6 +46,7 @@ func TestFastBatchInsertBuilder(t *testing.T) {
 		insertBuilder.RowStruct(hungerRow{
 			Name:        "bubba2",
 			HungerLevel: "9",
+			JsonValue:   []byte(`{"bump_to": "98"}`),
 		}),
 	)
 
@@ -74,8 +78,8 @@ func TestFastBatchInsertBuilder(t *testing.T) {
 		t,
 		found,
 		[]person{
-			{Name: "bubba", HungerLevel: "1"},
-			{Name: "bubba2", HungerLevel: "9"},
+			{Name: "bubba", HungerLevel: "1", JsonValue: null.NewString(`{"bump_to": "97"}`, true)},
+			{Name: "bubba2", HungerLevel: "9", JsonValue: null.NewString(`{"bump_to": "98"}`, true)},
 		},
 	)
 
@@ -116,8 +120,8 @@ func TestFastBatchInsertBuilder(t *testing.T) {
 		t,
 		found,
 		[]person{
-			{Name: "bubba", HungerLevel: "1"},
-			{Name: "bubba2", HungerLevel: "9"},
+			{Name: "bubba", HungerLevel: "1", JsonValue: null.NewString(`{"bump_to": "97"}`, true)},
+			{Name: "bubba2", HungerLevel: "9", JsonValue: null.NewString(`{"bump_to": "98"}`, true)},
 		},
 	)
 	assert.NoError(t, sess.Rollback())
diff --git a/support/db/internal_test.go b/support/db/internal_test.go
index 8ce0370a92..3e1a06dabc 100644
--- a/support/db/internal_test.go
+++ b/support/db/internal_test.go
@@ -7,6 +7,7 @@ const testSchema = `
 CREATE TABLE  IF NOT EXISTS people (
     name character varying NOT NULL,
     hunger_level integer NOT NULL,
+    json_value jsonb,
     PRIMARY KEY (name)
 );
 DELETE FROM people;
diff --git a/support/db/main_test.go b/support/db/main_test.go
index 68724d197d..301b533aa4 100644
--- a/support/db/main_test.go
+++ b/support/db/main_test.go
@@ -4,15 +4,16 @@ import (
 	"testing"
 	"time"
 
+	"github.com/guregu/null"
 	"github.com/stellar/go/support/db/dbtest"
 	"github.com/stretchr/testify/assert"
 )
 
 type person struct {
-	Name        string `db:"name"`
-	HungerLevel string `db:"hunger_level"`
-
-	SomethingIgnored int `db:"-"`
+	Name             string      `db:"name"`
+	HungerLevel      string      `db:"hunger_level"`
+	JsonValue        null.String `db:"json_value"`
+	SomethingIgnored int         `db:"-"`
 }
 
 func TestGetTable(t *testing.T) {

From 59cc1f0d3d69362a7523de30f088c8663887de8b Mon Sep 17 00:00:00 2001
From: Shawn Reuland <shawn@stellar.org>
Date: Mon, 30 Oct 2023 23:43:20 -0700
Subject: [PATCH 14/17] update new processor, batch loader to work with latest
 from master

---
 .../horizon/internal/action_offers_test.go    |  2 +-
 .../horizon/internal/actions/account_test.go  |  6 ++--
 .../horizon/internal/actions/offer_test.go    |  4 +--
 .../internal/actions/operation_test.go        | 30 ++++++++++++-------
 .../horizon/internal/actions_account_test.go  |  2 +-
 .../horizon/internal/actions_data_test.go     |  2 +-
 .../horizon/internal/actions_trade_test.go    |  2 +-
 .../effect_batch_insert_builder_test.go       |  2 +-
 .../internal/db2/history/effect_test.go       |  4 +--
 .../internal/db2/history/fee_bump_scenario.go |  2 +-
 .../internal/db2/history/ledger_test.go       |  4 +--
 .../operation_batch_insert_builder_test.go    |  3 +-
 ...n_participant_batch_insert_builder_test.go |  2 +-
 .../internal/db2/history/operation_test.go    |  2 +-
 .../internal/db2/history/participants_test.go |  2 +-
 .../internal/db2/history/trade_scenario.go    |  2 +-
 .../internal/db2/history/transaction_test.go  |  8 ++---
 .../ingest/processors/effects_processor.go    |  2 +-
 .../processors/effects_processor_test.go      |  2 +-
 .../ingest/processors/operations_processor.go |  8 ++---
 .../processors/operations_processor_test.go   |  3 +-
 services/horizon/internal/middleware_test.go  |  2 +-
 support/db/fast_batch_insert_builder.go       | 10 ++++++-
 support/db/fast_batch_insert_builder_test.go  |  7 +++--
 24 files changed, 65 insertions(+), 48 deletions(-)

diff --git a/services/horizon/internal/action_offers_test.go b/services/horizon/internal/action_offers_test.go
index d10e720636..464757c94e 100644
--- a/services/horizon/internal/action_offers_test.go
+++ b/services/horizon/internal/action_offers_test.go
@@ -24,7 +24,7 @@ func TestOfferActions_Show(t *testing.T) {
 	ht.Assert.NoError(err)
 
 	ledgerCloseTime := time.Now().Unix()
-	ht.Assert.NoError(q.Begin())
+	ht.Assert.NoError(q.Begin(ctx))
 	ledgerBatch := q.NewLedgerBatchInsertBuilder()
 	err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
diff --git a/services/horizon/internal/actions/account_test.go b/services/horizon/internal/actions/account_test.go
index ff265721da..b8c9d0d03b 100644
--- a/services/horizon/internal/actions/account_test.go
+++ b/services/horizon/internal/actions/account_test.go
@@ -228,7 +228,7 @@ func TestAccountInfo(t *testing.T) {
 	assert.NoError(t, err)
 
 	ledgerFourCloseTime := time.Now().Unix()
-	assert.NoError(t, q.Begin())
+	assert.NoError(t, q.Begin(tt.Ctx))
 	ledgerBatch := q.NewLedgerBatchInsertBuilder()
 	err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
@@ -412,7 +412,7 @@ func TestGetAccountsHandlerPageResultsByAsset(t *testing.T) {
 	err := q.UpsertAccounts(tt.Ctx, []history.AccountEntry{account1, account2})
 	assert.NoError(t, err)
 	ledgerCloseTime := time.Now().Unix()
-	assert.NoError(t, q.Begin())
+	assert.NoError(t, q.Begin(tt.Ctx))
 	ledgerBatch := q.NewLedgerBatchInsertBuilder()
 	err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
@@ -519,7 +519,7 @@ func TestGetAccountsHandlerPageResultsByLiquidityPool(t *testing.T) {
 	assert.NoError(t, err)
 
 	ledgerCloseTime := time.Now().Unix()
-	assert.NoError(t, q.Begin())
+	assert.NoError(t, q.Begin(tt.Ctx))
 	ledgerBatch := q.NewLedgerBatchInsertBuilder()
 	err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
diff --git a/services/horizon/internal/actions/offer_test.go b/services/horizon/internal/actions/offer_test.go
index 578e284bc6..091dfba14c 100644
--- a/services/horizon/internal/actions/offer_test.go
+++ b/services/horizon/internal/actions/offer_test.go
@@ -79,7 +79,7 @@ func TestGetOfferByIDHandler(t *testing.T) {
 	handler := GetOfferByID{}
 
 	ledgerCloseTime := time.Now().Unix()
-	assert.NoError(t, q.Begin())
+	assert.NoError(t, q.Begin(tt.Ctx))
 	ledgerBatch := q.NewLedgerBatchInsertBuilder()
 	err := ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
@@ -190,7 +190,7 @@ func TestGetOffersHandler(t *testing.T) {
 	handler := GetOffersHandler{}
 
 	ledgerCloseTime := time.Now().Unix()
-	assert.NoError(t, q.Begin())
+	assert.NoError(t, q.Begin(tt.Ctx))
 	ledgerBatch := q.NewLedgerBatchInsertBuilder()
 	err := ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
diff --git a/services/horizon/internal/actions/operation_test.go b/services/horizon/internal/actions/operation_test.go
index 099f8d1886..9aa033fd24 100644
--- a/services/horizon/internal/actions/operation_test.go
+++ b/services/horizon/internal/actions/operation_test.go
@@ -36,17 +36,21 @@ func TestInvokeHostFnDetailsInPaymentOperations(t *testing.T) {
 	opID1 := toid.New(sequence, txIndex, 1).ToInt64()
 
 	ledgerCloseTime := time.Now().Unix()
-	_, err := q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{
-		Header: xdr.LedgerHeader{
-			LedgerSeq: xdr.Uint32(sequence),
-			ScpValue: xdr.StellarValue{
-				CloseTime: xdr.TimePoint(ledgerCloseTime),
+	ledgerBatch := q.NewLedgerBatchInsertBuilder()
+	err := ledgerBatch.Add(
+		xdr.LedgerHeaderHistoryEntry{
+			Header: xdr.LedgerHeader{
+				LedgerSeq: xdr.Uint32(sequence),
+				ScpValue: xdr.StellarValue{
+					CloseTime: xdr.TimePoint(ledgerCloseTime),
+				},
 			},
-		},
-	}, 1, 0, 1, 0, 0)
+		}, 1, 0, 1, 0, 0)
 	tt.Assert.NoError(err)
+	tt.Assert.NoError(q.Begin(tt.Ctx))
+	tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q))
 
-	transactionBuilder := q.NewTransactionBatchInsertBuilder(1)
+	transactionBuilder := q.NewTransactionBatchInsertBuilder()
 	firstTransaction := buildLedgerTransaction(tt.T, testTransaction{
 		index:         uint32(txIndex),
 		envelopeXDR:   "AAAAACiSTRmpH6bHC6Ekna5e82oiGY5vKDEEUgkq9CB//t+rAAAAyAEXUhsAADDRAAAAAAAAAAAAAAABAAAAAAAAAAsBF1IbAABX4QAAAAAAAAAA",
@@ -55,11 +59,13 @@ func TestInvokeHostFnDetailsInPaymentOperations(t *testing.T) {
 		metaXDR:       "AAAAAQAAAAAAAAAA",
 		hash:          "19aaa18db88605aedec04659fb45e06f240b022eb2d429e05133e4d53cd945ba",
 	})
-	err = transactionBuilder.Add(tt.Ctx, firstTransaction, uint32(sequence))
+	err = transactionBuilder.Add(firstTransaction, uint32(sequence))
 	tt.Assert.NoError(err)
+	tt.Assert.NoError(transactionBuilder.Exec(tt.Ctx, q))
+
+	operationBuilder := q.NewOperationBatchInsertBuilder()
 
-	operationBuilder := q.NewOperationBatchInsertBuilder(1)
-	err = operationBuilder.Add(tt.Ctx,
+	err = operationBuilder.Add(
 		opID1,
 		txID,
 		1,
@@ -118,6 +124,8 @@ func TestInvokeHostFnDetailsInPaymentOperations(t *testing.T) {
 		null.String{},
 		true)
 	tt.Assert.NoError(err)
+	tt.Assert.NoError(operationBuilder.Exec(tt.Ctx, q))
+	tt.Assert.NoError(q.Commit())
 
 	records, err := handler.GetResourcePage(
 		httptest.NewRecorder(),
diff --git a/services/horizon/internal/actions_account_test.go b/services/horizon/internal/actions_account_test.go
index e3e71e0b3a..1fa2b12b14 100644
--- a/services/horizon/internal/actions_account_test.go
+++ b/services/horizon/internal/actions_account_test.go
@@ -19,7 +19,7 @@ func TestAccountActions_InvalidID(t *testing.T) {
 	err = q.UpdateIngestVersion(ht.Ctx, ingest.CurrentVersion)
 	ht.Assert.NoError(err)
 
-	ht.Assert.NoError(q.Begin())
+	ht.Assert.NoError(q.Begin(ht.Ctx))
 	ledgerBatch := q.NewLedgerBatchInsertBuilder()
 	err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
diff --git a/services/horizon/internal/actions_data_test.go b/services/horizon/internal/actions_data_test.go
index 8cdc4a07db..1142a4000a 100644
--- a/services/horizon/internal/actions_data_test.go
+++ b/services/horizon/internal/actions_data_test.go
@@ -44,7 +44,7 @@ func TestDataActions_Show(t *testing.T) {
 	ht.Assert.NoError(err)
 	err = q.UpdateIngestVersion(ht.Ctx, ingest.CurrentVersion)
 	ht.Assert.NoError(err)
-	ht.Assert.NoError(q.Begin())
+	ht.Assert.NoError(q.Begin(ht.Ctx))
 	ledgerBatch := q.NewLedgerBatchInsertBuilder()
 	err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 		Header: xdr.LedgerHeader{
diff --git a/services/horizon/internal/actions_trade_test.go b/services/horizon/internal/actions_trade_test.go
index d46e22fc9d..72991c5da2 100644
--- a/services/horizon/internal/actions_trade_test.go
+++ b/services/horizon/internal/actions_trade_test.go
@@ -820,7 +820,7 @@ func IngestTestTrade(
 		return err
 	}
 
-	if err = q.Begin(); err != nil {
+	if err = q.Begin(ctx); err != nil {
 		return err
 	}
 	batch := q.NewTradeBatchInsertBuilder()
diff --git a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go
index cef5f3745d..dc02148a7d 100644
--- a/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go
+++ b/services/horizon/internal/db2/history/effect_batch_insert_builder_test.go
@@ -15,7 +15,7 @@ func TestAddEffect(t *testing.T) {
 	defer tt.Finish()
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(tt.Ctx))
 
 	address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY"
 	muxedAddres := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26"
diff --git a/services/horizon/internal/db2/history/effect_test.go b/services/horizon/internal/db2/history/effect_test.go
index 96d68208cb..498d5e92df 100644
--- a/services/horizon/internal/db2/history/effect_test.go
+++ b/services/horizon/internal/db2/history/effect_test.go
@@ -18,7 +18,7 @@ func TestEffectsForLiquidityPool(t *testing.T) {
 	defer tt.Finish()
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(tt.Ctx))
 
 	// Insert Effect
 	address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY"
@@ -74,7 +74,7 @@ func TestEffectsForTrustlinesSponsorshipEmptyAssetType(t *testing.T) {
 	defer tt.Finish()
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(tt.Ctx))
 
 	address := "GAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSTVY"
 	muxedAddres := "MAQAA5L65LSYH7CQ3VTJ7F3HHLGCL3DSLAR2Y47263D56MNNGHSQSAAAAAAAAAAE2LP26"
diff --git a/services/horizon/internal/db2/history/fee_bump_scenario.go b/services/horizon/internal/db2/history/fee_bump_scenario.go
index b7d2c0a01e..da6563c732 100644
--- a/services/horizon/internal/db2/history/fee_bump_scenario.go
+++ b/services/horizon/internal/db2/history/fee_bump_scenario.go
@@ -248,7 +248,7 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 		hash:          "edba3051b2f2d9b713e8a08709d631eccb72c59864ff3c564c68792271bb24a7",
 	})
 	ctx := context.Background()
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(ctx))
 
 	insertBuilder := q.NewTransactionBatchInsertBuilder()
 	prefilterInsertBuilder := q.NewTransactionFilteredTmpBatchInsertBuilder()
diff --git a/services/horizon/internal/db2/history/ledger_test.go b/services/horizon/internal/db2/history/ledger_test.go
index c8526fddd6..4fe9125fbe 100644
--- a/services/horizon/internal/db2/history/ledger_test.go
+++ b/services/horizon/internal/db2/history/ledger_test.go
@@ -129,7 +129,7 @@ func TestInsertLedger(t *testing.T) {
 		int(expectedLedger.ImporterVersion),
 	)
 	tt.Assert.NoError(err)
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(tt.Ctx))
 	tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q.SessionInterface))
 	tt.Assert.NoError(q.Commit())
 
@@ -217,7 +217,7 @@ func insertLedgerWithSequence(tt *test.T, q *Q, seq uint32) {
 		int(expectedLedger.ImporterVersion),
 	)
 	tt.Assert.NoError(err)
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(tt.Ctx))
 	tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q.SessionInterface))
 	tt.Assert.NoError(q.Commit())
 }
diff --git a/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go b/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go
index e9e7e0ba4b..0c06d545f3 100644
--- a/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go
+++ b/services/horizon/internal/db2/history/operation_batch_insert_builder_test.go
@@ -16,7 +16,7 @@ func TestAddOperation(t *testing.T) {
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
 
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(tt.Ctx))
 
 	txBatch := q.NewTransactionBatchInsertBuilder()
 
@@ -64,7 +64,6 @@ func TestAddOperation(t *testing.T) {
 
 	err = builder.Exec(tt.Ctx, q)
 	tt.Assert.NoError(err)
-
 	tt.Assert.NoError(q.Commit())
 
 	ops := []Operation{}
diff --git a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
index 508bfa22cf..7e823064f2 100644
--- a/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
+++ b/services/horizon/internal/db2/history/operation_participant_batch_insert_builder_test.go
@@ -17,7 +17,7 @@ func TestAddOperationParticipants(t *testing.T) {
 
 	accountLoader := NewAccountLoader()
 	address := keypair.MustRandom().Address()
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(tt.Ctx))
 	builder := q.NewOperationParticipantBatchInsertBuilder()
 	err := builder.Add(240518172673, accountLoader.GetFuture(address))
 	tt.Assert.NoError(err)
diff --git a/services/horizon/internal/db2/history/operation_test.go b/services/horizon/internal/db2/history/operation_test.go
index eb0a5ce2b1..f7533ee5f3 100644
--- a/services/horizon/internal/db2/history/operation_test.go
+++ b/services/horizon/internal/db2/history/operation_test.go
@@ -78,7 +78,7 @@ func TestOperationByLiquidityPool(t *testing.T) {
 	opID1 := toid.New(sequence, txIndex, 1).ToInt64()
 	opID2 := toid.New(sequence, txIndex, 2).ToInt64()
 
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(tt.Ctx))
 
 	// Insert a phony transaction
 	transactionBuilder := q.NewTransactionBatchInsertBuilder()
diff --git a/services/horizon/internal/db2/history/participants_test.go b/services/horizon/internal/db2/history/participants_test.go
index 07f6d59c3e..37f7654abb 100644
--- a/services/horizon/internal/db2/history/participants_test.go
+++ b/services/horizon/internal/db2/history/participants_test.go
@@ -50,7 +50,7 @@ func TestTransactionParticipantsBatch(t *testing.T) {
 	addresses = append(addresses, address)
 	tt.Assert.NoError(batch.Add(otherTransactionID, accountLoader.GetFuture(address)))
 
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(tt.Ctx))
 	tt.Assert.NoError(accountLoader.Exec(tt.Ctx, q))
 	tt.Assert.NoError(batch.Exec(tt.Ctx, q))
 	tt.Assert.NoError(q.Commit())
diff --git a/services/horizon/internal/db2/history/trade_scenario.go b/services/horizon/internal/db2/history/trade_scenario.go
index d993b3912c..7296238a35 100644
--- a/services/horizon/internal/db2/history/trade_scenario.go
+++ b/services/horizon/internal/db2/history/trade_scenario.go
@@ -229,7 +229,7 @@ func TradeScenario(tt *test.T, q *Q) TradeFixtures {
 
 	inserts := createInsertTrades(accountIDs, assetIDs, poolIDs, 3)
 
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(tt.Ctx))
 	tt.Assert.NoError(
 		builder.Add(inserts...),
 	)
diff --git a/services/horizon/internal/db2/history/transaction_test.go b/services/horizon/internal/db2/history/transaction_test.go
index a145a13d43..30c22c7660 100644
--- a/services/horizon/internal/db2/history/transaction_test.go
+++ b/services/horizon/internal/db2/history/transaction_test.go
@@ -57,7 +57,7 @@ func TestTransactionByLiquidityPool(t *testing.T) {
 	}, 0, 0, 0, 0, 0)
 	tt.Assert.NoError(err)
 
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(tt.Ctx))
 
 	tt.Assert.NoError(ledgerBatch.Exec(tt.Ctx, q.SessionInterface))
 
@@ -209,7 +209,7 @@ func TestInsertTransactionDoesNotAllowDuplicateIndex(t *testing.T) {
 	test.ResetHorizonDB(t, tt.HorizonDB)
 	q := &Q{tt.HorizonSession()}
 
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(tt.Ctx))
 
 	sequence := uint32(123)
 	insertBuilder := q.NewTransactionBatchInsertBuilder()
@@ -235,7 +235,7 @@ func TestInsertTransactionDoesNotAllowDuplicateIndex(t *testing.T) {
 	tt.Assert.NoError(insertBuilder.Exec(tt.Ctx, q))
 	tt.Assert.NoError(q.Commit())
 
-	tt.Assert.NoError(q.Begin())
+	tt.Assert.NoError(q.Begin(tt.Ctx))
 	insertBuilder = q.NewTransactionBatchInsertBuilder()
 	tt.Assert.NoError(insertBuilder.Add(secondTransaction, sequence))
 	tt.Assert.EqualError(
@@ -828,7 +828,7 @@ func TestInsertTransaction(t *testing.T) {
 	} {
 		t.Run(testCase.name, func(t *testing.T) {
 			insertBuilder := q.NewTransactionBatchInsertBuilder()
-			tt.Assert.NoError(q.Begin())
+			tt.Assert.NoError(q.Begin(tt.Ctx))
 			tt.Assert.NoError(insertBuilder.Add(testCase.toInsert, sequence))
 			tt.Assert.NoError(insertBuilder.Exec(tt.Ctx, q))
 			tt.Assert.NoError(q.Commit())
diff --git a/services/horizon/internal/ingest/processors/effects_processor.go b/services/horizon/internal/ingest/processors/effects_processor.go
index 60ee1356fc..02df955bf0 100644
--- a/services/horizon/internal/ingest/processors/effects_processor.go
+++ b/services/horizon/internal/ingest/processors/effects_processor.go
@@ -16,9 +16,9 @@ import (
 	"github.com/stellar/go/keypair"
 	"github.com/stellar/go/protocols/horizon/base"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
-	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/strkey"
 	"github.com/stellar/go/support/contractevents"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/xdr"
 )
diff --git a/services/horizon/internal/ingest/processors/effects_processor_test.go b/services/horizon/internal/ingest/processors/effects_processor_test.go
index 2d499b6e00..0243768fde 100644
--- a/services/horizon/internal/ingest/processors/effects_processor_test.go
+++ b/services/horizon/internal/ingest/processors/effects_processor_test.go
@@ -23,8 +23,8 @@ import (
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	. "github.com/stellar/go/services/horizon/internal/test/transactions"
-	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/contractevents"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
diff --git a/services/horizon/internal/ingest/processors/operations_processor.go b/services/horizon/internal/ingest/processors/operations_processor.go
index 5dae103eb1..c8ae1a9585 100644
--- a/services/horizon/internal/ingest/processors/operations_processor.go
+++ b/services/horizon/internal/ingest/processors/operations_processor.go
@@ -11,8 +11,8 @@ import (
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/protocols/horizon/base"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
-	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/contractevents"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/toid"
 	"github.com/stellar/go/xdr"
@@ -20,13 +20,13 @@ import (
 
 // OperationProcessor operations processor
 type OperationProcessor struct {
-	batch history.OperationBatchInsertBuilder
-	network  string
+	batch   history.OperationBatchInsertBuilder
+	network string
 }
 
 func NewOperationProcessor(batch history.OperationBatchInsertBuilder, network string) *OperationProcessor {
 	return &OperationProcessor{
-		batch: batch,
+		batch:   batch,
 		network: network,
 	}
 }
diff --git a/services/horizon/internal/ingest/processors/operations_processor_test.go b/services/horizon/internal/ingest/processors/operations_processor_test.go
index 7e63f30657..83ef1636a8 100644
--- a/services/horizon/internal/ingest/processors/operations_processor_test.go
+++ b/services/horizon/internal/ingest/processors/operations_processor_test.go
@@ -16,9 +16,9 @@ import (
 	"github.com/stellar/go/ingest"
 	"github.com/stellar/go/keypair"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
-	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/strkey"
 	"github.com/stellar/go/support/contractevents"
+	"github.com/stellar/go/support/db"
 	"github.com/stellar/go/support/errors"
 	"github.com/stellar/go/xdr"
 )
@@ -494,6 +494,7 @@ func (s *OperationsProcessorTestSuiteLedger) TestExecFails() {
 			mock.Anything,
 			mock.Anything,
 			mock.Anything,
+			mock.Anything,
 		).Return(nil).Once()
 	s.Assert().NoError(s.processor.ProcessTransaction(lcm, tx))
 
diff --git a/services/horizon/internal/middleware_test.go b/services/horizon/internal/middleware_test.go
index 6dd1a46681..40a74ea69e 100644
--- a/services/horizon/internal/middleware_test.go
+++ b/services/horizon/internal/middleware_test.go
@@ -285,7 +285,7 @@ func TestStateMiddleware(t *testing.T) {
 			stateMiddleware.NoStateVerification = testCase.noStateVerification
 			tt.Assert.NoError(q.UpdateExpStateInvalid(context.Background(), testCase.stateInvalid))
 
-			tt.Assert.NoError(q.Begin())
+			tt.Assert.NoError(q.Begin(tt.Ctx))
 			ledgerBatch := q.NewLedgerBatchInsertBuilder()
 			err = ledgerBatch.Add(xdr.LedgerHeaderHistoryEntry{
 				Hash: xdr.Hash{byte(i)},
diff --git a/support/db/fast_batch_insert_builder.go b/support/db/fast_batch_insert_builder.go
index ec235ee31d..e951b2ccf1 100644
--- a/support/db/fast_batch_insert_builder.go
+++ b/support/db/fast_batch_insert_builder.go
@@ -58,6 +58,10 @@ func (b *FastBatchInsertBuilder) Row(row map[string]interface{}) error {
 		if !ok {
 			return errors.Errorf(`column "%s" does not exist`, column)
 		}
+
+		if b, ok := val.([]byte); ok {
+			val = string(b)
+		}
 		rowSlice = append(rowSlice, val)
 	}
 	b.rows = append(b.rows, rowSlice)
@@ -91,7 +95,11 @@ func (b *FastBatchInsertBuilder) RowStruct(row interface{}) error {
 	// convert fields values to interface{}
 	columnValues := make([]interface{}, len(b.columns))
 	for i, rval := range rvals {
-		columnValues[i] = rval.Interface()
+		if b, ok := rval.Interface().([]byte); ok {
+			columnValues[i] = string(b)
+		} else {
+			columnValues[i] = rval.Interface()
+		}
 	}
 
 	b.rows = append(b.rows, columnValues)
diff --git a/support/db/fast_batch_insert_builder_test.go b/support/db/fast_batch_insert_builder_test.go
index bfd2f8407b..a0c2a1fcb6 100644
--- a/support/db/fast_batch_insert_builder_test.go
+++ b/support/db/fast_batch_insert_builder_test.go
@@ -61,14 +61,15 @@ func TestFastBatchInsertBuilder(t *testing.T) {
 	assert.Equal(t, 2, insertBuilder.Len())
 	assert.Equal(t, false, insertBuilder.sealed)
 
+	ctx := context.Background()
 	assert.EqualError(t,
-		insertBuilder.Exec(context.Background(), sess, "people"),
+		insertBuilder.Exec(ctx, sess, "people"),
 		"cannot call Exec() outside of a transaction",
 	)
 	assert.Equal(t, true, insertBuilder.sealed)
 
-	assert.NoError(t, sess.Begin())
-	assert.NoError(t, insertBuilder.Exec(context.Background(), sess, "people"))
+	assert.NoError(t, sess.Begin(ctx))
+	assert.NoError(t, insertBuilder.Exec(ctx, sess, "people"))
 	assert.Equal(t, 2, insertBuilder.Len())
 	assert.Equal(t, true, insertBuilder.sealed)
 

From 2be340b866477847e6861aa9d569b0cecd75b702 Mon Sep 17 00:00:00 2001
From: Shawn Reuland <shawn@stellar.org>
Date: Tue, 31 Oct 2023 12:24:32 -0700
Subject: [PATCH 15/17] #4909: review feedback on err handling in processor

---
 .../ingest/processors/effects_processor.go    | 34 +++++++++++++------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/services/horizon/internal/ingest/processors/effects_processor.go b/services/horizon/internal/ingest/processors/effects_processor.go
index 02df955bf0..496c4bc9b5 100644
--- a/services/horizon/internal/ingest/processors/effects_processor.go
+++ b/services/horizon/internal/ingest/processors/effects_processor.go
@@ -1477,7 +1477,9 @@ func (e *effectsWrapper) addInvokeHostFunctionEffects(events []contractevents.Ev
 		}
 
 		details := make(map[string]interface{}, 4)
-		addAssetDetails(details, evt.GetAsset(), "")
+		if err := addAssetDetails(details, evt.GetAsset(), ""); err != nil {
+			return errors.Wrapf(err, "invokeHostFunction asset details had an error")
+		}
 
 		//
 		// Note: We ignore effects that involve contracts (until the day we have
@@ -1496,24 +1498,28 @@ func (e *effectsWrapper) addInvokeHostFunctionEffects(events []contractevents.Ev
 			}
 
 			if strkey.IsValidEd25519PublicKey(transferEvent.From) {
-				e.add(
+				if err := e.add(
 					transferEvent.From,
 					null.String{},
 					history.EffectAccountDebited,
 					details,
-				)
+				); err != nil {
+					return errors.Wrapf(err, "invokeHostFunction asset details from contract xfr-from had an error")
+				}
 			} else {
 				details["contract"] = transferEvent.From
 				e.addMuxed(source, history.EffectContractDebited, details)
 			}
 
 			if strkey.IsValidEd25519PublicKey(transferEvent.To) {
-				e.add(
+				if err := e.add(
 					transferEvent.To,
 					null.String{},
 					history.EffectAccountCredited,
 					toDetails,
-				)
+				); err != nil {
+					return errors.Wrapf(err, "invokeHostFunction asset details from contract xfr-to had an error")
+				}
 			} else {
 				toDetails["contract"] = transferEvent.To
 				e.addMuxed(source, history.EffectContractCredited, toDetails)
@@ -1525,12 +1531,14 @@ func (e *effectsWrapper) addInvokeHostFunctionEffects(events []contractevents.Ev
 			mintEvent := evt.(*contractevents.MintEvent)
 			details["amount"] = amount.String128(mintEvent.Amount)
 			if strkey.IsValidEd25519PublicKey(mintEvent.To) {
-				e.add(
+				if err := e.add(
 					mintEvent.To,
 					null.String{},
 					history.EffectAccountCredited,
 					details,
-				)
+				); err != nil {
+					return errors.Wrapf(err, "invokeHostFunction asset details from contract mint had an error")
+				}
 			} else {
 				details["contract"] = mintEvent.To
 				e.addMuxed(source, history.EffectContractCredited, details)
@@ -1542,12 +1550,14 @@ func (e *effectsWrapper) addInvokeHostFunctionEffects(events []contractevents.Ev
 			cbEvent := evt.(*contractevents.ClawbackEvent)
 			details["amount"] = amount.String128(cbEvent.Amount)
 			if strkey.IsValidEd25519PublicKey(cbEvent.From) {
-				e.add(
+				if err := e.add(
 					cbEvent.From,
 					null.String{},
 					history.EffectAccountDebited,
 					details,
-				)
+				); err != nil {
+					return errors.Wrapf(err, "invokeHostFunction asset details from contract clawback had an error")
+				}
 			} else {
 				details["contract"] = cbEvent.From
 				e.addMuxed(source, history.EffectContractDebited, details)
@@ -1557,12 +1567,14 @@ func (e *effectsWrapper) addInvokeHostFunctionEffects(events []contractevents.Ev
 			burnEvent := evt.(*contractevents.BurnEvent)
 			details["amount"] = amount.String128(burnEvent.Amount)
 			if strkey.IsValidEd25519PublicKey(burnEvent.From) {
-				e.add(
+				if err := e.add(
 					burnEvent.From,
 					null.String{},
 					history.EffectAccountDebited,
 					details,
-				)
+				); err != nil {
+					return errors.Wrapf(err, "invokeHostFunction asset details from contract burn had an error")
+				}
 			} else {
 				details["contract"] = burnEvent.From
 				e.addMuxed(source, history.EffectContractDebited, details)

From 769a287371478837d1d65235f911e41fd5da3a25 Mon Sep 17 00:00:00 2001
From: Shawn Reuland <shawn@stellar.org>
Date: Wed, 1 Nov 2023 10:58:54 -0700
Subject: [PATCH 16/17] #4909: convert byte[] values to string before sending
 to fast batch builder rows

---
 .../db2/history/effect_batch_insert_builder.go         |  2 +-
 .../db2/history/operation_batch_insert_builder.go      |  2 +-
 support/db/batch_insert_builder_test.go                |  2 +-
 support/db/fast_batch_insert_builder.go                | 10 +---------
 support/db/fast_batch_insert_builder_test.go           |  6 +++---
 5 files changed, 7 insertions(+), 15 deletions(-)

diff --git a/services/horizon/internal/db2/history/effect_batch_insert_builder.go b/services/horizon/internal/db2/history/effect_batch_insert_builder.go
index bd9aa0687a..e4ae333cb2 100644
--- a/services/horizon/internal/db2/history/effect_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/effect_batch_insert_builder.go
@@ -51,7 +51,7 @@ func (i *effectBatchInsertBuilder) Add(
 		"history_operation_id": operationID,
 		"order":                order,
 		"type":                 effectType,
-		"details":              details,
+		"details":              string(details),
 	})
 }
 
diff --git a/services/horizon/internal/db2/history/operation_batch_insert_builder.go b/services/horizon/internal/db2/history/operation_batch_insert_builder.go
index 5d8fae9938..e786ec97f7 100644
--- a/services/horizon/internal/db2/history/operation_batch_insert_builder.go
+++ b/services/horizon/internal/db2/history/operation_batch_insert_builder.go
@@ -54,7 +54,7 @@ func (i *operationBatchInsertBuilder) Add(
 		"transaction_id":       transactionID,
 		"application_order":    applicationOrder,
 		"type":                 operationType,
-		"details":              details,
+		"details":              string(details),
 		"source_account":       sourceAccount,
 		"source_account_muxed": sourceAccountMuxed,
 		"is_payment":           isPayment,
diff --git a/support/db/batch_insert_builder_test.go b/support/db/batch_insert_builder_test.go
index e0d28e145d..a7a772757f 100644
--- a/support/db/batch_insert_builder_test.go
+++ b/support/db/batch_insert_builder_test.go
@@ -13,7 +13,7 @@ import (
 type hungerRow struct {
 	Name        string `db:"name"`
 	HungerLevel string `db:"hunger_level"`
-	JsonValue   []byte `db:"json_value"`
+	JsonValue   string `db:"json_value"`
 }
 
 type invalidHungerRow struct {
diff --git a/support/db/fast_batch_insert_builder.go b/support/db/fast_batch_insert_builder.go
index e951b2ccf1..ec235ee31d 100644
--- a/support/db/fast_batch_insert_builder.go
+++ b/support/db/fast_batch_insert_builder.go
@@ -58,10 +58,6 @@ func (b *FastBatchInsertBuilder) Row(row map[string]interface{}) error {
 		if !ok {
 			return errors.Errorf(`column "%s" does not exist`, column)
 		}
-
-		if b, ok := val.([]byte); ok {
-			val = string(b)
-		}
 		rowSlice = append(rowSlice, val)
 	}
 	b.rows = append(b.rows, rowSlice)
@@ -95,11 +91,7 @@ func (b *FastBatchInsertBuilder) RowStruct(row interface{}) error {
 	// convert fields values to interface{}
 	columnValues := make([]interface{}, len(b.columns))
 	for i, rval := range rvals {
-		if b, ok := rval.Interface().([]byte); ok {
-			columnValues[i] = string(b)
-		} else {
-			columnValues[i] = rval.Interface()
-		}
+		columnValues[i] = rval.Interface()
 	}
 
 	b.rows = append(b.rows, columnValues)
diff --git a/support/db/fast_batch_insert_builder_test.go b/support/db/fast_batch_insert_builder_test.go
index a0c2a1fcb6..4acc09369c 100644
--- a/support/db/fast_batch_insert_builder_test.go
+++ b/support/db/fast_batch_insert_builder_test.go
@@ -22,7 +22,7 @@ func TestFastBatchInsertBuilder(t *testing.T) {
 		insertBuilder.Row(map[string]interface{}{
 			"name":         "bubba",
 			"hunger_level": "1",
-			"json_value":   []byte(`{"bump_to": "97"}`),
+			"json_value":   `{"bump_to": "97"}`,
 		}),
 	)
 
@@ -37,7 +37,7 @@ func TestFastBatchInsertBuilder(t *testing.T) {
 		insertBuilder.Row(map[string]interface{}{
 			"name":       "bubba",
 			"city":       "London",
-			"json_value": []byte(`{"bump_to": "98"}`),
+			"json_value": `{"bump_to": "98"}`,
 		}),
 		"column \"hunger_level\" does not exist",
 	)
@@ -46,7 +46,7 @@ func TestFastBatchInsertBuilder(t *testing.T) {
 		insertBuilder.RowStruct(hungerRow{
 			Name:        "bubba2",
 			HungerLevel: "9",
-			JsonValue:   []byte(`{"bump_to": "98"}`),
+			JsonValue:   `{"bump_to": "98"}`,
 		}),
 	)
 

From 1efab1fea9ade54a94411ce84a9d3a16b5252a2d Mon Sep 17 00:00:00 2001
From: Shawn Reuland <shawn@stellar.org>
Date: Wed, 1 Nov 2023 12:16:42 -0700
Subject: [PATCH 17/17] fixed dependecnies for verify-range to to get go
 version dynmaically from go.mod

---
 services/horizon/docker/verify-range/dependencies | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/services/horizon/docker/verify-range/dependencies b/services/horizon/docker/verify-range/dependencies
index c205d8d112..910ee9cf21 100644
--- a/services/horizon/docker/verify-range/dependencies
+++ b/services/horizon/docker/verify-range/dependencies
@@ -11,11 +11,6 @@ echo "deb https://apt.stellar.org $(lsb_release -cs) stable" | sudo tee -a /etc/
 apt-get update
 apt-get install -y stellar-core=${STELLAR_CORE_VERSION}
 
-GO_VERSION=$(sed -En 's/^go[[:space:]]+([[:digit:].]+)$/\1/p' go.mod)
-wget -q https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz
-tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz
-rm -f go${GO_VERSION}.linux-amd64.tar.gz
-
 git clone https://github.com/stellar/go.git stellar-go
 cd stellar-go
 
@@ -23,4 +18,10 @@ cd stellar-go
 # Below ensures we also fetch PR refs
 git config --add remote.origin.fetch "+refs/pull/*/head:refs/remotes/origin/pull/*"
 git fetch --force --quiet origin
+
+GO_VERSION=$(sed -En 's/^go[[:space:]]+([[:digit:].]+)$/\1/p' go.mod)
+wget -q https://dl.google.com/go/go${GO_VERSION}.linux-amd64.tar.gz
+tar -C /usr/local -xzf go${GO_VERSION}.linux-amd64.tar.gz
+rm -f go${GO_VERSION}.linux-amd64.tar.gz
+
 /usr/local/go/bin/go build -v ./services/horizon