#4728: include contract asset balance changes in payments

protocols/horizon/operations/main.go

@@ -348,8 +348,10 @@ type LiquidityPoolWithdraw struct {
 // InvokeHostFunction is the json resource representing a single smart contract
 // function invocation operation, having type InvokeHostFunction.
 //
-// The model for InvokeHostFunction is intentionally simplified, Footprint
-// just contains a base64 encoded string of it's xdr serialization.
+// The model for InvokeHostFunction is intentionally simplified.
+// Parameters - array of tuples of each function input parameter value and it's data type
+// Function - name of contract function
+// Footprint - base64 encoded string of it's xdr serialization.
 type InvokeHostFunction struct {
 	Base
 	Parameters []HostFunctionParameter `json:"parameters"`

services/horizon/internal/actions/operation_test.go

@@ -7,14 +7,82 @@ import (
 	"testing"
 	"time"
 
+	"github.com/guregu/null"
 	"github.com/stellar/go/protocols/horizon/operations"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	"github.com/stellar/go/services/horizon/internal/ledger"
 	"github.com/stellar/go/services/horizon/internal/render/problem"
 	"github.com/stellar/go/services/horizon/internal/test"
 	supportProblem "github.com/stellar/go/support/render/problem"
+	"github.com/stellar/go/toid"
+	"github.com/stellar/go/xdr"
 )
 
+func TestContractEventsInPaymentOperations(t *testing.T) {
+	tt := test.Start(t)
+	defer tt.Finish()
+	test.ResetHorizonDB(t, tt.HorizonDB)
+
+	q := &history.Q{tt.HorizonSession()}
+	handler := GetOperationsHandler{OnlyPayments: true}
+
+	txIndex := int32(1)
+	sequence := int32(56)
+	txID := toid.New(sequence, txIndex, 0).ToInt64()
+	opID1 := toid.New(sequence, txIndex, 1).ToInt64()
+
+	ledgerCloseTime := time.Now().Unix()
+	_, err := q.InsertLedger(tt.Ctx, xdr.LedgerHeaderHistoryEntry{
+		Header: xdr.LedgerHeader{
+			LedgerSeq: xdr.Uint32(sequence),
+			ScpValue: xdr.StellarValue{
+				CloseTime: xdr.TimePoint(ledgerCloseTime),
+			},
+		},
+	}, 1, 0, 1, 0, 0)
+	tt.Assert.NoError(err)
+
+	transactionBuilder := q.NewTransactionBatchInsertBuilder(1)
+	firstTransaction := buildLedgerTransaction(tt.T, testTransaction{
+		index:         uint32(txIndex),
+		envelopeXDR:   "AAAAACiSTRmpH6bHC6Ekna5e82oiGY5vKDEEUgkq9CB//t+rAAAAyAEXUhsAADDRAAAAAAAAAAAAAAABAAAAAAAAAAsBF1IbAABX4QAAAAAAAAAA",
+		resultXDR:     "AAAAAAAAASwAAAAAAAAAAwAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAFAAAAAAAAAAA=",
+		feeChangesXDR: "AAAAAA==",
+		metaXDR:       "AAAAAQAAAAAAAAAA",
+		hash:          "19aaa18db88605aedec04659fb45e06f240b022eb2d429e05133e4d53cd945ba",
+	})
+	err = transactionBuilder.Add(tt.Ctx, firstTransaction, uint32(sequence))
+	tt.Assert.NoError(err)
+
+	operationBuilder := q.NewOperationBatchInsertBuilder(1)
+	err = operationBuilder.Add(tt.Ctx,
+		opID1,
+		txID,
+		1,
+		xdr.OperationTypeInvokeHostFunction,
+		[]byte(`{
+			"parameters": [],
+	        "function": "fn",
+	        "footprint": ""
+		}`),
+		"GAUJETIZVEP2NRYLUESJ3LS66NVCEGMON4UDCBCSBEVPIID773P2W6AY",
+		null.String{},
+		true)
+	tt.Assert.NoError(err)
+
+	records, err := handler.GetResourcePage(
+		httptest.NewRecorder(),
+		makeRequest(
+			t, map[string]string{}, map[string]string{}, q,
+		),
+	)
+	tt.Assert.NoError(err)
+	tt.Assert.Len(records, 1)
+
+	op := records[0].(operations.InvokeHostFunction)
+	tt.Assert.Equal(op.Function, "fn")
+}
+
 func TestGetOperationsWithoutFilter(t *testing.T) {
 	tt := test.Start(t)
 	defer tt.Finish()

services/horizon/internal/actions/test.go

@@ -0,0 +1,45 @@
+package actions
+
+import (
+	"encoding/hex"
+	"testing"
+
+	"github.com/stellar/go/ingest"
+	"github.com/stellar/go/xdr"
+	"github.com/stretchr/testify/assert"
+)
+
+type testTransaction struct {
+	index         uint32
+	envelopeXDR   string
+	resultXDR     string
+	feeChangesXDR string
+	metaXDR       string
+	hash          string
+}
+
+func buildLedgerTransaction(t *testing.T, tx testTransaction) ingest.LedgerTransaction {
+	transaction := ingest.LedgerTransaction{
+		Index:      tx.index,
+		Envelope:   xdr.TransactionEnvelope{},
+		Result:     xdr.TransactionResultPair{},
+		FeeChanges: xdr.LedgerEntryChanges{},
+		UnsafeMeta: xdr.TransactionMeta{},
+	}
+
+	tt := assert.New(t)
+
+	err := xdr.SafeUnmarshalBase64(tx.envelopeXDR, &transaction.Envelope)
+	tt.NoError(err)
+	err = xdr.SafeUnmarshalBase64(tx.resultXDR, &transaction.Result.Result)
+	tt.NoError(err)
+	err = xdr.SafeUnmarshalBase64(tx.metaXDR, &transaction.UnsafeMeta)
+	tt.NoError(err)
+	err = xdr.SafeUnmarshalBase64(tx.feeChangesXDR, &transaction.FeeChanges)
+	tt.NoError(err)
+
+	_, err = hex.Decode(transaction.Result.TransactionHash[:], []byte(tx.hash))
+	tt.NoError(err)
+
+	return transaction
+}

services/horizon/internal/db2/history/fee_bump_scenario.go

@@ -277,6 +277,7 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 		details,
 		account.Address(),
 		null.String{},
+		false,
 	))
 	tt.Assert.NoError(opBuilder.Exec(ctx))
 

services/horizon/internal/db2/history/main.go

@@ -648,6 +648,7 @@ type Operation struct {
 	SourceAccount         string            `db:"source_account"`
 	SourceAccountMuxed    null.String       `db:"source_account_muxed"`
 	TransactionSuccessful bool              `db:"transaction_successful"`
+	IsPayment             bool              `db:"is_payment"`
 }
 
 // ManageOffer is a struct of data from `operations.DetailsString`

services/horizon/internal/db2/history/mock_operations_batch_insert_builder.go

@@ -22,6 +22,7 @@ func (m *MockOperationsBatchInsertBuilder) Add(ctx context.Context,
 	details []byte,
 	sourceAccount string,
 	sourceAccountMuxed null.String,
+	isPayment bool,
 ) error {
 	a := m.Called(ctx,
 		id,
@@ -31,6 +32,7 @@ func (m *MockOperationsBatchInsertBuilder) Add(ctx context.Context,
 		details,
 		sourceAccount,
 		sourceAccountMuxed,
+		isPayment,
 	)
 	return a.Error(0)
 }

services/horizon/internal/db2/history/operation.go

@@ -235,16 +235,20 @@ func (q *OperationsQ) ForTransaction(ctx context.Context, hash string) *Operatio
 }
 
 // OnlyPayments filters the query being built to only include operations that
-// are in the "payment" class of operations:  CreateAccountOps, Payments, and
-// PathPayments.
+// are in the "payment" class of classic operations:  CreateAccountOps, Payments, and
+// PathPayments. OR also includes contract asset balance changes as expressed in 'is_payment' flag
+// on the history operations table.
 func (q *OperationsQ) OnlyPayments() *OperationsQ {
-	q.sql = q.sql.Where(sq.Eq{"hop.type": []xdr.OperationType{
-		xdr.OperationTypeCreateAccount,
-		xdr.OperationTypePayment,
-		xdr.OperationTypePathPaymentStrictReceive,
-		xdr.OperationTypePathPaymentStrictSend,
-		xdr.OperationTypeAccountMerge,
-	}})
+	q.sql = q.sql.Where(sq.Or{
+		sq.Eq{"hop.type": []xdr.OperationType{
+			xdr.OperationTypeCreateAccount,
+			xdr.OperationTypePayment,
+			xdr.OperationTypePathPaymentStrictReceive,
+			xdr.OperationTypePathPaymentStrictSend,
+			xdr.OperationTypeAccountMerge,
+		}},
+		sq.Eq{"hop.is_payment": 1}})
+
 	return q
 }
 
@@ -390,6 +394,7 @@ var selectOperation = sq.Select(
 		"hop.details, " +
 		"hop.source_account, " +
 		"hop.source_account_muxed, " +
+		"hop.is_payment, " +
 		"ht.transaction_hash, " +
 		"ht.tx_result, " +
 		"COALESCE(ht.successful, true) as transaction_successful").

services/horizon/internal/db2/history/operation_batch_insert_builder.go

@@ -20,6 +20,7 @@ type OperationBatchInsertBuilder interface {
 		details []byte,
 		sourceAccount string,
 		sourceAcccountMuxed null.String,
+		isPayment bool,
 	) error
 	Exec(ctx context.Context) error
 }
@@ -49,7 +50,12 @@ func (i *operationBatchInsertBuilder) Add(
 	details []byte,
 	sourceAccount string,
 	sourceAccountMuxed null.String,
+	isPayment bool,
 ) error {
+	dbIsPayment := 0
+	if isPayment {
+		dbIsPayment = 1
+	}
 	return i.builder.Row(ctx, map[string]interface{}{
 		"id":                   id,
 		"transaction_id":       transactionID,
@@ -58,6 +64,7 @@ func (i *operationBatchInsertBuilder) Add(
 		"details":              details,
 		"source_account":       sourceAccount,
 		"source_account_muxed": sourceAccountMuxed,
+		"is_payment":           dbIsPayment,
 	})
 
 }

services/horizon/internal/db2/history/operation_batch_insert_builder_test.go

@@ -56,6 +56,7 @@ func TestAddOperation(t *testing.T) {
 		details,
 		sourceAccount,
 		null.StringFrom(sourceAccountMuxed),
+		true,
 	)
 	tt.Assert.NoError(err)
 

services/horizon/internal/db2/history/operation_test.go

@@ -103,6 +103,7 @@ func TestOperationByLiquidityPool(t *testing.T) {
 		[]byte("{}"),
 		"GAUJETIZVEP2NRYLUESJ3LS66NVCEGMON4UDCBCSBEVPIID773P2W6AY",
 		null.String{},
+		false,
 	)
 	tt.Assert.NoError(err)
 	err = operationBuilder.Exec(tt.Ctx)
@@ -117,6 +118,7 @@ func TestOperationByLiquidityPool(t *testing.T) {
 		[]byte("{}"),
 		"GAUJETIZVEP2NRYLUESJ3LS66NVCEGMON4UDCBCSBEVPIID773P2W6AY",
 		null.String{},
+		false,
 	)
 	tt.Assert.NoError(err)
 	err = operationBuilder.Exec(tt.Ctx)
@@ -170,7 +172,7 @@ func TestOperationQueryBuilder(t *testing.T) {
 	tt.Assert.NoError(err)
 
 	// Operations for account queries will use hopp.history_operation_id in their predicates.
-	want := "SELECT hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, hop.source_account_muxed, ht.transaction_hash, ht.tx_result, COALESCE(ht.successful, true) as transaction_successful FROM history_operations hop LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id JOIN history_operation_participants hopp ON hopp.history_operation_id = hop.id WHERE hopp.history_account_id = ? AND hopp.history_operation_id > ? ORDER BY hopp.history_operation_id asc LIMIT 10"
+	want := "SELECT hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, hop.source_account_muxed, hop.is_payment, ht.transaction_hash, ht.tx_result, COALESCE(ht.successful, true) as transaction_successful FROM history_operations hop LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id JOIN history_operation_participants hopp ON hopp.history_operation_id = hop.id WHERE hopp.history_account_id = ? AND hopp.history_operation_id > ? ORDER BY hopp.history_operation_id asc LIMIT 10"
 	tt.Assert.EqualValues(want, got)
 
 	opsQ = q.Operations().ForLedger(tt.Ctx, 2).Page(db2.PageQuery{Cursor: "8589938689", Order: "asc", Limit: 10})
@@ -179,7 +181,7 @@ func TestOperationQueryBuilder(t *testing.T) {
 	tt.Assert.NoError(err)
 
 	// Other operation queries will use hop.id in their predicates.
-	want = "SELECT hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, hop.source_account_muxed, ht.transaction_hash, ht.tx_result, COALESCE(ht.successful, true) as transaction_successful FROM history_operations hop LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id WHERE hop.id >= ? AND hop.id < ? AND hop.id > ? ORDER BY hop.id asc LIMIT 10"
+	want = "SELECT hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, hop.source_account_muxed, hop.is_payment, ht.transaction_hash, ht.tx_result, COALESCE(ht.successful, true) as transaction_successful FROM history_operations hop LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id WHERE hop.id >= ? AND hop.id < ? AND hop.id > ? ORDER BY hop.id asc LIMIT 10"
 	tt.Assert.EqualValues(want, got)
 }
 
@@ -241,7 +243,7 @@ func TestOperationIncludeFailed(t *testing.T) {
 
 	sql, _, err := query.sql.ToSql()
 	tt.Assert.NoError(err)
-	tt.Assert.Equal("SELECT hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, hop.source_account_muxed, ht.transaction_hash, ht.tx_result, COALESCE(ht.successful, true) as transaction_successful FROM history_operations hop LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id JOIN history_operation_participants hopp ON hopp.history_operation_id = hop.id WHERE hopp.history_account_id = ?", sql)
+	tt.Assert.Equal("SELECT hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, hop.source_account_muxed, hop.is_payment, ht.transaction_hash, ht.tx_result, COALESCE(ht.successful, true) as transaction_successful FROM history_operations hop LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id JOIN history_operation_participants hopp ON hopp.history_operation_id = hop.id WHERE hopp.history_account_id = ?", sql)
 }
 
 // TestPaymentsSuccessfulOnly tests if default query returns payments in
@@ -271,7 +273,7 @@ func TestPaymentsSuccessfulOnly(t *testing.T) {
 	sql, _, err := query.sql.ToSql()
 	tt.Assert.NoError(err)
 	// Note: brackets around `(ht.successful = true OR ht.successful IS NULL)` are critical!
-	tt.Assert.Contains(sql, "WHERE hop.type IN (?,?,?,?,?) AND hopp.history_account_id = ? AND (ht.successful = true OR ht.successful IS NULL)")
+	tt.Assert.Contains(sql, "WHERE (hop.type IN (?,?,?,?,?) OR hop.is_payment = ?) AND hopp.history_account_id = ? AND (ht.successful = true OR ht.successful IS NULL)")
 }
 
 // TestPaymentsIncludeFailed tests `IncludeFailed` method.
@@ -304,7 +306,7 @@ func TestPaymentsIncludeFailed(t *testing.T) {
 
 	sql, _, err := query.sql.ToSql()
 	tt.Assert.NoError(err)
-	tt.Assert.Equal("SELECT hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, hop.source_account_muxed, ht.transaction_hash, ht.tx_result, COALESCE(ht.successful, true) as transaction_successful FROM history_operations hop LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id JOIN history_operation_participants hopp ON hopp.history_operation_id = hop.id WHERE hop.type IN (?,?,?,?,?) AND hopp.history_account_id = ?", sql)
+	tt.Assert.Equal("SELECT hop.id, hop.transaction_id, hop.application_order, hop.type, hop.details, hop.source_account, hop.source_account_muxed, hop.is_payment, ht.transaction_hash, ht.tx_result, COALESCE(ht.successful, true) as transaction_successful FROM history_operations hop LEFT JOIN history_transactions ht ON ht.id = hop.transaction_id JOIN history_operation_participants hopp ON hopp.history_operation_id = hop.id WHERE (hop.type IN (?,?,?,?,?) OR hop.is_payment = ?) AND hopp.history_account_id = ?", sql)
 }
 
 func TestExtraChecksOperationsTransactionSuccessfulTrueResultFalse(t *testing.T) {

services/horizon/internal/db2/schema/migrations/64_add_payment_flag_history_ops.sql

@@ -0,0 +1,9 @@
+-- +migrate Up
+
+ALTER TABLE history_operations ADD is_payment smallint DEFAULT 0;
+CREATE INDEX "index_history_operations_on_is_payment" ON history_operations USING btree (is_payment);
+
+-- +migrate Down
+
+DROP INDEX "index_history_operations_on_is_payment";
+ALTER TABLE history_operations DROP COLUMN is_payment;

services/horizon/internal/ingest/processor_runner.go

@@ -145,7 +145,7 @@ func (s *ProcessorRunner) buildTransactionProcessor(
 		statsLedgerTransactionProcessor,
 		processors.NewEffectProcessor(s.historyQ, sequence),
 		processors.NewLedgerProcessor(s.historyQ, ledger, CurrentVersion),
-		processors.NewOperationProcessor(s.historyQ, sequence),
+		processors.NewOperationProcessor(s.historyQ, sequence, s.config.NetworkPassphrase),
 		tradeProcessor,
 		processors.NewParticipantsProcessor(s.historyQ, sequence),
 		processors.NewTransactionProcessor(s.historyQ, sequence),