#4418: fixing test failures

services/horizon/internal/actions/submit_transaction.go

@@ -18,7 +18,7 @@ import (
 )
 
 type NetworkSubmitter interface {
-	Submit(ctx context.Context, rawTx string, envelope xdr.TransactionEnvelope, hash string, innerHash string) <-chan txsub.Result
+	Submit(ctx context.Context, rawTx string, envelope xdr.TransactionEnvelope, hash string) <-chan txsub.Result
 }
 
 type SubmitTransactionHandler struct {
@@ -155,7 +155,7 @@ func (handler SubmitTransactionHandler) GetResource(w HeaderWriter, r *http.Requ
 		return nil, hProblem.StaleHistory
 	}
 
-	submission := handler.Submitter.Submit(r.Context(), info.raw, info.parsed, info.hash, info.innerHash)
+	submission := handler.Submitter.Submit(r.Context(), info.raw, info.parsed, info.hash)
 
 	select {
 	case result := <-submission:

services/horizon/internal/actions/submit_transaction_test.go

@@ -44,7 +44,7 @@ type networkSubmitterMock struct {
 	mock.Mock
 }
 
-func (m *networkSubmitterMock) Submit(ctx context.Context, rawTx string, envelope xdr.TransactionEnvelope, hash string, innerHash string) <-chan txsub.Result {
+func (m *networkSubmitterMock) Submit(ctx context.Context, rawTx string, envelope xdr.TransactionEnvelope, hash string) <-chan txsub.Result {
 	a := m.Called()
 	return a.Get(0).(chan txsub.Result)
 }

services/horizon/internal/db2/history/fee_bump_scenario.go

@@ -254,12 +254,6 @@ func FeeBumpScenario(tt *test.T, q *Q, successful bool) FeeBumpFixture {
 	tt.Assert.NoError(insertBuilder.Add(ctx, normalTransaction, sequence))
 	tt.Assert.NoError(insertBuilder.Exec(ctx))
 
-	tt.Assert.NoError(q.InitEmptyTxSubmissionResult(ctx, hex.EncodeToString(normalTransaction.Result.TransactionHash[:]), ""))
-	tt.Assert.NoError(q.InitEmptyTxSubmissionResult(ctx, fixture.OuterHash, fixture.InnerHash))
-	txs := []ingest.LedgerTransaction{normalTransaction, feeBumpTransaction}
-	affectedRows, err := q.SetTxSubmissionResults(ctx, txs, uint32(fixture.Ledger.Sequence), fixture.Ledger.ClosedAt)
-	tt.Assert.NoError(err)
-	tt.Assert.Equal(int64(2), affectedRows)
 	account := fixture.Envelope.SourceAccount().ToAccountId()
 	feeBumpAccount := fixture.Envelope.FeeBumpAccount().ToAccountId()
 

services/horizon/internal/db2/history/operation_test.go

@@ -408,13 +408,13 @@ func TestOperationIncludeTransactions(t *testing.T) {
 func TestValidateTransactionForOperation(t *testing.T) {
 	tt := test.Start(t)
 	tt.Scenario("failed_transactions")
-	selectTransactionCopy := selectTransaction
+	selectTransactionCopy := selectTransactionHistory
 	defer func() {
-		selectTransaction = selectTransactionCopy
+		selectTransactionHistory = selectTransactionCopy
 		tt.Finish()
 	}()
 
-	selectTransaction = sq.Select(
+	selectTransactionHistory = sq.Select(
 		"ht.transaction_hash, " +
 			"ht.tx_result, " +
 			"COALESCE(ht.successful, true) as successful").
@@ -435,7 +435,7 @@ func TestValidateTransactionForOperation(t *testing.T) {
 	tt.Assert.Error(err)
 	tt.Assert.EqualError(err, "transaction id 0 does not match transaction id in operation 17179877376")
 
-	selectTransaction = sq.Select(
+	selectTransactionHistory = sq.Select(
 		"ht.id, " +
 			"ht.transaction_hash, " +
 			"COALESCE(ht.successful, true) as successful").
@@ -452,7 +452,7 @@ func TestValidateTransactionForOperation(t *testing.T) {
 	tt.Assert.Error(err)
 	tt.Assert.EqualError(err, "transaction result  does not match transaction result in operation AAAAAAAAAGQAAAAAAAAAAQAAAAAAAAABAAAAAAAAAAA=")
 
-	selectTransaction = sq.Select(
+	selectTransactionHistory = sq.Select(
 		"ht.id, " +
 			"ht.tx_result, " +
 			"COALESCE(ht.successful, true) as successful").
@@ -469,7 +469,7 @@ func TestValidateTransactionForOperation(t *testing.T) {
 	tt.Assert.Error(err)
 	tt.Assert.EqualError(err, "transaction hash  does not match transaction hash in operation 1c454630267aa8767ec8c8e30450cea6ba660145e9c924abb75d7a6669b6c28a")
 
-	selectTransaction = sq.Select(
+	selectTransactionHistory = sq.Select(
 		"ht.id, " +
 			"ht.tx_result, " +
 			"ht.transaction_hash").

services/horizon/internal/db2/history/transaction.go

@@ -16,35 +16,27 @@ import (
 // `history_transactions` table based upon the provided hash.
 func (q *Q) TransactionByHash(ctx context.Context, dest interface{}, hash string) error {
 	innerOrOuter := sq.Or{sq.Eq{"ht.transaction_hash": hash}, sq.Eq{"ht.inner_transaction_hash": hash}}
-	byHashOrInnerHashFilteredIn := selectTransactionFilteredIn.Where(innerOrOuter)
-	byHashOrInnerHashFilteredOut := selectTransactionFilteredOut.Where(innerOrOuter)
+	byHashOrInnerHashHistory := selectTransactionHistory.Where(innerOrOuter)
 
-	filteredInString, args, err := byHashOrInnerHashFilteredIn.ToSql()
-	if err != nil {
-		return errors.Wrap(err, "could not get string for inner hash sql query")
-	}
-	union := byHashOrInnerHashFilteredOut.Suffix("UNION ALL "+filteredInString, args...)
+	return q.Get(ctx, dest, byHashOrInnerHashHistory)
+}
+
+func (q *Q) PreFilteredTransactionByHash(ctx context.Context, dest interface{}, hash string) error {
+	innerOrOuter := sq.Or{sq.Eq{"ht.transaction_hash": hash}, sq.Eq{"ht.inner_transaction_hash": hash}}
+	byHashOrInnerHashPreFilter := selectTransactionPreFilteredTmp.Where(innerOrOuter)
 
-	return q.Get(ctx, dest, union)
+	return q.Get(ctx, dest, byHashOrInnerHashPreFilter)
 }
 
 // TransactionsByHashesSinceLedger fetches transactions from the `history_transactions`
 // table which match the given hash since the given ledger sequence (for perf reasons).
-func (q *Q) TransactionsByHashesSinceLedger(ctx context.Context, hashes []string, sinceLedgerSeq uint32) ([]Transaction, error) {
+func (q *Q) PreFilteredTransactionsByHashesSinceLedger(ctx context.Context, hashes []string, sinceLedgerSeq uint32) ([]Transaction, error) {
 	var dest []Transaction
 	innerOrOuterAndSeqGtEq :=
 		sq.And{sq.GtOrEq{"ht.ledger_sequence": sinceLedgerSeq}, sq.Or{sq.Eq{"ht.transaction_hash": hashes}, sq.Eq{"ht.inner_transaction_hash": hashes}}}
-	filteredIn := selectTransactionFilteredIn.Where(innerOrOuterAndSeqGtEq)
-	filteredOut := selectTransactionFilteredOut.Where(innerOrOuterAndSeqGtEq)
+	preFilterTxs := selectTransactionPreFilteredTmp.Where(innerOrOuterAndSeqGtEq)
 
-	filteredInString, args, err := filteredIn.ToSql()
-	if err != nil {
-		return nil, errors.Wrap(err, "could not get string for inner hash sql query")
-	}
-	union := filteredOut.Suffix("UNION ALL "+filteredInString, args...)
-
-	err = q.Select(ctx, &dest, union)
-	if err != nil {
+	if err := q.Select(ctx, &dest, preFilterTxs); err != nil {
 		return nil, err
 	}
 
@@ -58,7 +50,7 @@ func (q *Q) TransactionsByIDs(ctx context.Context, ids ...int64) (map[int64]Tran
 		return nil, errors.New("no id arguments provided")
 	}
 
-	sql := selectTransactionFilteredIn.Where(map[string]interface{}{
+	sql := selectTransactionHistory.Where(map[string]interface{}{
 		"ht.id": ids,
 	})
 
@@ -92,7 +84,7 @@ func (q *Q) DeleteTransactionsFilteredTmpOlderThan(ctx context.Context, howOldIn
 func (q *Q) Transactions() *TransactionsQ {
 	return &TransactionsQ{
 		parent:        q,
-		sql:           selectTransactionFilteredIn,
+		sql:           selectTransactionHistory,
 		includeFailed: false,
 	}
 }
@@ -277,5 +269,5 @@ func selectTransaction(table string) sq.SelectBuilder {
 		LeftJoin("history_ledgers hl ON ht.ledger_sequence = hl.sequence")
 }
 
-var selectTransactionFilteredIn = selectTransaction("history_transactions")
-var selectTransactionFilteredOut = selectTransaction("history_transactions_filtered_tmp")
+var selectTransactionHistory = selectTransaction("history_transactions")
+var selectTransactionPreFilteredTmp = selectTransaction("history_transactions_filtered_tmp")

services/horizon/internal/db2/schema/migrations/57_txsub_read_only.sql

@@ -4,9 +4,12 @@ DROP TABLE txsub_results;
 
 -- used to temporarily store filtered-out transactions
 -- needed by the transaction system
-CREATE TABLE history_transactions_filtered_tmp (
-    created_at timestamp NOT NULL DEFAULT NOW()
-) INHERITS (history_transactions);
+-- TODO - is this acceptable to do CREATE..SELECT..AS, 
+-- any future migrations that would change history_transactions
+-- require dropping history_transactions_filtered_tmp, and rerunning this CREATE..SELECT..AS
+CREATE TABLE history_transactions_filtered_tmp AS 
+  select * FROM history_transactions
+  WHERE ledger_sequence IS NULL;
 
 -- +migrate Down
 

services/horizon/internal/ingest/main_test.go

@@ -244,7 +244,6 @@ type mockDBQ struct {
 	history.MockQSigners
 	history.MockQTransactions
 	history.MockQTrustLines
-	history.MockQTxSubmissionResult
 }
 
 func (m *mockDBQ) Begin() error {
@@ -382,6 +381,11 @@ func (m *mockDBQ) CreateAssets(ctx context.Context, assets []xdr.Asset, batchSiz
 	return args.Get(0).(map[string]history.Asset), args.Error(1)
 }
 
+func (m *mockDBQ) DeleteTransactionsFilteredTmpOlderThan(ctx context.Context, howOldInSeconds uint64) (int64, error) {
+	args := m.Called(ctx, howOldInSeconds)
+	return args.Get(0).(int64), args.Error(1)
+}
+
 type mockLedgerBackend struct {
 	mock.Mock
 }

services/horizon/internal/ingest/processor_runner_test.go

@@ -291,16 +291,20 @@ func TestProcessorRunnerWithFilterEnabled(t *testing.T) {
 
 	mockTransactionsBatchInsertBuilder := &history.MockTransactionsBatchInsertBuilder{}
 	defer mock.AssertExpectationsForObjects(t, mockTransactionsBatchInsertBuilder)
-	mockTransactionsBatchInsertBuilder.On("Exec", ctx).Return(nil).Once()
+	mockTransactionsBatchInsertBuilder.On("Exec", ctx).Return(nil).Twice()
+
 	q.MockQTransactions.On("NewTransactionBatchInsertBuilder", maxBatchSize).
-		Return(mockTransactionsBatchInsertBuilder).Twice()
+		Return(mockTransactionsBatchInsertBuilder)
+
+	q.MockQTransactions.On("NewTransactionFilteredTmpBatchInsertBuilder", maxBatchSize).
+		Return(mockTransactionsBatchInsertBuilder)
+
+	q.On("DeleteTransactionsFilteredTmpOlderThan", ctx, mock.AnythingOfType("uint64")).
+		Return(int64(0), nil)
 
 	q.MockQLedgers.On("InsertLedger", ctx, ledger.V0.LedgerHeader, 0, 0, 0, 0, CurrentVersion).
 		Return(int64(1), nil).Once()
 
-	q.MockQTxSubmissionResult.On("SetTxSubmissionResults", ctx, []ingest.LedgerTransaction(nil), uint32(0)).
-		Return(int64(0), nil).Once()
-
 	runner := ProcessorRunner{
 		ctx:      ctx,
 		config:   config,
@@ -354,9 +358,6 @@ func TestProcessorRunnerRunAllProcessorsOnLedger(t *testing.T) {
 	q.MockQLedgers.On("InsertLedger", ctx, ledger.V0.LedgerHeader, 0, 0, 0, 0, CurrentVersion).
 		Return(int64(1), nil).Once()
 
-	q.MockQTxSubmissionResult.On("SetTxSubmissionResults", ctx, []ingest.LedgerTransaction(nil), uint32(0)).
-		Return(int64(0), nil).Once()
-
 	runner := ProcessorRunner{
 		ctx:      ctx,
 		config:   config,

services/horizon/internal/test/scenarios/base-horizon.sql

@@ -1570,12 +1570,13 @@ CREATE TABLE txsub_results (
     submitted_at           timestamp NOT NULL DEFAULT NOW()
 );
 
-INSERT INTO txsub_results
-VALUES (
-    '2374e99349b9ef7dba9a5db3339b78fda8f34777b1af33ba468ad5c0df946d4d',
-    NULL,
-    '{ "TxResult": "AAAAAAAAAGQAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAA=" }',
-    '2019-06-03 18:28:47.032496+02');
+-- migration 57
+
+CREATE TABLE history_transactions_filtered_tmp AS 
+  select * FROM history_transactions
+  WHERE ledger_sequence IS NULL;
+
+INSERT INTO history_transactions_filtered_tmp VALUES ('2374e99349b9ef7dba9a5db3339b78fda8f34777b1af33ba468ad5c0df946d4d', 2, 1, 'GBRPYHIL2CI3FNQ4BXLFMNDLFJUNPU2HY3ZMFSHONUCEOASW7QC7OX2H', 1, 100, 1, '2019-10-31 13:19:49.409714', '2019-10-31 13:19:49.409714', 8589938688, 'AAAAAGL8HQvQkbK2HA3WVjRrKmjX00fG8sLI7m0ERwJW/AX3AAAAZAAAAAAAAAABAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAArqN6LeOagjxMaUP96Bzfs9e0corNZXzBWJkFoK7kvkwAAAAAO5rKAAAAAAAAAAABVvwF9wAAAECDzqvkQBQoNAJifPRXDoLhvtycT3lFPCQ51gkdsFHaBNWw05S/VhW0Xgkr0CBPE4NaFV2Kmcs3ZwLmib4TRrML', 'AAAAAAAAAGQAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAA=', 'AAAAAQAAAAAAAAABAAAAAwAAAAMAAAACAAAAAAAAAABi/B0L0JGythwN1lY0aypo19NHxvLCyO5tBEcCVvwF9w3gtrOnY/7UAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAEAAAACAAAAAAAAAABi/B0L0JGythwN1lY0aypo19NHxvLCyO5tBEcCVvwF9w3gtrNryTTUAAAAAAAAAAMAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAACuo3ot45qCPExpQ/3oHN+z17Ryis1lfMFYmQWgruS+TAAAAAA7msoAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAA==', 'AAAAAgAAAAMAAAABAAAAAAAAAABi/B0L0JGythwN1lY0aypo19NHxvLCyO5tBEcCVvwF9w3gtrOnZAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAAAAAAEAAAACAAAAAAAAAABi/B0L0JGythwN1lY0aypo19NHxvLCyO5tBEcCVvwF9w3gtrOnY/+cAAAAAAAAAAEAAAAAAAAAAAAAAAAAAAAAAQAAAAAAAAAAAAAAAAAAAA==', '{g86r5EAUKDQCYnz0Vw6C4b7cnE95RTwkOdYJHbBR2gTVsNOUv1YVtF4JK9AgTxODWhVdipnLN2cC5om+E0azCw==}', 'none', NULL, NULL, NULL, NULL, NULL, NULL, NULL, true, 100);
 
 --
 -- PostgreSQL database dump complete