From b11efd9bdc6293b8d9c2aa63c15f547a098ca281 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Sat, 25 Jun 2022 13:43:11 -0700 Subject: [PATCH] #4430: added unit test coverage, use adapter pattern for archive --- exp/lighthorizon/actions/operation.go | 3 +- exp/lighthorizon/actions/transaction.go | 1 + exp/lighthorizon/archive/ingest_archive.go | 62 +++++++++ exp/lighthorizon/archive/main.go | 32 +++-- exp/lighthorizon/archive/main_test.go | 143 +++++++++++++++++++++ exp/lighthorizon/archive/mock_archive.go | 36 ++++++ exp/lighthorizon/main.go | 19 +-- 7 files changed, 272 insertions(+), 24 deletions(-) create mode 100644 exp/lighthorizon/archive/ingest_archive.go create mode 100644 exp/lighthorizon/archive/main_test.go create mode 100644 exp/lighthorizon/archive/mock_archive.go diff --git a/exp/lighthorizon/actions/operation.go b/exp/lighthorizon/actions/operation.go index 07ba4819d0..6c64b89d3f 100644 --- a/exp/lighthorizon/actions/operation.go +++ b/exp/lighthorizon/actions/operation.go @@ -79,7 +79,8 @@ func Operations(archiveWrapper archive.Wrapper, indexStore index.Store) func(htt paginate.Cursor = toid.New(ledger, 1, 1).ToInt64() } - ops, err := archiveWrapper.GetOperations(paginate.Cursor, paginate.Limit) + //TODO - implement paginate.Order(asc/desc) + ops, err := archiveWrapper.GetOperations(r.Context(), paginate.Cursor, paginate.Limit) if err != nil { log.Error(err) sendErrorResponse(w, http.StatusInternalServerError, "") diff --git a/exp/lighthorizon/actions/transaction.go b/exp/lighthorizon/actions/transaction.go index e1b87e9c39..a0884b5743 100644 --- a/exp/lighthorizon/actions/transaction.go +++ b/exp/lighthorizon/actions/transaction.go @@ -83,6 +83,7 @@ func Transactions(archiveWrapper archive.Wrapper, indexStore index.Store) func(h } } + //TODO - implement paginate.Order(asc/desc) txns, err := archiveWrapper.GetTransactions(r.Context(), paginate.Cursor, paginate.Limit) if err != nil { log.Error(err) diff --git a/exp/lighthorizon/archive/ingest_archive.go b/exp/lighthorizon/archive/ingest_archive.go new file mode 100644 index 0000000000..0951c95e6a --- /dev/null +++ b/exp/lighthorizon/archive/ingest_archive.go @@ -0,0 +1,62 @@ +package archive + +import ( + "context" + + "github.com/stellar/go/ingest" + "github.com/stellar/go/ingest/ledgerbackend" + + "github.com/stellar/go/historyarchive" + "github.com/stellar/go/xdr" +) + +type ingestArchive struct { + *ledgerbackend.HistoryArchiveBackend +} + +func (ingestArchive) NewLedgerTransactionReaderFromLedgerCloseMeta(networkPassphrase string, ledgerCloseMeta xdr.LedgerCloseMeta) (LedgerTransactionReader, error) { + ingestReader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(networkPassphrase, ledgerCloseMeta) + + if err != nil { + return nil, err + } + + return &ingestTransactionReaderAdaption{ingestReader}, nil +} + +type ingestTransactionReaderAdaption struct { + *ingest.LedgerTransactionReader +} + +func (adaptation *ingestTransactionReaderAdaption) Read() (LedgerTransaction, error) { + tx := LedgerTransaction{} + ingestLedgerTransaction, err := adaptation.LedgerTransactionReader.Read() + if err != nil { + return tx, err + } + + tx.Index = ingestLedgerTransaction.Index + tx.Envelope = ingestLedgerTransaction.Envelope + tx.Result = ingestLedgerTransaction.Result + tx.FeeChanges = ingestLedgerTransaction.FeeChanges + tx.UnsafeMeta = ingestLedgerTransaction.UnsafeMeta + + return tx, nil +} + +// LightHorizon Archive adaptation based on existing horizon ingest package +func NewIngestArchive(sourceUrl string, networkPassphrase string) (Archive, error) { + // Simple file os access + source, err := historyarchive.ConnectBackend( + sourceUrl, + historyarchive.ConnectOptions{ + Context: context.Background(), + NetworkPassphrase: networkPassphrase, + }, + ) + if err != nil { + return nil, err + } + ledgerBackend := ledgerbackend.NewHistoryArchiveBackend(source) + return ingestArchive{ledgerBackend}, nil +} diff --git a/exp/lighthorizon/archive/main.go b/exp/lighthorizon/archive/main.go index b4d773b51f..03e40a4f10 100644 --- a/exp/lighthorizon/archive/main.go +++ b/exp/lighthorizon/archive/main.go @@ -5,7 +5,6 @@ import ( "io" "github.com/stellar/go/exp/lighthorizon/common" - "github.com/stellar/go/ingest" "github.com/stellar/go/support/errors" "github.com/stellar/go/support/log" "github.com/stellar/go/toid" @@ -22,9 +21,24 @@ import ( //lint:ignore U1000 Ignore unused temporarily const checkpointsToLookup = 1 -// Archive here only has the methods we care about, to make caching/wrapping easier +// LightHorizon data model +type LedgerTransaction struct { + Index uint32 + Envelope xdr.TransactionEnvelope + Result xdr.TransactionResultPair + FeeChanges xdr.LedgerEntryChanges + UnsafeMeta xdr.TransactionMeta +} + +type LedgerTransactionReader interface { + Read() (LedgerTransaction, error) +} + +// Archive here only has the methods LightHorizon cares about, to make caching/wrapping easier type Archive interface { GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, error) + Close() error + NewLedgerTransactionReaderFromLedgerCloseMeta(networkPassphrase string, ledgerCloseMeta xdr.LedgerCloseMeta) (LedgerTransactionReader, error) } type Wrapper struct { @@ -32,7 +46,7 @@ type Wrapper struct { Passphrase string } -func (a *Wrapper) GetOperations(cursor int64, limit int64) ([]common.Operation, error) { +func (a *Wrapper) GetOperations(ctx context.Context, cursor int64, limit int64) ([]common.Operation, error) { parsedID := toid.Parse(cursor) ledgerSequence := uint32(parsedID.LedgerSequence) if ledgerSequence < 2 { @@ -44,16 +58,15 @@ func (a *Wrapper) GetOperations(cursor int64, limit int64) ([]common.Operation, ops := []common.Operation{} appending := false - ctx := context.Background() for { log.Debugf("Checking ledger %d", ledgerSequence) ledger, err := a.GetLedger(ctx, ledgerSequence) if err != nil { - return nil, err + return ops, nil } - reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(a.Passphrase, ledger) + reader, err := a.NewLedgerTransactionReaderFromLedgerCloseMeta(a.Passphrase, ledger) if err != nil { return nil, errors.Wrapf(err, "error in ledger %d", ledgerSequence) } @@ -85,7 +98,7 @@ func (a *Wrapper) GetOperations(cursor int64, limit int64) ([]common.Operation, TransactionResult: &tx.Result.Result, // TODO: Use a method to get the header LedgerHeader: &ledger.V0.LedgerHeader.Header, - OpIndex: int32(operationOrder), + OpIndex: int32(operationOrder + 1), TxIndex: int32(transactionOrder), }) } @@ -117,10 +130,11 @@ func (a *Wrapper) GetTransactions(ctx context.Context, cursor int64, limit int64 log.Debugf("Checking ledger %d", ledgerSequence) ledger, err := a.GetLedger(ctx, ledgerSequence) if err != nil { - return nil, err + // no 'NotFound' distinction on err, treat all as not found. + return txns, nil } - reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(a.Passphrase, ledger) + reader, err := a.NewLedgerTransactionReaderFromLedgerCloseMeta(a.Passphrase, ledger) if err != nil { return nil, err } diff --git a/exp/lighthorizon/archive/main_test.go b/exp/lighthorizon/archive/main_test.go new file mode 100644 index 0000000000..a4eb4bc2f5 --- /dev/null +++ b/exp/lighthorizon/archive/main_test.go @@ -0,0 +1,143 @@ +package archive + +import ( + "context" + "fmt" + "io" + "testing" + + "github.com/stellar/go/xdr" + "github.com/stretchr/testify/require" +) + +func TestItGetsSequentialOperationsForLimitBeyondEnd(tt *testing.T) { + // l=1586111, t=1, o=1 + ctx := context.Background() + cursor := int64(6812294872829953) + passphrase := "Red New England clam chowder" + archiveWrapper := Wrapper{Archive: mockArchiveFixture(ctx, passphrase), Passphrase: passphrase} + ops, err := archiveWrapper.GetOperations(ctx, cursor, 5) + require.NoError(tt, err) + require.Len(tt, ops, 3) + require.Equal(tt, ops[0].LedgerHeader.LedgerSeq, xdr.Uint32(1586111)) + require.Equal(tt, ops[0].TxIndex, int32(1)) + require.Equal(tt, ops[0].OpIndex, int32(2)) + require.Equal(tt, ops[1].LedgerHeader.LedgerSeq, xdr.Uint32(1586111)) + require.Equal(tt, ops[1].TxIndex, int32(2)) + require.Equal(tt, ops[1].OpIndex, int32(1)) + require.Equal(tt, ops[2].LedgerHeader.LedgerSeq, xdr.Uint32(1586112)) + require.Equal(tt, ops[2].TxIndex, int32(1)) + require.Equal(tt, ops[2].OpIndex, int32(1)) +} + +func TestItGetsSequentialOperationsForLimitBeforeEnd(tt *testing.T) { + // l=1586111, t=1, o=1 + ctx := context.Background() + cursor := int64(6812294872829953) + passphrase := "White New England clam chowder" + archiveWrapper := Wrapper{Archive: mockArchiveFixture(ctx, passphrase), Passphrase: passphrase} + ops, err := archiveWrapper.GetOperations(ctx, cursor, 2) + require.NoError(tt, err) + require.Len(tt, ops, 2) + require.Equal(tt, ops[0].LedgerHeader.LedgerSeq, xdr.Uint32(1586111)) + require.Equal(tt, ops[0].TxIndex, int32(1)) + require.Equal(tt, ops[0].OpIndex, int32(2)) + require.Equal(tt, ops[1].LedgerHeader.LedgerSeq, xdr.Uint32(1586111)) + require.Equal(tt, ops[1].TxIndex, int32(2)) + require.Equal(tt, ops[1].OpIndex, int32(1)) +} + +func TestItGetsSequentialTransactionsForLimitBeyondEnd(tt *testing.T) { + // l=1586111, t=1, o=1 + ctx := context.Background() + cursor := int64(6812294872829953) + passphrase := "White New England clam chowder" + archiveWrapper := Wrapper{Archive: mockArchiveFixture(ctx, passphrase), Passphrase: passphrase} + txs, err := archiveWrapper.GetTransactions(ctx, cursor, 5) + require.NoError(tt, err) + require.Len(tt, txs, 2) + require.Equal(tt, txs[0].LedgerHeader.LedgerSeq, xdr.Uint32(1586111)) + require.Equal(tt, txs[0].TxIndex, int32(2)) + require.Equal(tt, txs[1].LedgerHeader.LedgerSeq, xdr.Uint32(1586112)) + require.Equal(tt, txs[1].TxIndex, int32(1)) +} + +func TestItGetsSequentialTransactionsForLimitBeforeEnd(tt *testing.T) { + // l=1586111, t=1, o=1 + ctx := context.Background() + cursor := int64(6812294872829953) + passphrase := "White New England clam chowder" + archiveWrapper := Wrapper{Archive: mockArchiveFixture(ctx, passphrase), Passphrase: passphrase} + txs, err := archiveWrapper.GetTransactions(ctx, cursor, 1) + require.NoError(tt, err) + require.Len(tt, txs, 1) + require.Equal(tt, txs[0].LedgerHeader.LedgerSeq, xdr.Uint32(1586111)) + require.Equal(tt, txs[0].TxIndex, int32(2)) +} + +func mockArchiveFixture(ctx context.Context, passphrase string) *MockArchive { + mockArchive := &MockArchive{} + mockReaderLedger1 := &MockLedgerTransactionReader{} + mockReaderLedger2 := &MockLedgerTransactionReader{} + + expectedLedger1 := testLedger(1586111) + expectedLedger2 := testLedger(1586112) + source := xdr.MustAddress("GCXKG6RN4ONIEPCMNFB732A436Z5PNDSRLGWK7GBLCMQLIFO4S7EYWVU") + // assert results iterate sequentially across ops-tx-ledgers + expectedLedger1Transaction1 := testLedgerTx(source, 34, 34) + expectedLedger1Transaction2 := testLedgerTx(source, 34) + expectedLedger2Transaction1 := testLedgerTx(source, 34) + + mockArchive.On("GetLedger", ctx, uint32(1586111)).Return(expectedLedger1, nil) + mockArchive.On("GetLedger", ctx, uint32(1586112)).Return(expectedLedger2, nil) + mockArchive.On("GetLedger", ctx, uint32(1586113)).Return(xdr.LedgerCloseMeta{}, fmt.Errorf("ledger not found")) + mockArchive.On("NewLedgerTransactionReaderFromLedgerCloseMeta", passphrase, expectedLedger1).Return(mockReaderLedger1, nil) + mockArchive.On("NewLedgerTransactionReaderFromLedgerCloseMeta", passphrase, expectedLedger2).Return(mockReaderLedger2, nil) + mockReaderLedger1.On("Read").Return(expectedLedger1Transaction1, nil).Once() + mockReaderLedger1.On("Read").Return(expectedLedger1Transaction2, nil).Once() + mockReaderLedger1.On("Read").Return(LedgerTransaction{}, io.EOF).Once() + mockReaderLedger2.On("Read").Return(expectedLedger2Transaction1, nil).Once() + mockReaderLedger2.On("Read").Return(LedgerTransaction{}, io.EOF).Once() + return mockArchive +} + +func testLedger(seq int) xdr.LedgerCloseMeta { + return xdr.LedgerCloseMeta{ + V0: &xdr.LedgerCloseMetaV0{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + LedgerSeq: xdr.Uint32(seq), + }, + }, + }, + } +} + +func testLedgerTx(source xdr.AccountId, bumpTos ...int) LedgerTransaction { + + ops := []xdr.Operation{} + for _, bumpTo := range bumpTos { + ops = append(ops, xdr.Operation{ + Body: xdr.OperationBody{ + BumpSequenceOp: &xdr.BumpSequenceOp{ + BumpTo: xdr.SequenceNumber(bumpTo), + }, + }, + }) + } + + tx := LedgerTransaction{ + Envelope: xdr.TransactionEnvelope{ + Type: xdr.EnvelopeTypeEnvelopeTypeTx, + V1: &xdr.TransactionV1Envelope{ + Tx: xdr.Transaction{ + SourceAccount: source.ToMuxedAccount(), + Fee: xdr.Uint32(1), + Operations: ops, + }, + }, + }, + } + + return tx +} diff --git a/exp/lighthorizon/archive/mock_archive.go b/exp/lighthorizon/archive/mock_archive.go new file mode 100644 index 0000000000..bdfd6b9149 --- /dev/null +++ b/exp/lighthorizon/archive/mock_archive.go @@ -0,0 +1,36 @@ +package archive + +import ( + "context" + + "github.com/stellar/go/xdr" + "github.com/stretchr/testify/mock" +) + +type MockLedgerTransactionReader struct { + mock.Mock +} + +func (m *MockLedgerTransactionReader) Read() (LedgerTransaction, error) { + args := m.Called() + return args.Get(0).(LedgerTransaction), args.Error(1) +} + +type MockArchive struct { + mock.Mock +} + +func (m *MockArchive) GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, error) { + args := m.Called(ctx, sequence) + return args.Get(0).(xdr.LedgerCloseMeta), args.Error(1) +} + +func (m *MockArchive) Close() error { + args := m.Called() + return args.Error(0) +} + +func (m *MockArchive) NewLedgerTransactionReaderFromLedgerCloseMeta(networkPassphrase string, ledgerCloseMeta xdr.LedgerCloseMeta) (LedgerTransactionReader, error) { + args := m.Called(networkPassphrase, ledgerCloseMeta) + return args.Get(0).(LedgerTransactionReader), args.Error(1) +} diff --git a/exp/lighthorizon/main.go b/exp/lighthorizon/main.go index f7dac375d5..f278f55c09 100644 --- a/exp/lighthorizon/main.go +++ b/exp/lighthorizon/main.go @@ -1,15 +1,13 @@ package main import ( - "context" "flag" "net/http" "github.com/stellar/go/exp/lighthorizon/actions" "github.com/stellar/go/exp/lighthorizon/archive" "github.com/stellar/go/exp/lighthorizon/index" - "github.com/stellar/go/historyarchive" - "github.com/stellar/go/ingest/ledgerbackend" + "github.com/stellar/go/network" "github.com/stellar/go/support/log" ) @@ -28,20 +26,13 @@ func main() { log.SetLevel(log.DebugLevel) log.Info("Starting lighthorizon!") - // Simple file os access - source, err := historyarchive.ConnectBackend( - *sourceUrl, - historyarchive.ConnectOptions{ - Context: context.Background(), - NetworkPassphrase: *networkPassphrase, - }, - ) + ingestArchive, err := archive.NewIngestArchive(*sourceUrl, *networkPassphrase) if err != nil { panic(err) } - ledgerBackend := ledgerbackend.NewHistoryArchiveBackend(source) - defer ledgerBackend.Close() - archiveWrapper := archive.Wrapper{Archive: ledgerBackend, Passphrase: *networkPassphrase} + defer ingestArchive.Close() + + archiveWrapper := archive.Wrapper{Archive: ingestArchive, Passphrase: *networkPassphrase} http.HandleFunc("/operations", actions.Operations(archiveWrapper, indexStore)) http.HandleFunc("/transactions", actions.Transactions(archiveWrapper, indexStore)) http.HandleFunc("/", actions.ApiDocs())