Skip to content

Commit

Permalink
stellar#4430: added unit test coverage, use adapter pattern for archive
Browse files Browse the repository at this point in the history
  • Loading branch information
sreuland committed Jun 25, 2022
1 parent cf59b4c commit 067d525
Show file tree
Hide file tree
Showing 7 changed files with 272 additions and 24 deletions.
3 changes: 2 additions & 1 deletion exp/lighthorizon/actions/operation.go
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,8 @@ func Operations(archiveWrapper archive.Wrapper, indexStore index.Store) func(htt
paginate.Cursor = toid.New(ledger, 1, 1).ToInt64()
}

ops, err := archiveWrapper.GetOperations(paginate.Cursor, paginate.Limit)
//TODO - implement paginate.Order(asc/desc)
ops, err := archiveWrapper.GetOperations(r.Context(), paginate.Cursor, paginate.Limit)
if err != nil {
log.Error(err)
sendErrorResponse(w, http.StatusInternalServerError, "")
Expand Down
1 change: 1 addition & 0 deletions exp/lighthorizon/actions/transaction.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@ func Transactions(archiveWrapper archive.Wrapper, indexStore index.Store) func(h
}
}

//TODO - implement paginate.Order(asc/desc)
txns, err := archiveWrapper.GetTransactions(r.Context(), paginate.Cursor, paginate.Limit)
if err != nil {
log.Error(err)
Expand Down
62 changes: 62 additions & 0 deletions exp/lighthorizon/archive/ingest_archive.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,62 @@
package archive

import (
"context"

"github.com/stellar/go/ingest"
"github.com/stellar/go/ingest/ledgerbackend"

"github.com/stellar/go/historyarchive"
"github.com/stellar/go/xdr"
)

type ingestArchive struct {
*ledgerbackend.HistoryArchiveBackend
}

func (ingestArchive) NewLedgerTransactionReaderFromLedgerCloseMeta(networkPassphrase string, ledgerCloseMeta xdr.LedgerCloseMeta) (LedgerTransactionReader, error) {
ingestReader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(networkPassphrase, ledgerCloseMeta)

if err != nil {
return nil, err
}

return &ingestTransactionReaderAdaption{ingestReader}, nil
}

type ingestTransactionReaderAdaption struct {
*ingest.LedgerTransactionReader
}

func (adaptation *ingestTransactionReaderAdaption) Read() (LedgerTransaction, error) {
tx := LedgerTransaction{}
ingestLedgerTransaction, err := adaptation.LedgerTransactionReader.Read()
if err != nil {
return tx, err
}

tx.Index = ingestLedgerTransaction.Index
tx.Envelope = ingestLedgerTransaction.Envelope
tx.Result = ingestLedgerTransaction.Result
tx.FeeChanges = ingestLedgerTransaction.FeeChanges
tx.UnsafeMeta = ingestLedgerTransaction.UnsafeMeta

return tx, nil
}

// LightHorizon Archive adaptation based on existing horizon ingest package
func NewIngestArchive(sourceUrl string, networkPassphrase string) (Archive, error) {
// Simple file os access
source, err := historyarchive.ConnectBackend(
sourceUrl,
historyarchive.ConnectOptions{
Context: context.Background(),
NetworkPassphrase: networkPassphrase,
},
)
if err != nil {
return nil, err
}
ledgerBackend := ledgerbackend.NewHistoryArchiveBackend(source)
return ingestArchive{ledgerBackend}, nil
}
32 changes: 23 additions & 9 deletions exp/lighthorizon/archive/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"io"

"github.com/stellar/go/exp/lighthorizon/common"
"github.com/stellar/go/ingest"
"github.com/stellar/go/support/errors"
"github.com/stellar/go/support/log"
"github.com/stellar/go/toid"
Expand All @@ -22,17 +21,32 @@ import (
//lint:ignore U1000 Ignore unused temporarily
const checkpointsToLookup = 1

// Archive here only has the methods we care about, to make caching/wrapping easier
// LightHorizon data model
type LedgerTransaction struct {
Index uint32
Envelope xdr.TransactionEnvelope
Result xdr.TransactionResultPair
FeeChanges xdr.LedgerEntryChanges
UnsafeMeta xdr.TransactionMeta
}

type LedgerTransactionReader interface {
Read() (LedgerTransaction, error)
}

// Archive here only has the methods LightHorizon cares about, to make caching/wrapping easier
type Archive interface {
GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, error)
Close() error
NewLedgerTransactionReaderFromLedgerCloseMeta(networkPassphrase string, ledgerCloseMeta xdr.LedgerCloseMeta) (LedgerTransactionReader, error)
}

type Wrapper struct {
Archive
Passphrase string
}

func (a *Wrapper) GetOperations(cursor int64, limit int64) ([]common.Operation, error) {
func (a *Wrapper) GetOperations(ctx context.Context, cursor int64, limit int64) ([]common.Operation, error) {
parsedID := toid.Parse(cursor)
ledgerSequence := uint32(parsedID.LedgerSequence)
if ledgerSequence < 2 {
Expand All @@ -44,16 +58,15 @@ func (a *Wrapper) GetOperations(cursor int64, limit int64) ([]common.Operation,

ops := []common.Operation{}
appending := false
ctx := context.Background()

for {
log.Debugf("Checking ledger %d", ledgerSequence)
ledger, err := a.GetLedger(ctx, ledgerSequence)
if err != nil {
return nil, err
return ops, nil
}

reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(a.Passphrase, ledger)
reader, err := a.NewLedgerTransactionReaderFromLedgerCloseMeta(a.Passphrase, ledger)
if err != nil {
return nil, errors.Wrapf(err, "error in ledger %d", ledgerSequence)
}
Expand Down Expand Up @@ -85,7 +98,7 @@ func (a *Wrapper) GetOperations(cursor int64, limit int64) ([]common.Operation,
TransactionResult: &tx.Result.Result,
// TODO: Use a method to get the header
LedgerHeader: &ledger.V0.LedgerHeader.Header,
OpIndex: int32(operationOrder),
OpIndex: int32(operationOrder + 1),
TxIndex: int32(transactionOrder),
})
}
Expand Down Expand Up @@ -117,10 +130,11 @@ func (a *Wrapper) GetTransactions(ctx context.Context, cursor int64, limit int64
log.Debugf("Checking ledger %d", ledgerSequence)
ledger, err := a.GetLedger(ctx, ledgerSequence)
if err != nil {
return nil, err
// no 'NotFound' distinction on err, treat all as not found.
return txns, nil
}

reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(a.Passphrase, ledger)
reader, err := a.NewLedgerTransactionReaderFromLedgerCloseMeta(a.Passphrase, ledger)
if err != nil {
return nil, err
}
Expand Down
143 changes: 143 additions & 0 deletions exp/lighthorizon/archive/main_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,143 @@
package archive

import (
"context"
"fmt"
"io"
"testing"

"github.com/stellar/go/xdr"
"github.com/stretchr/testify/require"
)

func TestItGetsSequentialOperationsForLimitBeyondEnd(tt *testing.T) {
// l=1586111, t=1, o=1
ctx := context.Background()
cursor := int64(6812294872829953)
passphrase := "Red New England clam chowder"
archiveWrapper := Wrapper{Archive: mockArchiveFixture(ctx, passphrase), Passphrase: passphrase}
ops, err := archiveWrapper.GetOperations(ctx, cursor, 5)
require.NoError(tt, err)
require.Len(tt, ops, 3)
require.Equal(tt, ops[0].LedgerHeader.LedgerSeq, xdr.Uint32(1586111))
require.Equal(tt, ops[0].TxIndex, int32(1))
require.Equal(tt, ops[0].OpIndex, int32(2))
require.Equal(tt, ops[1].LedgerHeader.LedgerSeq, xdr.Uint32(1586111))
require.Equal(tt, ops[1].TxIndex, int32(2))
require.Equal(tt, ops[1].OpIndex, int32(1))
require.Equal(tt, ops[2].LedgerHeader.LedgerSeq, xdr.Uint32(1586112))
require.Equal(tt, ops[2].TxIndex, int32(1))
require.Equal(tt, ops[2].OpIndex, int32(1))
}

func TestItGetsSequentialOperationsForLimitBeforeEnd(tt *testing.T) {
// l=1586111, t=1, o=1
ctx := context.Background()
cursor := int64(6812294872829953)
passphrase := "White New England clam chowder"
archiveWrapper := Wrapper{Archive: mockArchiveFixture(ctx, passphrase), Passphrase: passphrase}
ops, err := archiveWrapper.GetOperations(ctx, cursor, 2)
require.NoError(tt, err)
require.Len(tt, ops, 2)
require.Equal(tt, ops[0].LedgerHeader.LedgerSeq, xdr.Uint32(1586111))
require.Equal(tt, ops[0].TxIndex, int32(1))
require.Equal(tt, ops[0].OpIndex, int32(2))
require.Equal(tt, ops[1].LedgerHeader.LedgerSeq, xdr.Uint32(1586111))
require.Equal(tt, ops[1].TxIndex, int32(2))
require.Equal(tt, ops[1].OpIndex, int32(1))
}

func TestItGetsSequentialTransactionsForLimitBeyondEnd(tt *testing.T) {
// l=1586111, t=1, o=1
ctx := context.Background()
cursor := int64(6812294872829953)
passphrase := "White New England clam chowder"
archiveWrapper := Wrapper{Archive: mockArchiveFixture(ctx, passphrase), Passphrase: passphrase}
txs, err := archiveWrapper.GetTransactions(ctx, cursor, 5)
require.NoError(tt, err)
require.Len(tt, txs, 2)
require.Equal(tt, txs[0].LedgerHeader.LedgerSeq, xdr.Uint32(1586111))
require.Equal(tt, txs[0].TxIndex, int32(2))
require.Equal(tt, txs[1].LedgerHeader.LedgerSeq, xdr.Uint32(1586112))
require.Equal(tt, txs[1].TxIndex, int32(1))
}

func TestItGetsSequentialTransactionsForLimitBeforeEnd(tt *testing.T) {
// l=1586111, t=1, o=1
ctx := context.Background()
cursor := int64(6812294872829953)
passphrase := "White New England clam chowder"
archiveWrapper := Wrapper{Archive: mockArchiveFixture(ctx, passphrase), Passphrase: passphrase}
txs, err := archiveWrapper.GetTransactions(ctx, cursor, 1)
require.NoError(tt, err)
require.Len(tt, txs, 1)
require.Equal(tt, txs[0].LedgerHeader.LedgerSeq, xdr.Uint32(1586111))
require.Equal(tt, txs[0].TxIndex, int32(2))
}

func mockArchiveFixture(ctx context.Context, passphrase string) *MockArchive {
mockArchive := &MockArchive{}
mockReaderLedger1 := &MockLedgerTransactionReader{}
mockReaderLedger2 := &MockLedgerTransactionReader{}

expectedLedger1 := testLedger(1586111)
expectedLedger2 := testLedger(1586112)
source := xdr.MustAddress("GCXKG6RN4ONIEPCMNFB732A436Z5PNDSRLGWK7GBLCMQLIFO4S7EYWVU")
// assert results iterate sequentially across ops-tx-ledgers
expectedLedger1Transaction1 := testLedgerTx(source, 34, 34)
expectedLedger1Transaction2 := testLedgerTx(source, 34)
expectedLedger2Transaction1 := testLedgerTx(source, 34)

mockArchive.On("GetLedger", ctx, uint32(1586111)).Return(expectedLedger1, nil)
mockArchive.On("GetLedger", ctx, uint32(1586112)).Return(expectedLedger2, nil)
mockArchive.On("GetLedger", ctx, uint32(1586113)).Return(xdr.LedgerCloseMeta{}, fmt.Errorf("ledger not found"))
mockArchive.On("NewLedgerTransactionReaderFromLedgerCloseMeta", passphrase, expectedLedger1).Return(mockReaderLedger1, nil)
mockArchive.On("NewLedgerTransactionReaderFromLedgerCloseMeta", passphrase, expectedLedger2).Return(mockReaderLedger2, nil)
mockReaderLedger1.On("Read").Return(expectedLedger1Transaction1, nil).Once()
mockReaderLedger1.On("Read").Return(expectedLedger1Transaction2, nil).Once()
mockReaderLedger1.On("Read").Return(LedgerTransaction{}, io.EOF).Once()
mockReaderLedger2.On("Read").Return(expectedLedger2Transaction1, nil).Once()
mockReaderLedger2.On("Read").Return(LedgerTransaction{}, io.EOF).Once()
return mockArchive
}

func testLedger(seq int) xdr.LedgerCloseMeta {
return xdr.LedgerCloseMeta{
V0: &xdr.LedgerCloseMetaV0{
LedgerHeader: xdr.LedgerHeaderHistoryEntry{
Header: xdr.LedgerHeader{
LedgerSeq: xdr.Uint32(seq),
},
},
},
}
}

func testLedgerTx(source xdr.AccountId, bumpTos ...int) LedgerTransaction {

ops := []xdr.Operation{}
for _, bumpTo := range bumpTos {
ops = append(ops, xdr.Operation{
Body: xdr.OperationBody{
BumpSequenceOp: &xdr.BumpSequenceOp{
BumpTo: xdr.SequenceNumber(bumpTo),
},
},
})
}

tx := LedgerTransaction{
Envelope: xdr.TransactionEnvelope{
Type: xdr.EnvelopeTypeEnvelopeTypeTx,
V1: &xdr.TransactionV1Envelope{
Tx: xdr.Transaction{
SourceAccount: source.ToMuxedAccount(),
Fee: xdr.Uint32(1),
Operations: ops,
},
},
},
}

return tx
}
36 changes: 36 additions & 0 deletions exp/lighthorizon/archive/mock_archive.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
package archive

import (
"context"

"github.com/stellar/go/xdr"
"github.com/stretchr/testify/mock"
)

type MockLedgerTransactionReader struct {
mock.Mock
}

func (m *MockLedgerTransactionReader) Read() (LedgerTransaction, error) {
args := m.Called()
return args.Get(0).(LedgerTransaction), args.Error(1)
}

type MockArchive struct {
mock.Mock
}

func (m *MockArchive) GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, error) {
args := m.Called(ctx, sequence)
return args.Get(0).(xdr.LedgerCloseMeta), args.Error(1)
}

func (m *MockArchive) Close() error {
args := m.Called()
return args.Error(0)
}

func (m *MockArchive) NewLedgerTransactionReaderFromLedgerCloseMeta(networkPassphrase string, ledgerCloseMeta xdr.LedgerCloseMeta) (LedgerTransactionReader, error) {
args := m.Called(networkPassphrase, ledgerCloseMeta)
return args.Get(0).(LedgerTransactionReader), args.Error(1)
}
19 changes: 5 additions & 14 deletions exp/lighthorizon/main.go
Original file line number Diff line number Diff line change
@@ -1,15 +1,13 @@
package main

import (
"context"
"flag"
"net/http"

"github.com/stellar/go/exp/lighthorizon/actions"
"github.com/stellar/go/exp/lighthorizon/archive"
"github.com/stellar/go/exp/lighthorizon/index"
"github.com/stellar/go/historyarchive"
"github.com/stellar/go/ingest/ledgerbackend"

"github.com/stellar/go/network"
"github.com/stellar/go/support/log"
)
Expand All @@ -28,20 +26,13 @@ func main() {
log.SetLevel(log.DebugLevel)
log.Info("Starting lighthorizon!")

// Simple file os access
source, err := historyarchive.ConnectBackend(
*sourceUrl,
historyarchive.ConnectOptions{
Context: context.Background(),
NetworkPassphrase: *networkPassphrase,
},
)
ingestArchive, err := archive.NewIngestArchive(*sourceUrl, *networkPassphrase)
if err != nil {
panic(err)
}
ledgerBackend := ledgerbackend.NewHistoryArchiveBackend(source)
defer ledgerBackend.Close()
archiveWrapper := archive.Wrapper{Archive: ledgerBackend, Passphrase: *networkPassphrase}
defer ingestArchive.Close()

archiveWrapper := archive.Wrapper{Archive: ingestArchive, Passphrase: *networkPassphrase}
http.HandleFunc("/operations", actions.Operations(archiveWrapper, indexStore))
http.HandleFunc("/transactions", actions.Transactions(archiveWrapper, indexStore))
http.HandleFunc("/", actions.ApiDocs())
Expand Down

0 comments on commit 067d525

Please sign in to comment.