From 0a7e05a056ce604952c6e23e6e1588c50accecfb Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Fri, 31 May 2024 14:31:15 -0700 Subject: [PATCH 01/63] Ingest events into DB --- cmd/soroban-rpc/internal/db/db.go | 12 ++ cmd/soroban-rpc/internal/db/event.go | 81 ++++++++ cmd/soroban-rpc/internal/db/event_test.go | 180 ++++++++++++++++++ .../internal/db/migrations/03_events.sql | 16 ++ cmd/soroban-rpc/internal/ingest/service.go | 4 + 5 files changed, 293 insertions(+) create mode 100644 cmd/soroban-rpc/internal/db/event.go create mode 100644 cmd/soroban-rpc/internal/db/event_test.go create mode 100644 cmd/soroban-rpc/internal/db/migrations/03_events.sql diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go index 227e2115..e922ea35 100644 --- a/cmd/soroban-rpc/internal/db/db.go +++ b/cmd/soroban-rpc/internal/db/db.go @@ -37,6 +37,7 @@ type ReadWriter interface { type WriteTx interface { TransactionWriter() TransactionWriter + EventWriter() EventWriter LedgerEntryWriter() LedgerEntryWriter LedgerWriter() LedgerWriter @@ -226,6 +227,12 @@ func (rw *readWriter) NewTx(ctx context.Context) (WriteTx, error) { stmtCache: stmtCache, passphrase: rw.passphrase, }, + eventWriter: eventHandler{ + log: rw.log, + db: txSession, + stmtCache: stmtCache, + passphrase: rw.passphrase, + }, } writer.txWriter.RegisterMetrics( rw.metrics.TxIngestDuration, @@ -242,6 +249,7 @@ type writeTx struct { ledgerEntryWriter ledgerEntryWriter ledgerWriter ledgerWriter txWriter transactionHandler + eventWriter eventHandler ledgerRetentionWindow uint32 } @@ -257,6 +265,10 @@ func (w writeTx) TransactionWriter() TransactionWriter { return &w.txWriter } +func (w writeTx) EventWriter() EventWriter { + return &w.eventWriter +} + func (w writeTx) Commit(ledgerSeq uint32) error { if err := w.ledgerEntryWriter.flush(); err != nil { return err diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go new file mode 100644 index 00000000..7dea9987 --- /dev/null +++ b/cmd/soroban-rpc/internal/db/event.go @@ -0,0 +1,81 @@ +package db + +import ( + sq "github.com/Masterminds/squirrel" + "github.com/prometheus/client_golang/prometheus" + "github.com/stellar/go/ingest" + "github.com/stellar/go/support/db" + "github.com/stellar/go/support/log" + "github.com/stellar/go/xdr" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" + "io" +) + +const eventTableName = "events" + +type EventWriter interface { + InsertEvents(lcm xdr.LedgerCloseMeta) error +} + +type EventReader interface { + GetEvents(lcm xdr.LedgerCloseMeta) error +} + +type eventHandler struct { + log *log.Entry + db db.SessionInterface + stmtCache *sq.StmtCache + passphrase string + ingestMetric, countMetric prometheus.Observer +} + +func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) (err error) { + + var txReader *ingest.LedgerTransactionReader + txReader, err = ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) + if err != nil { + return + } + defer func() { + closeErr := txReader.Close() + if err == nil { + err = closeErr + } + }() + + for { + var tx ingest.LedgerTransaction + tx, err = txReader.Read() + if err == io.EOF { + err = nil + break + } + if err != nil { + return + } + + if !tx.Result.Successful() { + continue + } + + txEvents, err := tx.GetDiagnosticEvents() + if err != nil { + return err + } + + query := sq.Insert(eventTableName). + Columns("id", "ledger_sequence", "application_order", "contract_id", "event_type") + + for index, e := range txEvents { + id := events.Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() + query = query.Values(id, lcm.LedgerSequence(), tx.Index, e.Event.ContractId[:], int(e.Event.Type)) + } + + _, err = query.RunWith(eventHandler.stmtCache).Exec() + if err != nil { + return err + } + } + + return nil +} diff --git a/cmd/soroban-rpc/internal/db/event_test.go b/cmd/soroban-rpc/internal/db/event_test.go new file mode 100644 index 00000000..6cbfcca5 --- /dev/null +++ b/cmd/soroban-rpc/internal/db/event_test.go @@ -0,0 +1,180 @@ +package db + +import ( + "context" + "fmt" + sq "github.com/Masterminds/squirrel" + "github.com/sirupsen/logrus" + "github.com/stellar/go/keypair" + "github.com/stellar/go/network" + "github.com/stellar/go/support/log" + "github.com/stellar/go/xdr" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "testing" + "time" +) + +func transactionMetaWithEvents(events ...xdr.ContractEvent) xdr.TransactionMeta { + return xdr.TransactionMeta{ + V: 3, + Operations: &[]xdr.OperationMeta{}, + V3: &xdr.TransactionMetaV3{ + SorobanMeta: &xdr.SorobanTransactionMeta{ + Events: events, + }, + }, + } +} + +func contractEvent(contractID xdr.Hash, topic []xdr.ScVal, body xdr.ScVal) xdr.ContractEvent { + return xdr.ContractEvent{ + ContractId: &contractID, + Type: xdr.ContractEventTypeContract, + Body: xdr.ContractEventBody{ + V: 0, + V0: &xdr.ContractEventV0{ + Topics: topic, + Data: body, + }, + }, + } +} + +func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ...xdr.TransactionMeta) xdr.LedgerCloseMeta { + var txProcessing []xdr.TransactionResultMeta + var phases []xdr.TransactionPhase + + for _, item := range txMeta { + var operations []xdr.Operation + for range item.MustV3().SorobanMeta.Events { + operations = append(operations, + xdr.Operation{ + Body: xdr.OperationBody{ + Type: xdr.OperationTypeInvokeHostFunction, + InvokeHostFunctionOp: &xdr.InvokeHostFunctionOp{ + HostFunction: xdr.HostFunction{ + Type: xdr.HostFunctionTypeHostFunctionTypeInvokeContract, + InvokeContract: &xdr.InvokeContractArgs{ + ContractAddress: xdr.ScAddress{ + Type: xdr.ScAddressTypeScAddressTypeContract, + ContractId: &xdr.Hash{0x1, 0x2}, + }, + FunctionName: "foo", + Args: nil, + }, + }, + Auth: []xdr.SorobanAuthorizationEntry{}, + }, + }, + }) + } + envelope := xdr.TransactionEnvelope{ + Type: xdr.EnvelopeTypeEnvelopeTypeTx, + V1: &xdr.TransactionV1Envelope{ + Tx: xdr.Transaction{ + SourceAccount: xdr.MustMuxedAddress(keypair.MustRandom().Address()), + Operations: operations, + }, + }, + } + txHash, err := network.HashTransactionInEnvelope(envelope, network.FutureNetworkPassphrase) + if err != nil { + panic(err) + } + + txProcessing = append(txProcessing, xdr.TransactionResultMeta{ + TxApplyProcessing: item, + Result: xdr.TransactionResultPair{ + TransactionHash: txHash, + }, + }) + components := []xdr.TxSetComponent{ + { + Type: xdr.TxSetComponentTypeTxsetCompTxsMaybeDiscountedFee, + TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ + Txs: []xdr.TransactionEnvelope{ + envelope, + }, + }, + }, + } + phases = append(phases, xdr.TransactionPhase{ + V: 0, + V0Components: &components, + }) + } + + return xdr.LedgerCloseMeta{ + V: 1, + V1: &xdr.LedgerCloseMetaV1{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Hash: xdr.Hash{}, + Header: xdr.LedgerHeader{ + ScpValue: xdr.StellarValue{ + CloseTime: xdr.TimePoint(closeTimestamp), + }, + LedgerSeq: xdr.Uint32(sequence), + }, + }, + TxSet: xdr.GeneralizedTransactionSet{ + V: 1, + V1TxSet: &xdr.TransactionSetV1{ + PreviousLedgerHash: xdr.Hash{}, + Phases: phases, + }, + }, + TxProcessing: txProcessing, + }, + } +} + +func TestInsertEvents(t *testing.T) { + db := NewTestDB(t) + ctx := context.TODO() + log := log.DefaultLogger + log.SetLevel(logrus.TraceLevel) + now := time.Now().UTC() + + writer := NewReadWriter(log, db, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + require.NoError(t, err) + contractID := xdr.Hash([32]byte{}) + counter := xdr.ScSymbol("COUNTER") + + var txMeta []xdr.TransactionMeta + for i := 0; i < 10; i++ { + txMeta = append(txMeta, transactionMetaWithEvents( + contractEvent( + contractID, + xdr.ScVec{xdr.ScVal{ + Type: xdr.ScValTypeScvSymbol, + Sym: &counter, + }}, + xdr.ScVal{ + Type: xdr.ScValTypeScvSymbol, + Sym: &counter, + }, + ), + )) + } + ledgerCloseMeta := ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...) + + eventW := write.EventWriter() + err = eventW.InsertEvents(ledgerCloseMeta) + assert.NoError(t, err) + + var rows []struct { + ID string `db:"id"` + LedgerSequence uint32 `db:"ledger_sequence"` + ApplicationOrder uint32 `db:"application_order"` + ContractID []byte `db:"contract_id"` + EventType int `db:"event_type"` + } + + query1 := sq.Select("*").From(fmt.Sprintf("%s", eventTableName)) + err = db.Select(ctx, &rows, query1) + assert.NoError(t, err) + assert.Equal(t, 10, len(rows)) +} diff --git a/cmd/soroban-rpc/internal/db/migrations/03_events.sql b/cmd/soroban-rpc/internal/db/migrations/03_events.sql new file mode 100644 index 00000000..247a0e20 --- /dev/null +++ b/cmd/soroban-rpc/internal/db/migrations/03_events.sql @@ -0,0 +1,16 @@ +-- +migrate Up + +-- indexing table to find events in ledgers by contract_id +CREATE TABLE events ( + id TEXT PRIMARY KEY, + ledger_sequence INTEGER NOT NULL, + application_order INTEGER NOT NULL, + contract_id BLOB NOT NULL, + event_type INTEGER NOT NULL +); + +CREATE INDEX idx_ledger_sequence ON events(ledger_sequence); +CREATE INDEX idx_contract_id ON events(contract_id); + +-- +migrate Down +drop table events cascade; diff --git a/cmd/soroban-rpc/internal/ingest/service.go b/cmd/soroban-rpc/internal/ingest/service.go index 54ccbef4..16285100 100644 --- a/cmd/soroban-rpc/internal/ingest/service.go +++ b/cmd/soroban-rpc/internal/ingest/service.go @@ -337,6 +337,10 @@ func (s *Service) ingestLedgerCloseMeta(tx db.WriteTx, ledgerCloseMeta xdr.Ledge With(prometheus.Labels{"type": "transactions"}). Observe(time.Since(startTime).Seconds()) + if err := tx.EventWriter().InsertEvents(ledgerCloseMeta); err != nil { + return err + } + if err := s.eventStore.IngestEvents(ledgerCloseMeta); err != nil { return err } From 9fa8c2a8bfaa77f3cf69fbff2065796b474ed93a Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 3 Jun 2024 11:31:34 -0700 Subject: [PATCH 02/63] Update tests --- cmd/soroban-rpc/internal/db/event.go | 18 ++++++++++++++---- cmd/soroban-rpc/internal/db/event_test.go | 4 ++-- .../internal/ingest/mock_db_test.go | 5 +++++ 3 files changed, 21 insertions(+), 6 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 7dea9987..fb6cc78f 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -5,6 +5,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/stellar/go/ingest" "github.com/stellar/go/support/db" + "github.com/stellar/go/support/errors" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" @@ -29,12 +30,21 @@ type eventHandler struct { ingestMetric, countMetric prometheus.Observer } -func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) (err error) { +func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { + txCount := lcm.CountTransactions() + + if eventHandler.stmtCache == nil { + return errors.New("EventWriter incorrectly initialized without stmtCache") + } else if txCount == 0 { + return nil + } var txReader *ingest.LedgerTransactionReader - txReader, err = ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) + txReader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) if err != nil { - return + return errors.Wrapf(err, + "failed to open transaction reader for ledger %d", + lcm.LedgerSequence()) } defer func() { closeErr := txReader.Close() @@ -51,7 +61,7 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) (err err break } if err != nil { - return + return err } if !tx.Result.Successful() { diff --git a/cmd/soroban-rpc/internal/db/event_test.go b/cmd/soroban-rpc/internal/db/event_test.go index 6cbfcca5..ebe024b5 100644 --- a/cmd/soroban-rpc/internal/db/event_test.go +++ b/cmd/soroban-rpc/internal/db/event_test.go @@ -173,8 +173,8 @@ func TestInsertEvents(t *testing.T) { EventType int `db:"event_type"` } - query1 := sq.Select("*").From(fmt.Sprintf("%s", eventTableName)) - err = db.Select(ctx, &rows, query1) + query := sq.Select("*").From(fmt.Sprintf("%s", eventTableName)) + err = db.Select(ctx, &rows, query) assert.NoError(t, err) assert.Equal(t, 10, len(rows)) } diff --git a/cmd/soroban-rpc/internal/ingest/mock_db_test.go b/cmd/soroban-rpc/internal/ingest/mock_db_test.go index 6e57658d..320ec80a 100644 --- a/cmd/soroban-rpc/internal/ingest/mock_db_test.go +++ b/cmd/soroban-rpc/internal/ingest/mock_db_test.go @@ -36,6 +36,11 @@ type MockTx struct { mock.Mock } +func (m MockTx) EventWriter() db.EventWriter { + args := m.Called() + return args.Get(0).(db.EventWriter) +} + func (m MockTx) LedgerEntryWriter() db.LedgerEntryWriter { args := m.Called() return args.Get(0).(db.LedgerEntryWriter) From 4cc58b9acc9b6df04e4a079badd3d82be23eee58 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 3 Jun 2024 14:18:39 -0700 Subject: [PATCH 03/63] Update tests --- cmd/soroban-rpc/internal/db/event_test.go | 15 +-------------- cmd/soroban-rpc/internal/ingest/mock_db_test.go | 9 +++++++++ cmd/soroban-rpc/internal/ingest/service_test.go | 3 +++ 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event_test.go b/cmd/soroban-rpc/internal/db/event_test.go index ebe024b5..eb272ddb 100644 --- a/cmd/soroban-rpc/internal/db/event_test.go +++ b/cmd/soroban-rpc/internal/db/event_test.go @@ -2,8 +2,6 @@ package db import ( "context" - "fmt" - sq "github.com/Masterminds/squirrel" "github.com/sirupsen/logrus" "github.com/stellar/go/keypair" "github.com/stellar/go/network" @@ -165,16 +163,5 @@ func TestInsertEvents(t *testing.T) { err = eventW.InsertEvents(ledgerCloseMeta) assert.NoError(t, err) - var rows []struct { - ID string `db:"id"` - LedgerSequence uint32 `db:"ledger_sequence"` - ApplicationOrder uint32 `db:"application_order"` - ContractID []byte `db:"contract_id"` - EventType int `db:"event_type"` - } - - query := sq.Select("*").From(fmt.Sprintf("%s", eventTableName)) - err = db.Select(ctx, &rows, query) - assert.NoError(t, err) - assert.Equal(t, 10, len(rows)) + //TODO: Call getEvents and validate events data. } diff --git a/cmd/soroban-rpc/internal/ingest/mock_db_test.go b/cmd/soroban-rpc/internal/ingest/mock_db_test.go index 320ec80a..7c1ecd12 100644 --- a/cmd/soroban-rpc/internal/ingest/mock_db_test.go +++ b/cmd/soroban-rpc/internal/ingest/mock_db_test.go @@ -101,3 +101,12 @@ func (m MockTransactionWriter) InsertTransactions(ledger xdr.LedgerCloseMeta) er func (m MockTransactionWriter) RegisterMetrics(ingest, count prometheus.Observer) { m.Called(ingest, count) } + +type MockEventWriter struct { + mock.Mock +} + +func (m MockEventWriter) InsertEvents(ledger xdr.LedgerCloseMeta) error { + args := m.Called(ledger) + return args.Error(0) +} diff --git a/cmd/soroban-rpc/internal/ingest/service_test.go b/cmd/soroban-rpc/internal/ingest/service_test.go index 78f42494..10090c16 100644 --- a/cmd/soroban-rpc/internal/ingest/service_test.go +++ b/cmd/soroban-rpc/internal/ingest/service_test.go @@ -81,6 +81,7 @@ func TestIngestion(t *testing.T) { mockLedgerEntryWriter := &MockLedgerEntryWriter{} mockLedgerWriter := &MockLedgerWriter{} mockTxWriter := &MockTransactionWriter{} + mockEventWriter := &MockEventWriter{} ctx := context.Background() mockDB.On("NewTx", ctx).Return(mockTx, nil).Once() mockTx.On("Commit", sequence).Return(nil).Once() @@ -88,6 +89,7 @@ func TestIngestion(t *testing.T) { mockTx.On("LedgerEntryWriter").Return(mockLedgerEntryWriter).Twice() mockTx.On("LedgerWriter").Return(mockLedgerWriter).Once() mockTx.On("TransactionWriter").Return(mockTxWriter).Once() + mockTx.On("EventWriter").Return(mockEventWriter).Once() src := xdr.MustAddress("GBXGQJWVLWOYHFLVTKWV5FGHA3LNYY2JQKM7OAJAUEQFU6LPCSEFVXON") firstTx := xdr.TransactionEnvelope{ @@ -254,6 +256,7 @@ func TestIngestion(t *testing.T) { Return(nil).Once() mockLedgerWriter.On("InsertLedger", ledger).Return(nil).Once() mockTxWriter.On("InsertTransactions", ledger).Return(nil).Once() + mockEventWriter.On("InsertEvents", ledger).Return(nil).Once() assert.NoError(t, service.ingest(ctx, sequence)) mockDB.AssertExpectations(t) From 3940d277fbc0cf24c4f9d741ad5fe3bcab7ddf3b Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 11 Jun 2024 15:26:58 -0700 Subject: [PATCH 04/63] Fix tests --- cmd/soroban-rpc/internal/db/event.go | 6 +++++- cmd/soroban-rpc/internal/db/migrations/03_events.sql | 12 ++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index fb6cc78f..f49d1b6f 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -77,8 +77,12 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { Columns("id", "ledger_sequence", "application_order", "contract_id", "event_type") for index, e := range txEvents { + var contractId []byte + if e.Event.ContractId != nil { + contractId = e.Event.ContractId[:] + } id := events.Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() - query = query.Values(id, lcm.LedgerSequence(), tx.Index, e.Event.ContractId[:], int(e.Event.Type)) + query = query.Values(id, lcm.LedgerSequence(), tx.Index, contractId, int(e.Event.Type)) } _, err = query.RunWith(eventHandler.stmtCache).Exec() diff --git a/cmd/soroban-rpc/internal/db/migrations/03_events.sql b/cmd/soroban-rpc/internal/db/migrations/03_events.sql index 247a0e20..d8040575 100644 --- a/cmd/soroban-rpc/internal/db/migrations/03_events.sql +++ b/cmd/soroban-rpc/internal/db/migrations/03_events.sql @@ -1,12 +1,12 @@ -- +migrate Up -- indexing table to find events in ledgers by contract_id -CREATE TABLE events ( - id TEXT PRIMARY KEY, - ledger_sequence INTEGER NOT NULL, - application_order INTEGER NOT NULL, - contract_id BLOB NOT NULL, - event_type INTEGER NOT NULL +CREATE TABLE events( + id TEXT PRIMARY KEY, + ledger_sequence INTEGER NOT NULL, + application_order INTEGER NOT NULL, + contract_id BLOB, + event_type INTEGER NOT NULL ); CREATE INDEX idx_ledger_sequence ON events(ledger_sequence); From 4d0c280ba3d5a15cdc8b5c22acea3de03e02b3f3 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 11 Jun 2024 15:55:37 -0700 Subject: [PATCH 05/63] Ignore ingestion when events are empty --- cmd/soroban-rpc/internal/db/event.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index f49d1b6f..df249003 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -73,6 +73,10 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { return err } + if len(txEvents) == 0 { + continue + } + query := sq.Insert(eventTableName). Columns("id", "ledger_sequence", "application_order", "contract_id", "event_type") From 4f68a3564a16dd79f187b628a8523484805ca8da Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Thu, 13 Jun 2024 08:16:37 -0700 Subject: [PATCH 06/63] Add getEvents backed by DB --- cmd/soroban-rpc/internal/daemon/daemon.go | 1 + cmd/soroban-rpc/internal/db/event.go | 152 +++++++++++++++++- cmd/soroban-rpc/internal/db/event_test.go | 8 +- .../internal/db/transaction_test.go | 49 +++++- cmd/soroban-rpc/internal/jsonrpc.go | 5 +- .../internal/methods/get_events.go | 26 +-- .../internal/methods/get_events_test.go | 23 +-- 7 files changed, 234 insertions(+), 30 deletions(-) diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index c38ccde9..88c8a51a 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -283,6 +283,7 @@ func MustNew(cfg *config.Config) *Daemon { LedgerReader: db.NewLedgerReader(dbConn), LedgerEntryReader: db.NewLedgerEntryReader(dbConn), TransactionReader: db.NewTransactionReader(logger, dbConn, cfg.NetworkPassphrase), + EventReader: db.NewEventReader(logger, dbConn, cfg.NetworkPassphrase), PreflightGetter: preflightWorkerPool, }) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index df249003..425ba055 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -1,25 +1,33 @@ package db import ( + "context" + "encoding/json" + "fmt" sq "github.com/Masterminds/squirrel" "github.com/prometheus/client_golang/prometheus" "github.com/stellar/go/ingest" "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/support/log" + "github.com/stellar/go/toid" "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "io" + "strconv" + "strings" ) const eventTableName = "events" +// EventWriter is used during ingestion of events from LCM to DB type EventWriter interface { InsertEvents(lcm xdr.LedgerCloseMeta) error } +// EventReader has all the public methods to fetch events from DB type EventReader interface { - GetEvents(lcm xdr.LedgerCloseMeta) error + GetEvents(ctx context.Context, startLedgerSequence int, eventType int, contractIds []string, f ScanFunction) ([]xdr.DiagnosticEvent, error) } type eventHandler struct { @@ -30,6 +38,10 @@ type eventHandler struct { ingestMetric, countMetric prometheus.Observer } +func NewEventReader(log *log.Entry, db db.SessionInterface, passphrase string) EventReader { + return &eventHandler{log: log, db: db, passphrase: passphrase} +} + func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { txCount := lcm.CountTransactions() @@ -97,3 +109,141 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { return nil } + +// Cursor represents the position of a Soroban event. +// Soroban events are sorted in ascending order by +// ledger sequence, transaction index, operation index, +// and event index. +type Cursor struct { + // Ledger is the sequence of the ledger which emitted the event. + Ledger uint32 + // Tx is the index of the transaction within the ledger which emitted the event. + Tx uint32 + // Op is the index of the operation within the transaction which emitted the event. + // Note: Currently, there is no use for it (events are transaction-wide and not operation-specific) + // but we keep it in order to make the API future-proof. + Op uint32 + // Event is the index of the event within in the operation which emitted the event. + Event uint32 +} + +// String returns a string representation of this cursor +func (c Cursor) String() string { + return fmt.Sprintf( + "%019d-%010d", + toid.New(int32(c.Ledger), int32(c.Tx), int32(c.Op)).ToInt64(), + c.Event, + ) +} + +// MarshalJSON marshals the cursor into JSON +func (c Cursor) MarshalJSON() ([]byte, error) { + return json.Marshal(c.String()) +} + +// UnmarshalJSON unmarshalls a cursor from the given JSON +func (c *Cursor) UnmarshalJSON(b []byte) error { + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + + if parsed, err := ParseCursor(s); err != nil { + return err + } else { + *c = parsed + } + return nil +} + +// ParseCursor parses the given string and returns the corresponding cursor +func ParseCursor(input string) (Cursor, error) { + parts := strings.SplitN(input, "-", 2) + if len(parts) != 2 { + return Cursor{}, fmt.Errorf("invalid event id %s", input) + } + + // Parse the first part (toid) + idInt, err := strconv.ParseInt(parts[0], 10, 64) //lint:ignore gomnd + if err != nil { + return Cursor{}, fmt.Errorf("invalid event id %s: %w", input, err) + } + parsed := toid.Parse(idInt) + + // Parse the second part (event order) + eventOrder, err := strconv.ParseUint(parts[1], 10, 32) //lint:ignore gomnd + if err != nil { + return Cursor{}, fmt.Errorf("invalid event id %s: %w", input, err) + } + + return Cursor{ + Ledger: uint32(parsed.LedgerSequence), + Tx: uint32(parsed.TransactionOrder), + Op: uint32(parsed.OperationOrder), + Event: uint32(eventOrder), + }, nil +} + +type ScanFunction func(xdr.DiagnosticEvent, Cursor, int64, *xdr.Hash) bool + +// trimEvents removes all Events which fall outside the ledger retention window. +func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWindow uint32) error { + if latestLedgerSeq+1 <= retentionWindow { + return nil + } + + cutoff := latestLedgerSeq + 1 - retentionWindow + _, err := sq.StatementBuilder. + RunWith(eventHandler.stmtCache). + Delete(eventTableName). + Where(sq.Lt{"ledger_sequence": cutoff}). + Exec() + return err +} + +func (eventHandler *eventHandler) GetEvents(ctx context.Context, startLedgerSequence int, eventType int, contractIds []string, f ScanFunction) ([]xdr.DiagnosticEvent, error) { + + var rows []struct { + TxIndex int `db:"application_order"` + Lcm xdr.LedgerCloseMeta `db:"meta"` + } + + rowQ := sq. + Select("e.application_order", "lcm.meta"). + From(fmt.Sprintf("%s e", eventTableName)). + Join(fmt.Sprintf("%s lcm ON (e.ledger_sequence = lcm.sequence)", ledgerCloseMetaTableName)). + Where(sq.GtOrEq{"e.ledger_sequence": startLedgerSequence}) + + if len(contractIds) > 0 { + rowQ = rowQ.Where(sq.Eq{"e.contract_id": contractIds}) + } + + if eventType != -1 { + rowQ = rowQ.Where(sq.Eq{"e.event_type": eventType}) + } + + if err := eventHandler.db.Select(ctx, &rows, rowQ); err != nil { + return []xdr.DiagnosticEvent{}, + errors.Wrapf(err, "db read failed for startLedgerSequence= %d contractIds= %v eventType= %d ", startLedgerSequence, contractIds, eventType) + } else if len(rows) < 1 { + return []xdr.DiagnosticEvent{}, errors.New("No LCM found with requested event filters") + } + + txIndex, lcm := rows[0].TxIndex, rows[0].Lcm + reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) + reader.Seek(txIndex - 1) + + if err != nil { + return []xdr.DiagnosticEvent{}, + errors.Wrapf(err, "failed to index to tx %d in ledger %d", + txIndex, lcm.LedgerSequence()) + } + + ledgerTx, err := reader.Read() + events, diagErr := ledgerTx.GetDiagnosticEvents() + if diagErr != nil { + return []xdr.DiagnosticEvent{}, errors.Wrapf(err, "db read failed for startLedgerSequence %d", startLedgerSequence) + } + + return events, nil +} diff --git a/cmd/soroban-rpc/internal/db/event_test.go b/cmd/soroban-rpc/internal/db/event_test.go index eb272ddb..3acc4bb3 100644 --- a/cmd/soroban-rpc/internal/db/event_test.go +++ b/cmd/soroban-rpc/internal/db/event_test.go @@ -44,6 +44,7 @@ func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ... var txProcessing []xdr.TransactionResultMeta var phases []xdr.TransactionPhase + var components []xdr.TxSetComponent for _, item := range txMeta { var operations []xdr.Operation for range item.MustV3().SorobanMeta.Events { @@ -88,7 +89,7 @@ func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ... TransactionHash: txHash, }, }) - components := []xdr.TxSetComponent{ + components = []xdr.TxSetComponent{ { Type: xdr.TxSetComponentTypeTxsetCompTxsMaybeDiscountedFee, TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ @@ -120,7 +121,10 @@ func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ... V: 1, V1TxSet: &xdr.TransactionSetV1{ PreviousLedgerHash: xdr.Hash{}, - Phases: phases, + Phases: []xdr.TransactionPhase{{ + V: 0, + V0Components: &components, + }}, }, }, TxProcessing: txProcessing, diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 068f793d..39d1c8a4 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -26,6 +26,44 @@ func TestTransactionNotFound(t *testing.T) { require.Error(t, err, ErrNoTransaction) } +func txMetaWithEvents(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { + + meta := txMeta(acctSeq, successful) + + contractIDBytes, _ := hex.DecodeString("df06d62447fd25da07c0135eed7557e5a5497ee7d15b7fe345bd47e191d8f577") + var contractID xdr.Hash + copy(contractID[:], contractIDBytes) + counter := xdr.ScSymbol("COUNTER") + + meta.V1.TxProcessing[0].TxApplyProcessing.V3 = &xdr.TransactionMetaV3{ + SorobanMeta: &xdr.SorobanTransactionMeta{ + Events: []xdr.ContractEvent{{ + ContractId: &contractID, + Type: xdr.ContractEventTypeContract, + Body: xdr.ContractEventBody{ + V: 0, + V0: &xdr.ContractEventV0{ + Topics: []xdr.ScVal{{ + Type: xdr.ScValTypeScvSymbol, + Sym: &counter, + }}, + Data: xdr.ScVal{ + Type: xdr.ScValTypeScvSymbol, + Sym: &counter, + }, + }, + }, + }}, + ReturnValue: xdr.ScVal{ + Type: xdr.ScValTypeScvSymbol, + Sym: &counter, + }, + }, + } + + return meta +} + func TestTransactionFound(t *testing.T) { db := NewTestDB(t) ctx := context.TODO() @@ -37,16 +75,17 @@ func TestTransactionFound(t *testing.T) { require.NoError(t, err) lcms := []xdr.LedgerCloseMeta{ - txMeta(1234, true), - txMeta(1235, true), - txMeta(1236, true), - txMeta(1237, true), + txMetaWithEvents(1234, true), + txMetaWithEvents(1235, true), + txMetaWithEvents(1236, true), + txMetaWithEvents(1237, true), } - + eventW := write.EventWriter() ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() for _, lcm := range lcms { require.NoError(t, ledgerW.InsertLedger(lcm), "ingestion failed for ledger %+v", lcm.V1) require.NoError(t, txW.InsertTransactions(lcm), "ingestion failed for ledger %+v", lcm.V1) + require.NoError(t, eventW.InsertEvents(lcm), "ingestion failed for ledger %+v", lcm.V1) } require.NoError(t, write.Commit(lcms[len(lcms)-1].LedgerSequence())) diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index 014a75a1..2b554f39 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -49,6 +49,7 @@ type HandlerParams struct { EventStore *events.MemoryStore FeeStatWindows *feewindow.FeeWindows TransactionReader db.TransactionReader + EventReader db.EventReader LedgerEntryReader db.LedgerEntryReader LedgerReader db.LedgerReader Logger *log.Entry @@ -158,8 +159,8 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { }, { methodName: "getEvents", - underlyingHandler: methods.NewGetEventsHandler( - params.EventStore, cfg.MaxEventsLimit, cfg.DefaultEventsLimit), + underlyingHandler: methods.NewGetEventsHandler(params.Logger, params.EventReader, + params.EventStore, cfg.MaxEventsLimit, cfg.DefaultEventsLimit, cfg.NetworkPassphrase), longName: "get_events", queueLimit: cfg.RequestBacklogGetEventsQueueLimit, requestDurationLimit: cfg.MaxGetEventsExecutionDuration, diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 841ec192..dc913121 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -4,6 +4,8 @@ import ( "context" "encoding/json" "fmt" + "github.com/stellar/go/support/log" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" "strings" "time" @@ -303,12 +305,15 @@ type eventScanner interface { } type eventsRPCHandler struct { - scanner eventScanner - maxLimit uint - defaultLimit uint + dbReader db.EventReader + scanner eventScanner + maxLimit uint + defaultLimit uint + logger *log.Entry + networkPassphrase string } -func (h eventsRPCHandler) getEvents(request GetEventsRequest) (GetEventsResponse, error) { +func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsRequest) (GetEventsResponse, error) { if err := request.Valid(h.maxLimit); err != nil { return GetEventsResponse{}, &jrpc2.Error{ Code: jrpc2.InvalidParams, @@ -422,13 +427,16 @@ func eventInfoForEvent(event xdr.DiagnosticEvent, cursor events.Cursor, ledgerCl } // NewGetEventsHandler returns a json rpc handler to fetch and filter events -func NewGetEventsHandler(eventsStore *events.MemoryStore, maxLimit, defaultLimit uint) jrpc2.Handler { +func NewGetEventsHandler(logger *log.Entry, dbReader db.EventReader, eventsStore *events.MemoryStore, maxLimit, defaultLimit uint, networkPassphrase string) jrpc2.Handler { eventsHandler := eventsRPCHandler{ - scanner: eventsStore, - maxLimit: maxLimit, - defaultLimit: defaultLimit, + dbReader: dbReader, + scanner: eventsStore, + maxLimit: maxLimit, + defaultLimit: defaultLimit, + logger: logger, + networkPassphrase: networkPassphrase, } return NewHandler(func(ctx context.Context, request GetEventsRequest) (GetEventsResponse, error) { - return eventsHandler.getEvents(request) + return eventsHandler.getEvents(ctx, request) }) } diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 1960ad0d..d1510470 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -1,6 +1,7 @@ package methods import ( + "context" "encoding/json" "fmt" "strings" @@ -531,7 +532,7 @@ func TestGetEvents(t *testing.T) { maxLimit: 10000, defaultLimit: 100, } - _, err = handler.getEvents(GetEventsRequest{ + _, err = handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, }) assert.EqualError(t, err, "[-32600] event store is empty") @@ -561,12 +562,12 @@ func TestGetEvents(t *testing.T) { maxLimit: 10000, defaultLimit: 100, } - _, err = handler.getEvents(GetEventsRequest{ + _, err = handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, }) assert.EqualError(t, err, "[-32600] start is before oldest ledger") - _, err = handler.getEvents(GetEventsRequest{ + _, err = handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 3, }) assert.EqualError(t, err, "[-32600] start is after newest ledger") @@ -599,7 +600,7 @@ func TestGetEvents(t *testing.T) { maxLimit: 10000, defaultLimit: 100, } - results, err := handler.getEvents(GetEventsRequest{ + results, err := handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, }) assert.NoError(t, err) @@ -662,7 +663,7 @@ func TestGetEvents(t *testing.T) { maxLimit: 10000, defaultLimit: 100, } - results, err := handler.getEvents(GetEventsRequest{ + results, err := handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, Filters: []EventFilter{ {ContractIDs: []string{strkey.MustEncode(strkey.VersionByteContract, contractIds[0][:])}}, @@ -710,7 +711,7 @@ func TestGetEvents(t *testing.T) { maxLimit: 10000, defaultLimit: 100, } - results, err := handler.getEvents(GetEventsRequest{ + results, err := handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, Filters: []EventFilter{ {Topics: []TopicFilter{ @@ -804,7 +805,7 @@ func TestGetEvents(t *testing.T) { maxLimit: 10000, defaultLimit: 100, } - results, err := handler.getEvents(GetEventsRequest{ + results, err := handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, Filters: []EventFilter{ { @@ -879,7 +880,7 @@ func TestGetEvents(t *testing.T) { maxLimit: 10000, defaultLimit: 100, } - results, err := handler.getEvents(GetEventsRequest{ + results, err := handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, Filters: []EventFilter{ {EventType: map[string]interface{}{EventTypeSystem: nil}}, @@ -929,7 +930,7 @@ func TestGetEvents(t *testing.T) { maxLimit: 10000, defaultLimit: 100, } - results, err := handler.getEvents(GetEventsRequest{ + results, err := handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, Filters: []EventFilter{}, Pagination: &PaginationOptions{Limit: 10}, @@ -1015,7 +1016,7 @@ func TestGetEvents(t *testing.T) { maxLimit: 10000, defaultLimit: 100, } - results, err := handler.getEvents(GetEventsRequest{ + results, err := handler.getEvents(context.TODO(), GetEventsRequest{ Pagination: &PaginationOptions{ Cursor: id, Limit: 2, @@ -1047,7 +1048,7 @@ func TestGetEvents(t *testing.T) { } assert.Equal(t, GetEventsResponse{expected, 5}, results) - results, err = handler.getEvents(GetEventsRequest{ + results, err = handler.getEvents(context.TODO(), GetEventsRequest{ Pagination: &PaginationOptions{ Cursor: &events.Cursor{Ledger: 5, Tx: 2, Op: 0, Event: 1}, Limit: 2, From 9519aa2356bb5e94ad28cea1cc2916ac5ab34254 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 17 Jun 2024 12:42:02 -0700 Subject: [PATCH 07/63] Refactor getEvents to call fetch events from db --- cmd/soroban-rpc/internal/db/db.go | 5 +- cmd/soroban-rpc/internal/db/event.go | 49 +++++++----- cmd/soroban-rpc/internal/db/mocks.go | 10 +++ .../internal/db/transaction_test.go | 3 + .../internal/methods/get_events.go | 78 ++++++++++++++----- .../internal/methods/get_events_test.go | 2 +- 6 files changed, 104 insertions(+), 43 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go index e922ea35..3febcbe6 100644 --- a/cmd/soroban-rpc/internal/db/db.go +++ b/cmd/soroban-rpc/internal/db/db.go @@ -5,13 +5,12 @@ import ( "database/sql" "embed" "fmt" - "strconv" - "sync" - sq "github.com/Masterminds/squirrel" _ "github.com/mattn/go-sqlite3" "github.com/prometheus/client_golang/prometheus" migrate "github.com/rubenv/sql-migrate" + "strconv" + "sync" "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 425ba055..92ed00a9 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -27,7 +27,7 @@ type EventWriter interface { // EventReader has all the public methods to fetch events from DB type EventReader interface { - GetEvents(ctx context.Context, startLedgerSequence int, eventType int, contractIds []string, f ScanFunction) ([]xdr.DiagnosticEvent, error) + GetEvents(ctx context.Context, startLedgerSequence int, eventTypes []int, contractIds []string, f ScanFunction) error } type eventHandler struct { @@ -201,7 +201,7 @@ func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWi return err } -func (eventHandler *eventHandler) GetEvents(ctx context.Context, startLedgerSequence int, eventType int, contractIds []string, f ScanFunction) ([]xdr.DiagnosticEvent, error) { +func (eventHandler *eventHandler) GetEvents(ctx context.Context, startLedgerSequence int, eventTypes []int, contractIds []string, f ScanFunction) error { var rows []struct { TxIndex int `db:"application_order"` @@ -218,32 +218,41 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, startLedgerSequ rowQ = rowQ.Where(sq.Eq{"e.contract_id": contractIds}) } - if eventType != -1 { - rowQ = rowQ.Where(sq.Eq{"e.event_type": eventType}) + if len(eventTypes) > 0 { + rowQ = rowQ.Where(sq.Eq{"e.event_type": eventTypes}) } if err := eventHandler.db.Select(ctx, &rows, rowQ); err != nil { - return []xdr.DiagnosticEvent{}, - errors.Wrapf(err, "db read failed for startLedgerSequence= %d contractIds= %v eventType= %d ", startLedgerSequence, contractIds, eventType) + return errors.Wrapf(err, "db read failed for startLedgerSequence= %d contractIds= %v eventType= %d ", startLedgerSequence, contractIds, eventTypes) } else if len(rows) < 1 { - return []xdr.DiagnosticEvent{}, errors.New("No LCM found with requested event filters") + return errors.New("No LCM found with requested event filters") } - txIndex, lcm := rows[0].TxIndex, rows[0].Lcm - reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) - reader.Seek(txIndex - 1) + for _, row := range rows { + txIndex, lcm := row.TxIndex, row.Lcm + reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) + reader.Seek(txIndex - 1) - if err != nil { - return []xdr.DiagnosticEvent{}, - errors.Wrapf(err, "failed to index to tx %d in ledger %d", - txIndex, lcm.LedgerSequence()) - } + if err != nil { + return errors.Wrapf(err, "failed to index to tx %d in ledger %d", txIndex, lcm.LedgerSequence()) + } + ledgerCloseTime := lcm.LedgerCloseTime() + ledgerTx, err := reader.Read() + transactionHash := ledgerTx.Result.TransactionHash + events, diagErr := ledgerTx.GetDiagnosticEvents() - ledgerTx, err := reader.Read() - events, diagErr := ledgerTx.GetDiagnosticEvents() - if diagErr != nil { - return []xdr.DiagnosticEvent{}, errors.Wrapf(err, "db read failed for startLedgerSequence %d", startLedgerSequence) + if diagErr != nil { + return errors.Wrapf(err, "db read failed for startLedgerSequence %d", startLedgerSequence) + } + + // Find events based on filter passed in function f + for eventIndex, event := range events { + cur := Cursor{lcm.LedgerSequence(), uint32(txIndex), 0, uint32(eventIndex)} + if !f(event, cur, ledgerCloseTime, &transactionHash) { + return nil + } + } } - return events, nil + return nil } diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index d2dfba4f..f4bbc619 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -105,6 +105,16 @@ func (m *mockLedgerReader) StreamAllLedgers(ctx context.Context, f StreamLedgerF return nil } +type mockEventReader struct { +} + +func NewMockEventReader() { +} + +func (m *mockEventReader) GetEvents(ctx context.Context, startLedgerSequence int, eventTypes []int, contractIds []string, f ScanFunction) error { + return nil +} + var _ TransactionReader = &mockTransactionHandler{} var _ TransactionWriter = &mockTransactionHandler{} var _ LedgerReader = &mockLedgerReader{} diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 39d1c8a4..c7e21857 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -94,6 +94,9 @@ func TestTransactionFound(t *testing.T) { _, _, err = reader.GetTransaction(ctx, xdr.Hash{}) require.Error(t, err, ErrNoTransaction) + eventReader := NewEventReader(log, db, passphrase) + err = eventReader.GetEvents(ctx, 1, nil, nil, nil) + // check all 200 cases for _, lcm := range lcms { h := lcm.TransactionHash(0) diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index dc913121..63360138 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -135,6 +135,12 @@ var eventTypeFromXDR = map[xdr.ContractEventType]string{ xdr.ContractEventTypeDiagnostic: EventTypeDiagnostic, } +var eventTypeToInteger = map[string]int{ + EventTypeSystem: 1, + EventTypeContract: 2, + EventTypeDiagnostic: 3, +} + type EventFilter struct { EventType eventTypeSet `json:"type,omitempty"` ContractIDs []string `json:"contractIds,omitempty"` @@ -336,34 +342,67 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } type entry struct { - cursor events.Cursor + cursor db.Cursor ledgerCloseTimestamp int64 event xdr.DiagnosticEvent txHash *xdr.Hash } var found []entry - latestLedger, err := h.scanner.Scan( - events.Range{ - Start: start, - ClampStart: false, - End: events.MaxCursor, - ClampEnd: true, - }, - func(event xdr.DiagnosticEvent, cursor events.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { - if request.Matches(event) { + + if len(request.Filters) == 0 { + err := h.dbReader.GetEvents( + ctx, + int(request.StartLedger), + nil, + nil, + + func(event xdr.DiagnosticEvent, cursor db.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { + found = append(found, entry{cursor, ledgerCloseTimestamp, event, txHash}) + return uint(len(found)) < limit + }, + ) + + if err != nil { + return GetEventsResponse{}, &jrpc2.Error{ + Code: jrpc2.InvalidRequest, + Message: err.Error(), + } + } + + } + + for _, filter := range request.Filters { + + var eventTypes []int + for key := range filter.EventType { + eventTypes = append(eventTypes, eventTypeToInteger[key]) + } + + // Scan function to apply filter and to remove duplicates if any + f := func(event xdr.DiagnosticEvent, cursor db.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { + if filter.Matches(event) { found = append(found, entry{cursor, ledgerCloseTimestamp, event, txHash}) } return uint(len(found)) < limit - }, - ) - if err != nil { - return GetEventsResponse{}, &jrpc2.Error{ - Code: jrpc2.InvalidRequest, - Message: err.Error(), + } + + err := h.dbReader.GetEvents( + ctx, + int(request.StartLedger), + eventTypes, + filter.ContractIDs, + f, + ) + + if err != nil { + return GetEventsResponse{}, &jrpc2.Error{ + Code: jrpc2.InvalidRequest, + Message: err.Error(), + } } } - results := []EventInfo{} + var results []EventInfo for _, entry := range found { info, err := eventInfoForEvent( entry.event, @@ -376,13 +415,14 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } results = append(results, info) } + //TODO (prit): Refactor latest ledger code !! return GetEventsResponse{ - LatestLedger: uint32(latestLedger), + LatestLedger: 0, Events: results, }, nil } -func eventInfoForEvent(event xdr.DiagnosticEvent, cursor events.Cursor, ledgerClosedAt string, txHash string) (EventInfo, error) { +func eventInfoForEvent(event xdr.DiagnosticEvent, cursor db.Cursor, ledgerClosedAt string, txHash string) (EventInfo, error) { v0, ok := event.Event.Body.GetV0() if !ok { return EventInfo{}, errors.New("unknown event version") diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index d1510470..04e3733a 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "strings" "testing" "time" @@ -16,7 +17,6 @@ import ( "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" ) func TestEventTypeSetMatches(t *testing.T) { From 133ef9733b14a2ffd36a443e5e26bfaac5df012e Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 17 Jun 2024 13:54:49 -0700 Subject: [PATCH 08/63] Remove in-memory events store --- cmd/soroban-rpc/internal/daemon/daemon.go | 12 +- cmd/soroban-rpc/internal/db/mocks.go | 26 ++ cmd/soroban-rpc/internal/events/events.go | 176 +------- .../internal/events/events_test.go | 407 ------------------ cmd/soroban-rpc/internal/ingest/service.go | 9 - .../internal/ingest/service_test.go | 3 - cmd/soroban-rpc/internal/jsonrpc.go | 7 +- .../internal/methods/get_events.go | 8 +- .../internal/methods/get_events_test.go | 46 +- 9 files changed, 57 insertions(+), 637 deletions(-) delete mode 100644 cmd/soroban-rpc/internal/events/events_test.go diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index 88c8a51a..c6d19e4a 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -26,7 +26,6 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ingest" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" @@ -187,11 +186,6 @@ func MustNew(cfg *config.Config) *Daemon { }, metricsRegistry), } - eventStore := events.NewMemoryStore( - daemon, - cfg.NetworkPassphrase, - cfg.EventLedgerRetentionWindow, - ) feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase) // initialize the stores using what was on the DB @@ -214,9 +208,7 @@ func MustNew(cfg *config.Config) *Daemon { "seq": currentSeq, }).Debug("still initializing in-memory store") } - if err := eventStore.IngestEvents(txmeta); err != nil { - logger.WithError(err).Fatal("could not initialize event memory store") - } + if err := feewindows.IngestFees(txmeta); err != nil { logger.WithError(err).Fatal("could not initialize fee stats") } @@ -254,7 +246,6 @@ func MustNew(cfg *config.Config) *Daemon { maxRetentionWindow, cfg.NetworkPassphrase, ), - EventStore: eventStore, NetworkPassPhrase: cfg.NetworkPassphrase, Archive: historyArchive, LedgerBackend: core, @@ -277,7 +268,6 @@ func MustNew(cfg *config.Config) *Daemon { jsonRPCHandler := internal.NewJSONRPCHandler(cfg, internal.HandlerParams{ Daemon: daemon, - EventStore: eventStore, FeeStatWindows: feewindows, Logger: logger, LedgerReader: db.NewLedgerReader(dbConn), diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index f4bbc619..7620b2c7 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -30,6 +30,32 @@ func NewMockTransactionStore(passphrase string) *mockTransactionHandler { } } +type mockEventHandler struct { + passphrase string + + ledgerRange ledgerbucketwindow.LedgerRange + txs map[string]ingest.LedgerTransaction + txHashToMeta map[string]*xdr.LedgerCloseMeta + ledgerSeqToMeta map[uint32]*xdr.LedgerCloseMeta +} + +func NewMockEventStore(passphrase string) *mockEventHandler { + return &mockEventHandler{ + passphrase: passphrase, + txs: make(map[string]ingest.LedgerTransaction), + txHashToMeta: make(map[string]*xdr.LedgerCloseMeta), + ledgerSeqToMeta: make(map[uint32]*xdr.LedgerCloseMeta), + } +} + +func (txn *mockEventHandler) GetEvents(ctx context.Context, startLedgerSequence int, eventTypes []int, contractIds []string, f ScanFunction) error { + return nil +} + +func (txn *mockEventHandler) IngestEvents(lcm xdr.LedgerCloseMeta) error { + return nil +} + func (txn *mockTransactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) error { txn.ledgerSeqToMeta[lcm.LedgerSequence()] = &lcm diff --git a/cmd/soroban-rpc/internal/events/events.go b/cmd/soroban-rpc/internal/events/events.go index b724fbf9..931d69a0 100644 --- a/cmd/soroban-rpc/internal/events/events.go +++ b/cmd/soroban-rpc/internal/events/events.go @@ -2,16 +2,10 @@ package events import ( "errors" - "io" - "sort" - "sync" - "time" - "github.com/prometheus/client_golang/prometheus" - "github.com/stellar/go/ingest" "github.com/stellar/go/xdr" + "sync" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) @@ -44,39 +38,6 @@ type MemoryStore struct { eventCountMetric prometheus.Summary } -// NewMemoryStore creates a new MemoryStore. -// The retention window is in units of ledgers. -// All events occurring in the following ledger range -// [ latestLedger - retentionWindow, latestLedger ] -// will be included in the MemoryStore. If the MemoryStore -// is full, any events from new ledgers will evict -// older entries outside the retention window. -func NewMemoryStore(daemon interfaces.Daemon, networkPassphrase string, retentionWindow uint32) *MemoryStore { - window := ledgerbucketwindow.NewLedgerBucketWindow[[]event](retentionWindow) - - // eventsDurationMetric is a metric for measuring latency of event store operations - eventsDurationMetric := prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Namespace: daemon.MetricsNamespace(), Subsystem: "events", Name: "operation_duration_seconds", - Help: "event store operation durations, sliding window = 10m", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }, - []string{"operation"}, - ) - - eventCountMetric := prometheus.NewSummary(prometheus.SummaryOpts{ - Namespace: daemon.MetricsNamespace(), Subsystem: "events", Name: "count", - Help: "count of events ingested, sliding window = 10m", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, - }) - daemon.MetricsRegistry().MustRegister(eventCountMetric, eventsDurationMetric) - return &MemoryStore{ - networkPassphrase: networkPassphrase, - eventsByLedger: window, - eventsDurationMetric: eventsDurationMetric, - eventCountMetric: eventCountMetric, - } -} - // Range defines a [Start, End) interval of Soroban events. type Range struct { // Start defines the (inclusive) start of the range. @@ -100,47 +61,6 @@ type ScanFunction func(xdr.DiagnosticEvent, Cursor, int64, *xdr.Hash) bool // entire duration of the Scan function so f should be written in a way // to minimize latency. func (m *MemoryStore) Scan(eventRange Range, f ScanFunction) (lastLedgerInWindow uint32, err error) { - startTime := time.Now() - defer func() { - if err == nil { - m.eventsDurationMetric.With(prometheus.Labels{"operation": "scan"}). - Observe(time.Since(startTime).Seconds()) - } - }() - - m.lock.RLock() - defer m.lock.RUnlock() - - if err = m.validateRange(&eventRange); err != nil { - return - } - - firstLedgerInRange := eventRange.Start.Ledger - firstLedgerInWindow := m.eventsByLedger.Get(0).LedgerSeq - lastLedgerInWindow = firstLedgerInWindow + (m.eventsByLedger.Len() - 1) - for i := firstLedgerInRange - firstLedgerInWindow; i < m.eventsByLedger.Len(); i++ { - bucket := m.eventsByLedger.Get(i) - events := bucket.BucketContent - if bucket.LedgerSeq == firstLedgerInRange { - // we need to seek for the beginning of the events in the first bucket in the range - events = seek(events, eventRange.Start) - } - timestamp := bucket.LedgerCloseTimestamp - for _, event := range events { - cur := event.cursor(bucket.LedgerSeq) - if eventRange.End.Cmp(cur) <= 0 { - return - } - var diagnosticEvent xdr.DiagnosticEvent - err = xdr.SafeUnmarshal(event.diagnosticEventXDR, &diagnosticEvent) - if err != nil { - return - } - if !f(diagnosticEvent, cur, timestamp, event.txHash) { - return - } - } - } return } @@ -178,97 +98,3 @@ func (m *MemoryStore) validateRange(eventRange *Range) error { return nil } - -// seek returns the subset of all events which occur -// at a point greater than or equal to the given cursor. -// events must be sorted in ascending order. -func seek(events []event, cursor Cursor) []event { - j := sort.Search(len(events), func(i int) bool { - return cursor.Cmp(events[i].cursor(cursor.Ledger)) <= 0 - }) - return events[j:] -} - -// IngestEvents adds new events from the given ledger into the store. -// As a side effect, events which fall outside the retention window are -// removed from the store. -func (m *MemoryStore) IngestEvents(ledgerCloseMeta xdr.LedgerCloseMeta) error { - startTime := time.Now() - // no need to acquire the lock because the networkPassphrase field - // is immutable - events, err := readEvents(m.networkPassphrase, ledgerCloseMeta) - if err != nil { - return err - } - bucket := ledgerbucketwindow.LedgerBucket[[]event]{ - LedgerSeq: ledgerCloseMeta.LedgerSequence(), - LedgerCloseTimestamp: ledgerCloseMeta.LedgerCloseTime(), - BucketContent: events, - } - m.lock.Lock() - if _, err = m.eventsByLedger.Append(bucket); err != nil { - m.lock.Unlock() - return err - } - m.lock.Unlock() - m.eventsDurationMetric.With(prometheus.Labels{"operation": "ingest"}). - Observe(time.Since(startTime).Seconds()) - m.eventCountMetric.Observe(float64(len(events))) - return nil -} - -func readEvents(networkPassphrase string, ledgerCloseMeta xdr.LedgerCloseMeta) (events []event, err error) { - var txReader *ingest.LedgerTransactionReader - txReader, err = ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(networkPassphrase, ledgerCloseMeta) - if err != nil { - return - } - defer func() { - closeErr := txReader.Close() - if err == nil { - err = closeErr - } - }() - - for { - var tx ingest.LedgerTransaction - tx, err = txReader.Read() - if err == io.EOF { - err = nil - break - } - if err != nil { - return - } - - if !tx.Result.Successful() { - continue - } - - txEvents, err := tx.GetDiagnosticEvents() - if err != nil { - return nil, err - } - txHash := tx.Result.TransactionHash - for index, e := range txEvents { - diagnosticEventXDR, err := e.MarshalBinary() - if err != nil { - return nil, err - } - events = append(events, event{ - diagnosticEventXDR: diagnosticEventXDR, - txIndex: tx.Index, - eventIndex: uint32(index), - txHash: &txHash, - }) - } - } - return events, err -} - -// GetLedgerRange returns the first and latest ledger available in the store. -func (m *MemoryStore) GetLedgerRange() (ledgerbucketwindow.LedgerRange, error) { - m.lock.RLock() - defer m.lock.RUnlock() - return m.eventsByLedger.GetLedgerRange(), nil -} diff --git a/cmd/soroban-rpc/internal/events/events_test.go b/cmd/soroban-rpc/internal/events/events_test.go deleted file mode 100644 index c5fda34c..00000000 --- a/cmd/soroban-rpc/internal/events/events_test.go +++ /dev/null @@ -1,407 +0,0 @@ -package events - -import ( - "bytes" - "testing" - - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" - "github.com/stellar/go/xdr" - "github.com/stretchr/testify/require" - - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" -) - -var ( - ledger5CloseTime = ledgerCloseTime(5) - ledger5Events = []event{ - newEvent(1, 0, 100), - newEvent(1, 1, 200), - newEvent(2, 0, 300), - newEvent(2, 1, 400), - } - ledger6CloseTime = ledgerCloseTime(6) - ledger6Events []event = nil - ledger7CloseTime = ledgerCloseTime(7) - ledger7Events = []event{ - newEvent(1, 0, 500), - } - ledger8CloseTime = ledgerCloseTime(8) - ledger8Events = []event{ - newEvent(1, 0, 600), - newEvent(2, 0, 700), - newEvent(2, 1, 800), - newEvent(2, 2, 900), - newEvent(2, 3, 1000), - } -) - -func ledgerCloseTime(seq uint32) int64 { - return int64(seq)*25 + 100 -} - -func newEvent(txIndex, eventIndex, val uint32) event { - v := xdr.Uint32(val) - - e := xdr.DiagnosticEvent{ - InSuccessfulContractCall: true, - Event: xdr.ContractEvent{ - Type: xdr.ContractEventTypeSystem, - Body: xdr.ContractEventBody{ - V: 0, - V0: &xdr.ContractEventV0{ - Data: xdr.ScVal{ - Type: xdr.ScValTypeScvU32, - U32: &v, - }, - }, - }, - }, - } - diagnosticEventXDR, err := e.MarshalBinary() - if err != nil { - panic(err) - } - return event{ - diagnosticEventXDR: diagnosticEventXDR, - txIndex: txIndex, - eventIndex: eventIndex, - } -} - -func (e event) equals(other event) bool { - return e.txIndex == other.txIndex && - e.eventIndex == other.eventIndex && - bytes.Equal(e.diagnosticEventXDR, other.diagnosticEventXDR) -} - -func eventsAreEqual(t *testing.T, a, b []event) { - require.Equal(t, len(a), len(b)) - for i := range a { - require.True(t, a[i].equals(b[i])) - } -} - -func TestScanRangeValidation(t *testing.T) { - m := NewMemoryStore(interfaces.MakeNoOpDeamon(), "unit-tests", 4) - assertNoCalls := func(xdr.DiagnosticEvent, Cursor, int64, *xdr.Hash) bool { - t.Fatalf("unexpected call") - return true - } - _, err := m.Scan(Range{ - Start: MinCursor, - ClampStart: true, - End: MaxCursor, - ClampEnd: true, - }, assertNoCalls) - require.EqualError(t, err, "event store is empty") - - m = createStore(t) - - for _, testCase := range []struct { - input Range - err string - }{ - { - Range{ - Start: MinCursor, - ClampStart: false, - End: MaxCursor, - ClampEnd: true, - }, - "start is before oldest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 4}, - ClampStart: false, - End: MaxCursor, - ClampEnd: true, - }, - "start is before oldest ledger", - }, - { - Range{ - Start: MinCursor, - ClampStart: true, - End: MaxCursor, - ClampEnd: false, - }, - "end is after latest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 5}, - ClampStart: true, - End: Cursor{Ledger: 10}, - ClampEnd: false, - }, - "end is after latest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 10}, - ClampStart: true, - End: Cursor{Ledger: 3}, - ClampEnd: true, - }, - "start is after newest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 10}, - ClampStart: false, - End: Cursor{Ledger: 3}, - ClampEnd: false, - }, - "start is after newest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 9}, - ClampStart: false, - End: Cursor{Ledger: 10}, - ClampEnd: true, - }, - "start is after newest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 9}, - ClampStart: false, - End: Cursor{Ledger: 10}, - ClampEnd: false, - }, - "start is after newest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 2}, - ClampStart: true, - End: Cursor{Ledger: 3}, - ClampEnd: false, - }, - "start is not before end", - }, - { - Range{ - Start: Cursor{Ledger: 2}, - ClampStart: false, - End: Cursor{Ledger: 3}, - ClampEnd: false, - }, - "start is before oldest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 6}, - ClampStart: false, - End: Cursor{Ledger: 6}, - ClampEnd: false, - }, - "start is not before end", - }, - } { - _, err := m.Scan(testCase.input, assertNoCalls) - require.EqualError(t, err, testCase.err, testCase.input) - } -} - -func createStore(t *testing.T) *MemoryStore { - m := NewMemoryStore(interfaces.MakeNoOpDeamon(), "unit-tests", 4) - m.eventsByLedger.Append(ledgerbucketwindow.LedgerBucket[[]event]{ - LedgerSeq: 5, - LedgerCloseTimestamp: ledger5CloseTime, - BucketContent: ledger5Events, - }) - m.eventsByLedger.Append(ledgerbucketwindow.LedgerBucket[[]event]{ - LedgerSeq: 6, - LedgerCloseTimestamp: ledger6CloseTime, - BucketContent: nil, - }) - m.eventsByLedger.Append(ledgerbucketwindow.LedgerBucket[[]event]{ - LedgerSeq: 7, - LedgerCloseTimestamp: ledger7CloseTime, - BucketContent: ledger7Events, - }) - m.eventsByLedger.Append(ledgerbucketwindow.LedgerBucket[[]event]{ - LedgerSeq: 8, - LedgerCloseTimestamp: ledger8CloseTime, - BucketContent: ledger8Events, - }) - - return m -} - -func concat(slices ...[]event) []event { - var result []event - for _, slice := range slices { - result = append(result, slice...) - } - return result -} - -func getMetricValue(metric prometheus.Metric) *dto.Metric { - value := &dto.Metric{} - err := metric.Write(value) - if err != nil { - panic(err) - } - return value -} - -func TestScan(t *testing.T) { - genEquivalentInputs := func(input Range) []Range { - results := []Range{input} - if !input.ClampStart { - rangeCopy := input - rangeCopy.ClampStart = true - results = append(results, rangeCopy) - } - if !input.ClampEnd { - rangeCopy := input - rangeCopy.ClampEnd = true - results = append(results, rangeCopy) - } - if !input.ClampStart && !input.ClampEnd { - rangeCopy := input - rangeCopy.ClampStart = true - rangeCopy.ClampEnd = true - results = append(results, rangeCopy) - } - return results - } - - for _, testCase := range []struct { - input Range - expected []event - }{ - { - Range{ - Start: MinCursor, - ClampStart: true, - End: MaxCursor, - ClampEnd: true, - }, - concat(ledger5Events, ledger6Events, ledger7Events, ledger8Events), - }, - { - Range{ - Start: Cursor{Ledger: 5}, - ClampStart: false, - End: Cursor{Ledger: 9}, - ClampEnd: false, - }, - concat(ledger5Events, ledger6Events, ledger7Events, ledger8Events), - }, - { - Range{ - Start: Cursor{Ledger: 5, Tx: 2}, - ClampStart: false, - End: Cursor{Ledger: 9}, - ClampEnd: false, - }, - concat(ledger5Events[2:], ledger6Events, ledger7Events, ledger8Events), - }, - { - Range{ - Start: Cursor{Ledger: 5, Tx: 3}, - ClampStart: false, - End: MaxCursor, - ClampEnd: true, - }, - concat(ledger6Events, ledger7Events, ledger8Events), - }, - { - Range{ - Start: Cursor{Ledger: 6}, - ClampStart: false, - End: MaxCursor, - ClampEnd: true, - }, - concat(ledger7Events, ledger8Events), - }, - { - Range{ - Start: Cursor{Ledger: 6, Tx: 1}, - ClampStart: false, - End: MaxCursor, - ClampEnd: true, - }, - concat(ledger7Events, ledger8Events), - }, - { - Range{ - Start: Cursor{Ledger: 8, Tx: 2, Event: 3}, - ClampStart: false, - End: MaxCursor, - ClampEnd: true, - }, - ledger8Events[len(ledger8Events)-1:], - }, - { - Range{ - Start: Cursor{Ledger: 8, Tx: 2, Event: 3}, - ClampStart: false, - End: Cursor{Ledger: 9}, - ClampEnd: false, - }, - ledger8Events[len(ledger8Events)-1:], - }, - { - Range{ - Start: Cursor{Ledger: 5}, - ClampStart: false, - End: Cursor{Ledger: 7}, - ClampEnd: false, - }, - concat(ledger5Events, ledger6Events), - }, - { - Range{ - Start: Cursor{Ledger: 5, Tx: 2}, - ClampStart: false, - End: Cursor{Ledger: 8, Tx: 2}, - ClampEnd: false, - }, - concat(ledger5Events[2:], ledger6Events, ledger7Events, ledger8Events[:1]), - }, - } { - for _, input := range genEquivalentInputs(testCase.input) { - m := createStore(t) - var events []event - iterateAll := true - f := func(contractEvent xdr.DiagnosticEvent, cursor Cursor, ledgerCloseTimestamp int64, hash *xdr.Hash) bool { - require.Equal(t, ledgerCloseTime(cursor.Ledger), ledgerCloseTimestamp) - diagnosticEventXDR, err := contractEvent.MarshalBinary() - require.NoError(t, err) - events = append(events, event{ - diagnosticEventXDR: diagnosticEventXDR, - txIndex: cursor.Tx, - eventIndex: cursor.Event, - txHash: hash, - }) - return iterateAll - } - latest, err := m.Scan(input, f) - require.NoError(t, err) - require.Equal(t, uint32(8), latest) - eventsAreEqual(t, testCase.expected, events) - metric, err := m.eventsDurationMetric.MetricVec.GetMetricWith(prometheus.Labels{ - "operation": "scan", - }) - require.NoError(t, err) - require.Equal(t, uint64(1), getMetricValue(metric).GetSummary().GetSampleCount()) - if len(events) > 0 { - events = nil - iterateAll = false - latest, err := m.Scan(input, f) - require.NoError(t, err) - require.Equal(t, uint64(2), getMetricValue(metric).GetSummary().GetSampleCount()) - require.Equal(t, uint32(8), latest) - eventsAreEqual(t, []event{testCase.expected[0]}, events) - } - } - } -} diff --git a/cmd/soroban-rpc/internal/ingest/service.go b/cmd/soroban-rpc/internal/ingest/service.go index 16285100..90f622f3 100644 --- a/cmd/soroban-rpc/internal/ingest/service.go +++ b/cmd/soroban-rpc/internal/ingest/service.go @@ -20,8 +20,6 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/util" - - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" ) const ( @@ -33,7 +31,6 @@ var errEmptyArchives = fmt.Errorf("cannot start ingestion without history archiv type Config struct { Logger *log.Entry DB db.ReadWriter - EventStore *events.MemoryStore FeeWindows *feewindow.FeeWindows NetworkPassPhrase string Archive historyarchive.ArchiveInterface @@ -81,7 +78,6 @@ func newService(cfg Config) *Service { service := &Service{ logger: cfg.Logger, db: cfg.DB, - eventStore: cfg.EventStore, feeWindows: cfg.FeeWindows, ledgerBackend: cfg.LedgerBackend, networkPassPhrase: cfg.NetworkPassPhrase, @@ -133,7 +129,6 @@ type Metrics struct { type Service struct { logger *log.Entry db db.ReadWriter - eventStore *events.MemoryStore feeWindows *feewindow.FeeWindows ledgerBackend backends.LedgerBackend timeout time.Duration @@ -341,10 +336,6 @@ func (s *Service) ingestLedgerCloseMeta(tx db.WriteTx, ledgerCloseMeta xdr.Ledge return err } - if err := s.eventStore.IngestEvents(ledgerCloseMeta); err != nil { - return err - } - if err := s.feeWindows.IngestFees(ledgerCloseMeta); err != nil { return err } diff --git a/cmd/soroban-rpc/internal/ingest/service_test.go b/cmd/soroban-rpc/internal/ingest/service_test.go index 10090c16..3552b441 100644 --- a/cmd/soroban-rpc/internal/ingest/service_test.go +++ b/cmd/soroban-rpc/internal/ingest/service_test.go @@ -16,7 +16,6 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow" ) @@ -45,7 +44,6 @@ func TestRetryRunningIngestion(t *testing.T) { config := Config{ Logger: supportlog.New(), DB: &ErrorReadWriter{}, - EventStore: nil, NetworkPassPhrase: "", Archive: nil, LedgerBackend: nil, @@ -69,7 +67,6 @@ func TestIngestion(t *testing.T) { config := Config{ Logger: supportlog.New(), DB: mockDB, - EventStore: events.NewMemoryStore(daemon, network.TestNetworkPassphrase, 1), FeeWindows: feewindow.NewFeeWindows(1, 1, network.TestNetworkPassphrase), LedgerBackend: mockLedgerBackend, Daemon: daemon, diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index 2b554f39..40e24d57 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -19,7 +19,6 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/feewindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/methods" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/network" @@ -46,7 +45,6 @@ func (h Handler) Close() { } type HandlerParams struct { - EventStore *events.MemoryStore FeeStatWindows *feewindow.FeeWindows TransactionReader db.TransactionReader EventReader db.EventReader @@ -158,9 +156,8 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { requestDurationLimit: cfg.MaxGetHealthExecutionDuration, }, { - methodName: "getEvents", - underlyingHandler: methods.NewGetEventsHandler(params.Logger, params.EventReader, - params.EventStore, cfg.MaxEventsLimit, cfg.DefaultEventsLimit, cfg.NetworkPassphrase), + methodName: "getEvents", + underlyingHandler: methods.NewGetEventsHandler(params.Logger, params.EventReader, cfg.MaxEventsLimit, cfg.DefaultEventsLimit, cfg.NetworkPassphrase), longName: "get_events", queueLimit: cfg.RequestBacklogGetEventsQueueLimit, requestDurationLimit: cfg.MaxGetEventsExecutionDuration, diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 63360138..8f01f06b 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -306,13 +306,8 @@ type GetEventsResponse struct { LatestLedger uint32 `json:"latestLedger"` } -type eventScanner interface { - Scan(eventRange events.Range, f events.ScanFunction) (uint32, error) -} - type eventsRPCHandler struct { dbReader db.EventReader - scanner eventScanner maxLimit uint defaultLimit uint logger *log.Entry @@ -467,10 +462,9 @@ func eventInfoForEvent(event xdr.DiagnosticEvent, cursor db.Cursor, ledgerClosed } // NewGetEventsHandler returns a json rpc handler to fetch and filter events -func NewGetEventsHandler(logger *log.Entry, dbReader db.EventReader, eventsStore *events.MemoryStore, maxLimit, defaultLimit uint, networkPassphrase string) jrpc2.Handler { +func NewGetEventsHandler(logger *log.Entry, dbReader db.EventReader, maxLimit, defaultLimit uint, networkPassphrase string) jrpc2.Handler { eventsHandler := eventsRPCHandler{ dbReader: dbReader, - scanner: eventsStore, maxLimit: maxLimit, defaultLimit: defaultLimit, logger: logger, diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 04e3733a..fcc02fe5 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "strings" "testing" @@ -15,8 +16,6 @@ import ( "github.com/stellar/go/network" "github.com/stellar/go/strkey" "github.com/stellar/go/xdr" - - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" ) func TestEventTypeSetMatches(t *testing.T) { @@ -526,9 +525,9 @@ func TestGetEvents(t *testing.T) { assert.NoError(t, err) t.Run("empty", func(t *testing.T) { - store := events.NewMemoryStore(interfaces.MakeNoOpDeamon(), "unit-tests", 100) + store := db.NewMockEventStore("passphrase") handler := eventsRPCHandler{ - scanner: store, + dbReader: store, maxLimit: 10000, defaultLimit: 100, } @@ -540,7 +539,7 @@ func TestGetEvents(t *testing.T) { t.Run("startLedger validation", func(t *testing.T) { contractID := xdr.Hash([32]byte{}) - store := events.NewMemoryStore(interfaces.MakeNoOpDeamon(), "unit-tests", 100) + store := db.NewMockEventStore("passphrase") var txMeta []xdr.TransactionMeta txMeta = append(txMeta, transactionMetaWithEvents( contractEvent( @@ -558,7 +557,7 @@ func TestGetEvents(t *testing.T) { assert.NoError(t, store.IngestEvents(ledgerCloseMetaWithEvents(2, now.Unix(), txMeta...))) handler := eventsRPCHandler{ - scanner: store, + dbReader: store, maxLimit: 10000, defaultLimit: 100, } @@ -575,7 +574,8 @@ func TestGetEvents(t *testing.T) { t.Run("no filtering returns all", func(t *testing.T) { contractID := xdr.Hash([32]byte{}) - store := events.NewMemoryStore(interfaces.MakeNoOpDeamon(), "unit-tests", 100) + store := db.NewMockEventStore("passphrase") + var txMeta []xdr.TransactionMeta for i := 0; i < 10; i++ { txMeta = append(txMeta, transactionMetaWithEvents( @@ -596,7 +596,7 @@ func TestGetEvents(t *testing.T) { assert.NoError(t, store.IngestEvents(ledgerCloseMeta)) handler := eventsRPCHandler{ - scanner: store, + dbReader: store, maxLimit: 10000, defaultLimit: 100, } @@ -635,7 +635,8 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by contract id", func(t *testing.T) { - store := events.NewMemoryStore(interfaces.MakeNoOpDeamon(), "unit-tests", 100) + store := db.NewMockEventStore("passphrase") + var txMeta []xdr.TransactionMeta contractIds := []xdr.Hash{ xdr.Hash([32]byte{}), @@ -659,7 +660,7 @@ func TestGetEvents(t *testing.T) { assert.NoError(t, store.IngestEvents(ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...))) handler := eventsRPCHandler{ - scanner: store, + dbReader: store, maxLimit: 10000, defaultLimit: 100, } @@ -685,7 +686,8 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by topic", func(t *testing.T) { - store := events.NewMemoryStore(interfaces.MakeNoOpDeamon(), "unit-tests", 100) + store := db.NewMockEventStore("passphrase") + var txMeta []xdr.TransactionMeta contractID := xdr.Hash([32]byte{}) for i := 0; i < 10; i++ { @@ -707,7 +709,7 @@ func TestGetEvents(t *testing.T) { number := xdr.Uint64(4) handler := eventsRPCHandler{ - scanner: store, + dbReader: store, maxLimit: 10000, defaultLimit: 100, } @@ -749,7 +751,8 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by both contract id and topic", func(t *testing.T) { - store := events.NewMemoryStore(interfaces.MakeNoOpDeamon(), "unit-tests", 100) + store := db.NewMockEventStore("passphrase") + contractID := xdr.Hash([32]byte{}) otherContractID := xdr.Hash([32]byte{1}) number := xdr.Uint64(1) @@ -801,7 +804,7 @@ func TestGetEvents(t *testing.T) { assert.NoError(t, store.IngestEvents(ledgerCloseMeta)) handler := eventsRPCHandler{ - scanner: store, + dbReader: store, maxLimit: 10000, defaultLimit: 100, } @@ -845,7 +848,8 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by event type", func(t *testing.T) { - store := events.NewMemoryStore(interfaces.MakeNoOpDeamon(), "unit-tests", 100) + store := db.NewMockEventStore("passphrase") + contractID := xdr.Hash([32]byte{}) txMeta := []xdr.TransactionMeta{ transactionMetaWithEvents( @@ -876,7 +880,7 @@ func TestGetEvents(t *testing.T) { assert.NoError(t, store.IngestEvents(ledgerCloseMeta)) handler := eventsRPCHandler{ - scanner: store, + dbReader: store, maxLimit: 10000, defaultLimit: 100, } @@ -907,7 +911,8 @@ func TestGetEvents(t *testing.T) { }) t.Run("with limit", func(t *testing.T) { - store := events.NewMemoryStore(interfaces.MakeNoOpDeamon(), "unit-tests", 100) + store := db.NewMockEventStore("passphrase") + contractID := xdr.Hash([32]byte{}) var txMeta []xdr.TransactionMeta for i := 0; i < 180; i++ { @@ -926,7 +931,7 @@ func TestGetEvents(t *testing.T) { assert.NoError(t, store.IngestEvents(ledgerCloseMeta)) handler := eventsRPCHandler{ - scanner: store, + dbReader: store, maxLimit: 10000, defaultLimit: 100, } @@ -964,7 +969,8 @@ func TestGetEvents(t *testing.T) { }) t.Run("with cursor", func(t *testing.T) { - store := events.NewMemoryStore(interfaces.MakeNoOpDeamon(), "unit-tests", 100) + store := db.NewMockEventStore("passphrase") + contractID := xdr.Hash([32]byte{}) datas := []xdr.ScSymbol{ // ledger/transaction/operation/event @@ -1012,7 +1018,7 @@ func TestGetEvents(t *testing.T) { id := &events.Cursor{Ledger: 5, Tx: 1, Op: 0, Event: 0} handler := eventsRPCHandler{ - scanner: store, + dbReader: store, maxLimit: 10000, defaultLimit: 100, } From c52f6187489a8671f716576b5a8ba488929e4c0b Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 17 Jun 2024 22:12:30 -0700 Subject: [PATCH 09/63] Make use of cursor in pagination and combine filters --- cmd/soroban-rpc/internal/db/event.go | 106 +++--------------- cmd/soroban-rpc/internal/db/mocks.go | 3 +- .../internal/methods/get_events.go | 92 ++++++++------- 3 files changed, 62 insertions(+), 139 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 92ed00a9..f5501733 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -2,7 +2,6 @@ package db import ( "context" - "encoding/json" "fmt" sq "github.com/Masterminds/squirrel" "github.com/prometheus/client_golang/prometheus" @@ -10,12 +9,9 @@ import ( "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" "github.com/stellar/go/support/log" - "github.com/stellar/go/toid" "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "io" - "strconv" - "strings" ) const eventTableName = "events" @@ -27,7 +23,7 @@ type EventWriter interface { // EventReader has all the public methods to fetch events from DB type EventReader interface { - GetEvents(ctx context.Context, startLedgerSequence int, eventTypes []int, contractIds []string, f ScanFunction) error + GetEvents(ctx context.Context, startCursor events.Cursor, eventTypes []int, contractIds []string, f ScanFunction) error } type eventHandler struct { @@ -110,81 +106,7 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { return nil } -// Cursor represents the position of a Soroban event. -// Soroban events are sorted in ascending order by -// ledger sequence, transaction index, operation index, -// and event index. -type Cursor struct { - // Ledger is the sequence of the ledger which emitted the event. - Ledger uint32 - // Tx is the index of the transaction within the ledger which emitted the event. - Tx uint32 - // Op is the index of the operation within the transaction which emitted the event. - // Note: Currently, there is no use for it (events are transaction-wide and not operation-specific) - // but we keep it in order to make the API future-proof. - Op uint32 - // Event is the index of the event within in the operation which emitted the event. - Event uint32 -} - -// String returns a string representation of this cursor -func (c Cursor) String() string { - return fmt.Sprintf( - "%019d-%010d", - toid.New(int32(c.Ledger), int32(c.Tx), int32(c.Op)).ToInt64(), - c.Event, - ) -} - -// MarshalJSON marshals the cursor into JSON -func (c Cursor) MarshalJSON() ([]byte, error) { - return json.Marshal(c.String()) -} - -// UnmarshalJSON unmarshalls a cursor from the given JSON -func (c *Cursor) UnmarshalJSON(b []byte) error { - var s string - if err := json.Unmarshal(b, &s); err != nil { - return err - } - - if parsed, err := ParseCursor(s); err != nil { - return err - } else { - *c = parsed - } - return nil -} - -// ParseCursor parses the given string and returns the corresponding cursor -func ParseCursor(input string) (Cursor, error) { - parts := strings.SplitN(input, "-", 2) - if len(parts) != 2 { - return Cursor{}, fmt.Errorf("invalid event id %s", input) - } - - // Parse the first part (toid) - idInt, err := strconv.ParseInt(parts[0], 10, 64) //lint:ignore gomnd - if err != nil { - return Cursor{}, fmt.Errorf("invalid event id %s: %w", input, err) - } - parsed := toid.Parse(idInt) - - // Parse the second part (event order) - eventOrder, err := strconv.ParseUint(parts[1], 10, 32) //lint:ignore gomnd - if err != nil { - return Cursor{}, fmt.Errorf("invalid event id %s: %w", input, err) - } - - return Cursor{ - Ledger: uint32(parsed.LedgerSequence), - Tx: uint32(parsed.TransactionOrder), - Op: uint32(parsed.OperationOrder), - Event: uint32(eventOrder), - }, nil -} - -type ScanFunction func(xdr.DiagnosticEvent, Cursor, int64, *xdr.Hash) bool +type ScanFunction func(xdr.DiagnosticEvent, events.Cursor, int64, *xdr.Hash) bool // trimEvents removes all Events which fall outside the ledger retention window. func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWindow uint32) error { @@ -201,18 +123,19 @@ func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWi return err } -func (eventHandler *eventHandler) GetEvents(ctx context.Context, startLedgerSequence int, eventTypes []int, contractIds []string, f ScanFunction) error { +func (eventHandler *eventHandler) GetEvents(ctx context.Context, startCursor events.Cursor, eventTypes []int, contractIds []string, f ScanFunction) error { var rows []struct { - TxIndex int `db:"application_order"` - Lcm xdr.LedgerCloseMeta `db:"meta"` + EventCursorId string `db:"id"` + TxIndex int `db:"application_order"` + Lcm xdr.LedgerCloseMeta `db:"meta"` } rowQ := sq. - Select("e.application_order", "lcm.meta"). + Select("e.id", "e.application_order", "lcm.meta"). From(fmt.Sprintf("%s e", eventTableName)). Join(fmt.Sprintf("%s lcm ON (e.ledger_sequence = lcm.sequence)", ledgerCloseMetaTableName)). - Where(sq.GtOrEq{"e.ledger_sequence": startLedgerSequence}) + Where(sq.GtOrEq{"e.id": startCursor.String()}) if len(contractIds) > 0 { rowQ = rowQ.Where(sq.Eq{"e.contract_id": contractIds}) @@ -223,31 +146,32 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, startLedgerSequ } if err := eventHandler.db.Select(ctx, &rows, rowQ); err != nil { - return errors.Wrapf(err, "db read failed for startLedgerSequence= %d contractIds= %v eventType= %d ", startLedgerSequence, contractIds, eventTypes) + return errors.Wrapf(err, "db read failed for startLedgerSequence= %d contractIds= %v eventType= %d ", startCursor.Ledger, contractIds, eventTypes) } else if len(rows) < 1 { return errors.New("No LCM found with requested event filters") } for _, row := range rows { - txIndex, lcm := row.TxIndex, row.Lcm + eventCursorId, txIndex, lcm := row.EventCursorId, row.TxIndex, row.Lcm reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) reader.Seek(txIndex - 1) if err != nil { return errors.Wrapf(err, "failed to index to tx %d in ledger %d", txIndex, lcm.LedgerSequence()) } + ledgerCloseTime := lcm.LedgerCloseTime() ledgerTx, err := reader.Read() transactionHash := ledgerTx.Result.TransactionHash - events, diagErr := ledgerTx.GetDiagnosticEvents() + diagEvents, diagErr := ledgerTx.GetDiagnosticEvents() if diagErr != nil { - return errors.Wrapf(err, "db read failed for startLedgerSequence %d", startLedgerSequence) + return errors.Wrapf(err, "db read failed for Event Id %d", eventCursorId) } // Find events based on filter passed in function f - for eventIndex, event := range events { - cur := Cursor{lcm.LedgerSequence(), uint32(txIndex), 0, uint32(eventIndex)} + for eventIndex, event := range diagEvents { + cur := events.Cursor{lcm.LedgerSequence(), uint32(txIndex), 0, uint32(eventIndex)} if !f(event, cur, ledgerCloseTime, &transactionHash) { return nil } diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index 7620b2c7..be676660 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -2,6 +2,7 @@ package db import ( "context" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "io" "github.com/prometheus/client_golang/prometheus" @@ -48,7 +49,7 @@ func NewMockEventStore(passphrase string) *mockEventHandler { } } -func (txn *mockEventHandler) GetEvents(ctx context.Context, startLedgerSequence int, eventTypes []int, contractIds []string, f ScanFunction) error { +func (txn *mockEventHandler) GetEvents(ctx context.Context, start events.Cursor, eventTypes []int, contractIds []string, f ScanFunction) error { return nil } diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 8f01f06b..db6e9e68 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -314,6 +314,32 @@ type eventsRPCHandler struct { networkPassphrase string } +func combineFilters(filters []EventFilter) ([]int, []string) { + eventTypeIdSet := make(map[int]struct{}) + contractIDSet := make(map[string]struct{}) + + for _, filter := range filters { + for eventType := range filter.EventType { + eventTypeIdSet[eventTypeToInteger[eventType]] = struct{}{} + } + for _, contractID := range filter.ContractIDs { + contractIDSet[contractID] = struct{}{} + } + } + + contractIDs := make([]string, 0, len(contractIDSet)) + for contractID := range contractIDSet { + contractIDs = append(contractIDs, contractID) + } + + eventTypes := make([]int, 0, len(eventTypeIdSet)) + for eventType := range eventTypeIdSet { + eventTypes = append(eventTypes, eventType) + } + + return eventTypes, contractIDs +} + func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsRequest) (GetEventsResponse, error) { if err := request.Valid(h.maxLimit); err != nil { return GetEventsResponse{}, &jrpc2.Error{ @@ -337,63 +363,35 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } type entry struct { - cursor db.Cursor + cursor events.Cursor ledgerCloseTimestamp int64 event xdr.DiagnosticEvent txHash *xdr.Hash } var found []entry - if len(request.Filters) == 0 { - err := h.dbReader.GetEvents( - ctx, - int(request.StartLedger), - nil, - nil, - - func(event xdr.DiagnosticEvent, cursor db.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { - found = append(found, entry{cursor, ledgerCloseTimestamp, event, txHash}) - return uint(len(found)) < limit - }, - ) + eventTypeIds, contractIds := combineFilters(request.Filters) - if err != nil { - return GetEventsResponse{}, &jrpc2.Error{ - Code: jrpc2.InvalidRequest, - Message: err.Error(), - } + // Scan function to apply filters + f := func(event xdr.DiagnosticEvent, cursor events.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { + if request.Matches(event) && cursor.Cmp(start) >= 0 { + found = append(found, entry{cursor, ledgerCloseTimestamp, event, txHash}) } - + return uint(len(found)) < limit } - for _, filter := range request.Filters { - - var eventTypes []int - for key := range filter.EventType { - eventTypes = append(eventTypes, eventTypeToInteger[key]) - } - - // Scan function to apply filter and to remove duplicates if any - f := func(event xdr.DiagnosticEvent, cursor db.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { - if filter.Matches(event) { - found = append(found, entry{cursor, ledgerCloseTimestamp, event, txHash}) - } - return uint(len(found)) < limit - } - - err := h.dbReader.GetEvents( - ctx, - int(request.StartLedger), - eventTypes, - filter.ContractIDs, - f, - ) + err := h.dbReader.GetEvents( + ctx, + start, + eventTypeIds, + contractIds, + f, + ) - if err != nil { - return GetEventsResponse{}, &jrpc2.Error{ - Code: jrpc2.InvalidRequest, - Message: err.Error(), - } + if err != nil { + return GetEventsResponse{}, &jrpc2.Error{ + Code: jrpc2.InvalidRequest, + Message: err.Error(), } } @@ -417,7 +415,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques }, nil } -func eventInfoForEvent(event xdr.DiagnosticEvent, cursor db.Cursor, ledgerClosedAt string, txHash string) (EventInfo, error) { +func eventInfoForEvent(event xdr.DiagnosticEvent, cursor events.Cursor, ledgerClosedAt string, txHash string) (EventInfo, error) { v0, ok := event.Event.Body.GetV0() if !ok { return EventInfo{}, errors.New("unknown event version") From 2a6435d8e94e2c6b2b53bbd8173fda203c5f3282 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 17 Jun 2024 22:23:49 -0700 Subject: [PATCH 10/63] Update scan function logic in order to test --- cmd/soroban-rpc/internal/db/event.go | 4 ++-- cmd/soroban-rpc/internal/db/transaction_test.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index f5501733..f3ee4a72 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -166,13 +166,13 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, startCursor eve diagEvents, diagErr := ledgerTx.GetDiagnosticEvents() if diagErr != nil { - return errors.Wrapf(err, "db read failed for Event Id %d", eventCursorId) + return errors.Wrapf(err, "db read failed for Event Id %s", eventCursorId) } // Find events based on filter passed in function f for eventIndex, event := range diagEvents { cur := events.Cursor{lcm.LedgerSequence(), uint32(txIndex), 0, uint32(eventIndex)} - if !f(event, cur, ledgerCloseTime, &transactionHash) { + if f != nil && !f(event, cur, ledgerCloseTime, &transactionHash) { return nil } } diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index c7e21857..d0c822c1 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -3,6 +3,7 @@ package db import ( "context" "encoding/hex" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "math/rand" "testing" @@ -95,7 +96,7 @@ func TestTransactionFound(t *testing.T) { require.Error(t, err, ErrNoTransaction) eventReader := NewEventReader(log, db, passphrase) - err = eventReader.GetEvents(ctx, 1, nil, nil, nil) + err = eventReader.GetEvents(ctx, events.Cursor{1, 0, 0, 0}, nil, nil, nil) // check all 200 cases for _, lcm := range lcms { From 5c5b4071b35a3ada563c0c03b3927ad2da52d1c7 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 18 Jun 2024 15:02:40 -0700 Subject: [PATCH 11/63] Move NewTestDb util --- cmd/soroban-rpc/internal/db/db.go | 21 ++++++ cmd/soroban-rpc/internal/db/ledger_test.go | 19 ----- cmd/soroban-rpc/internal/db/mocks.go | 84 ++++++++++++++++++---- 3 files changed, 93 insertions(+), 31 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go index 3febcbe6..8ed58bce 100644 --- a/cmd/soroban-rpc/internal/db/db.go +++ b/cmd/soroban-rpc/internal/db/db.go @@ -9,8 +9,11 @@ import ( _ "github.com/mattn/go-sqlite3" "github.com/prometheus/client_golang/prometheus" migrate "github.com/rubenv/sql-migrate" + "github.com/stretchr/testify/assert" + "path" "strconv" "sync" + "testing" "github.com/stellar/go/support/db" "github.com/stellar/go/support/errors" @@ -341,3 +344,21 @@ func runMigrations(db *sql.DB, dialect string) error { _, err := migrate.ExecMax(db, dialect, m, migrate.Up, 0) return err } + +func NewTestDB(tb testing.TB) *DB { + tmp := tb.TempDir() + dbPath := path.Join(tmp, "test-db.sqlite") + db, err := OpenSQLiteDB(dbPath) + if err != nil { + assert.NoError(tb, db.Close()) + } + tb.Cleanup(func() { + assert.NoError(tb, db.Close()) + }) + return &DB{ + SessionInterface: db, + cache: dbCache{ + ledgerEntries: newTransactionalCache(), + }, + } +} diff --git a/cmd/soroban-rpc/internal/db/ledger_test.go b/cmd/soroban-rpc/internal/db/ledger_test.go index f6ebd70b..d28adc45 100644 --- a/cmd/soroban-rpc/internal/db/ledger_test.go +++ b/cmd/soroban-rpc/internal/db/ledger_test.go @@ -2,7 +2,6 @@ package db import ( "context" - "path" "testing" "github.com/stretchr/testify/assert" @@ -101,21 +100,3 @@ func TestLedgers(t *testing.T) { assertLedgerRange(t, reader, 8, 12) } - -func NewTestDB(tb testing.TB) *DB { - tmp := tb.TempDir() - dbPath := path.Join(tmp, "db.sqlite") - db, err := OpenSQLiteDB(dbPath) - if err != nil { - assert.NoError(tb, db.Close()) - } - tb.Cleanup(func() { - assert.NoError(tb, db.Close()) - }) - return &DB{ - SessionInterface: db, - cache: dbCache{ - ledgerEntries: newTransactionalCache(), - }, - } -} diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index be676660..0f0da8ad 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -32,28 +32,88 @@ func NewMockTransactionStore(passphrase string) *mockTransactionHandler { } type mockEventHandler struct { - passphrase string - - ledgerRange ledgerbucketwindow.LedgerRange - txs map[string]ingest.LedgerTransaction - txHashToMeta map[string]*xdr.LedgerCloseMeta - ledgerSeqToMeta map[uint32]*xdr.LedgerCloseMeta + passphrase string + ledgerRange ledgerbucketwindow.LedgerRange + contractIdToMeta map[string]ingest.LedgerTransaction + eventTypeToTx map[int]ingest.LedgerTransaction + eventIdToTx map[string]ingest.LedgerTransaction + ledgerSeqToMeta map[uint32]*xdr.LedgerCloseMeta } func NewMockEventStore(passphrase string) *mockEventHandler { return &mockEventHandler{ - passphrase: passphrase, - txs: make(map[string]ingest.LedgerTransaction), - txHashToMeta: make(map[string]*xdr.LedgerCloseMeta), - ledgerSeqToMeta: make(map[uint32]*xdr.LedgerCloseMeta), + passphrase: passphrase, + contractIdToMeta: make(map[string]ingest.LedgerTransaction), + eventTypeToTx: make(map[int]ingest.LedgerTransaction), + eventIdToTx: make(map[string]ingest.LedgerTransaction), + ledgerSeqToMeta: make(map[uint32]*xdr.LedgerCloseMeta), } } -func (txn *mockEventHandler) GetEvents(ctx context.Context, start events.Cursor, eventTypes []int, contractIds []string, f ScanFunction) error { +func (eventHandler *mockEventHandler) GetEvents(ctx context.Context, start events.Cursor, eventTypes []int, contractIds []string, f ScanFunction) error { + if contractIds != nil { + for _, contractId := range contractIds { + ledgerTx, ok := eventHandler.contractIdToMeta[contractId] + if ok { + diagEvents, diagErr := ledgerTx.GetDiagnosticEvents() + if diagErr != nil { + } + + for _, event := range diagEvents { + if !f(event, events.Cursor{0, 0, 0, 0}, 0, &ledgerTx.Result.TransactionHash) { + return nil + } + } + } + } + } return nil } -func (txn *mockEventHandler) IngestEvents(lcm xdr.LedgerCloseMeta) error { +func (eventHandler *mockEventHandler) IngestEvents(lcm xdr.LedgerCloseMeta) error { + eventHandler.ledgerSeqToMeta[lcm.LedgerSequence()] = &lcm + + reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) + if err != nil { + return err + } + + for { + tx, err := reader.Read() + if err == io.EOF { + break + } else if err != nil { + return err + } + + txEvents, err := tx.GetDiagnosticEvents() + if err != nil { + return err + } + + for index, e := range txEvents { + var contractId []byte + if e.Event.ContractId != nil { + contractId = e.Event.ContractId[:] + eventHandler.contractIdToMeta[string(contractId)] = tx + } + id := events.Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() + eventHandler.eventTypeToTx[int(e.Event.Type)] = tx + eventHandler.eventIdToTx[id] = tx + } + } + + if lcmSeq := lcm.LedgerSequence(); lcmSeq < eventHandler.ledgerRange.FirstLedger.Sequence || + eventHandler.ledgerRange.FirstLedger.Sequence == 0 { + eventHandler.ledgerRange.FirstLedger.Sequence = lcmSeq + eventHandler.ledgerRange.FirstLedger.CloseTime = lcm.LedgerCloseTime() + } + + if lcmSeq := lcm.LedgerSequence(); lcmSeq > eventHandler.ledgerRange.LastLedger.Sequence { + eventHandler.ledgerRange.LastLedger.Sequence = lcmSeq + eventHandler.ledgerRange.LastLedger.CloseTime = lcm.LedgerCloseTime() + } + return nil } From a566923b3f8d123b916877c97e4616841429c9c5 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 18 Jun 2024 15:51:25 -0700 Subject: [PATCH 12/63] Trim events db and remove eventTypes in SELECT query as event types are not indexed --- cmd/soroban-rpc/internal/db/db.go | 4 +++ cmd/soroban-rpc/internal/db/event.go | 10 ++----- cmd/soroban-rpc/internal/db/mocks.go | 2 +- .../internal/db/transaction_test.go | 2 +- .../internal/methods/get_events.go | 30 +++---------------- 5 files changed, 13 insertions(+), 35 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go index 8ed58bce..5a2fc100 100644 --- a/cmd/soroban-rpc/internal/db/db.go +++ b/cmd/soroban-rpc/internal/db/db.go @@ -283,6 +283,10 @@ func (w writeTx) Commit(ledgerSeq uint32) error { return err } + if err := w.eventWriter.trimEvents(ledgerSeq, w.ledgerRetentionWindow); err != nil { + return err + } + _, err := sq.Replace(metaTableName). Values(latestLedgerSequenceMetaKey, fmt.Sprintf("%d", ledgerSeq)). RunWith(w.stmtCache). diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index f3ee4a72..d1516b24 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -23,7 +23,7 @@ type EventWriter interface { // EventReader has all the public methods to fetch events from DB type EventReader interface { - GetEvents(ctx context.Context, startCursor events.Cursor, eventTypes []int, contractIds []string, f ScanFunction) error + GetEvents(ctx context.Context, startCursor events.Cursor, contractIds []string, f ScanFunction) error } type eventHandler struct { @@ -123,7 +123,7 @@ func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWi return err } -func (eventHandler *eventHandler) GetEvents(ctx context.Context, startCursor events.Cursor, eventTypes []int, contractIds []string, f ScanFunction) error { +func (eventHandler *eventHandler) GetEvents(ctx context.Context, startCursor events.Cursor, contractIds []string, f ScanFunction) error { var rows []struct { EventCursorId string `db:"id"` @@ -141,12 +141,8 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, startCursor eve rowQ = rowQ.Where(sq.Eq{"e.contract_id": contractIds}) } - if len(eventTypes) > 0 { - rowQ = rowQ.Where(sq.Eq{"e.event_type": eventTypes}) - } - if err := eventHandler.db.Select(ctx, &rows, rowQ); err != nil { - return errors.Wrapf(err, "db read failed for startLedgerSequence= %d contractIds= %v eventType= %d ", startCursor.Ledger, contractIds, eventTypes) + return errors.Wrapf(err, "db read failed for startLedgerSequence= %d contractIds= %v", startCursor.Ledger, contractIds) } else if len(rows) < 1 { return errors.New("No LCM found with requested event filters") } diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index 0f0da8ad..fb692c24 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -50,7 +50,7 @@ func NewMockEventStore(passphrase string) *mockEventHandler { } } -func (eventHandler *mockEventHandler) GetEvents(ctx context.Context, start events.Cursor, eventTypes []int, contractIds []string, f ScanFunction) error { +func (eventHandler *mockEventHandler) GetEvents(ctx context.Context, startCursor events.Cursor, contractIds []string, f ScanFunction) error { if contractIds != nil { for _, contractId := range contractIds { ledgerTx, ok := eventHandler.contractIdToMeta[contractId] diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index d0c822c1..fae79de1 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -96,7 +96,7 @@ func TestTransactionFound(t *testing.T) { require.Error(t, err, ErrNoTransaction) eventReader := NewEventReader(log, db, passphrase) - err = eventReader.GetEvents(ctx, events.Cursor{1, 0, 0, 0}, nil, nil, nil) + err = eventReader.GetEvents(ctx, events.Cursor{1, 0, 0, 0}, nil, nil) // check all 200 cases for _, lcm := range lcms { diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index db6e9e68..d3a0dc38 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -135,12 +135,6 @@ var eventTypeFromXDR = map[xdr.ContractEventType]string{ xdr.ContractEventTypeDiagnostic: EventTypeDiagnostic, } -var eventTypeToInteger = map[string]int{ - EventTypeSystem: 1, - EventTypeContract: 2, - EventTypeDiagnostic: 3, -} - type EventFilter struct { EventType eventTypeSet `json:"type,omitempty"` ContractIDs []string `json:"contractIds,omitempty"` @@ -314,14 +308,10 @@ type eventsRPCHandler struct { networkPassphrase string } -func combineFilters(filters []EventFilter) ([]int, []string) { - eventTypeIdSet := make(map[int]struct{}) +func combineContractIds(filters []EventFilter) []string { contractIDSet := make(map[string]struct{}) for _, filter := range filters { - for eventType := range filter.EventType { - eventTypeIdSet[eventTypeToInteger[eventType]] = struct{}{} - } for _, contractID := range filter.ContractIDs { contractIDSet[contractID] = struct{}{} } @@ -331,13 +321,7 @@ func combineFilters(filters []EventFilter) ([]int, []string) { for contractID := range contractIDSet { contractIDs = append(contractIDs, contractID) } - - eventTypes := make([]int, 0, len(eventTypeIdSet)) - for eventType := range eventTypeIdSet { - eventTypes = append(eventTypes, eventType) - } - - return eventTypes, contractIDs + return contractIDs } func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsRequest) (GetEventsResponse, error) { @@ -370,7 +354,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } var found []entry - eventTypeIds, contractIds := combineFilters(request.Filters) + contractIds := combineContractIds(request.Filters) // Scan function to apply filters f := func(event xdr.DiagnosticEvent, cursor events.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { @@ -380,13 +364,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques return uint(len(found)) < limit } - err := h.dbReader.GetEvents( - ctx, - start, - eventTypeIds, - contractIds, - f, - ) + err := h.dbReader.GetEvents(ctx, start, contractIds, f) if err != nil { return GetEventsResponse{}, &jrpc2.Error{ From c70c9e412f13221f1859b5cf44baab3e867d03cd Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 18 Jun 2024 16:58:28 -0700 Subject: [PATCH 13/63] Introduce cursor range and add logs for latency --- cmd/soroban-rpc/internal/db/event.go | 18 ++++++++++++++---- cmd/soroban-rpc/internal/db/mocks.go | 2 +- .../internal/db/transaction_test.go | 6 +++++- cmd/soroban-rpc/internal/events/cursor.go | 7 +++++++ cmd/soroban-rpc/internal/methods/get_events.go | 5 ++++- 5 files changed, 31 insertions(+), 7 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index d1516b24..0c8cd365 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -12,6 +12,7 @@ import ( "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "io" + "time" ) const eventTableName = "events" @@ -23,7 +24,7 @@ type EventWriter interface { // EventReader has all the public methods to fetch events from DB type EventReader interface { - GetEvents(ctx context.Context, startCursor events.Cursor, contractIds []string, f ScanFunction) error + GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIds []string, f ScanFunction) error } type eventHandler struct { @@ -123,7 +124,9 @@ func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWi return err } -func (eventHandler *eventHandler) GetEvents(ctx context.Context, startCursor events.Cursor, contractIds []string, f ScanFunction) error { +func (eventHandler *eventHandler) GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIds []string, f ScanFunction) error { + + start := time.Now() var rows []struct { EventCursorId string `db:"id"` @@ -135,14 +138,15 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, startCursor eve Select("e.id", "e.application_order", "lcm.meta"). From(fmt.Sprintf("%s e", eventTableName)). Join(fmt.Sprintf("%s lcm ON (e.ledger_sequence = lcm.sequence)", ledgerCloseMetaTableName)). - Where(sq.GtOrEq{"e.id": startCursor.String()}) + Where(sq.GtOrEq{"e.id": cursorRange.Start.String()}). + Where(sq.Lt{"e.id": cursorRange.End.String()}) if len(contractIds) > 0 { rowQ = rowQ.Where(sq.Eq{"e.contract_id": contractIds}) } if err := eventHandler.db.Select(ctx, &rows, rowQ); err != nil { - return errors.Wrapf(err, "db read failed for startLedgerSequence= %d contractIds= %v", startCursor.Ledger, contractIds) + return errors.Wrapf(err, "db read failed for start ledger cursor= %v contractIds= %v", cursorRange.Start.String(), contractIds) } else if len(rows) < 1 { return errors.New("No LCM found with requested event filters") } @@ -174,5 +178,11 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, startCursor eve } } + eventHandler.log. + WithField("startLedgerSequence", cursorRange.Start.Ledger). + WithField("endLedgerSequence", cursorRange.End.Ledger). + WithField("duration", time.Since(start)). + Debugf("Fetched and decoded all the events with filters - contractIds: %v ", contractIds) + return nil } diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index fb692c24..e1d95570 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -50,7 +50,7 @@ func NewMockEventStore(passphrase string) *mockEventHandler { } } -func (eventHandler *mockEventHandler) GetEvents(ctx context.Context, startCursor events.Cursor, contractIds []string, f ScanFunction) error { +func (eventHandler *mockEventHandler) GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIds []string, f ScanFunction) error { if contractIds != nil { for _, contractId := range contractIds { ledgerTx, ok := eventHandler.contractIdToMeta[contractId] diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index fae79de1..836c466b 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -96,7 +96,11 @@ func TestTransactionFound(t *testing.T) { require.Error(t, err, ErrNoTransaction) eventReader := NewEventReader(log, db, passphrase) - err = eventReader.GetEvents(ctx, events.Cursor{1, 0, 0, 0}, nil, nil) + start := events.Cursor{Ledger: 1} + end := events.Cursor{Ledger: 1000} + cursorRange := events.CursorRange{Start: start, End: end} + + err = eventReader.GetEvents(ctx, cursorRange, nil, nil) // check all 200 cases for _, lcm := range lcms { diff --git a/cmd/soroban-rpc/internal/events/cursor.go b/cmd/soroban-rpc/internal/events/cursor.go index 3fbfbecb..e42a3295 100644 --- a/cmd/soroban-rpc/internal/events/cursor.go +++ b/cmd/soroban-rpc/internal/events/cursor.go @@ -27,6 +27,13 @@ type Cursor struct { Event uint32 } +type CursorRange struct { + // Start defines the (inclusive) start of the range. + Start Cursor + // End defines the (exclusive) end of the range. + End Cursor +} + // String returns a string representation of this cursor func (c Cursor) String() string { return fmt.Sprintf( diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index d3a0dc38..d96fb50c 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/stellar/go/support/log" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" "strings" "time" @@ -345,6 +346,8 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques limit = request.Pagination.Limit } } + end := events.Cursor{Ledger: uint32(request.StartLedger + ledgerbucketwindow.OneDayOfLedgers)} + cursorRange := events.CursorRange{Start: start, End: end} type entry struct { cursor events.Cursor @@ -364,7 +367,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques return uint(len(found)) < limit } - err := h.dbReader.GetEvents(ctx, start, contractIds, f) + err := h.dbReader.GetEvents(ctx, cursorRange, contractIds, f) if err != nil { return GetEventsResponse{}, &jrpc2.Error{ From e86cebcec5a16540d63270a0d478c34a51bb9fc1 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Thu, 20 Jun 2024 08:10:22 -0700 Subject: [PATCH 14/63] remove event memory store --- cmd/soroban-rpc/internal/db/event.go | 8 +- cmd/soroban-rpc/internal/events/events.go | 100 ---------------------- 2 files changed, 7 insertions(+), 101 deletions(-) delete mode 100644 cmd/soroban-rpc/internal/events/events.go diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 0c8cd365..42f09351 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -25,6 +25,7 @@ type EventWriter interface { // EventReader has all the public methods to fetch events from DB type EventReader interface { GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIds []string, f ScanFunction) error + //GetLedgerRange(ctx context.Context) error } type eventHandler struct { @@ -124,6 +125,10 @@ func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWi return err } +// GetEvents applies f on all the events occurring in the given range with specified contract IDs if provided. +// The events are returned in sorted ascending Cursor order. +// If f returns false, the scan terminates early (f will not be applied on +// remaining events in the range). func (eventHandler *eventHandler) GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIds []string, f ScanFunction) error { start := time.Now() @@ -139,7 +144,8 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, cursorRange eve From(fmt.Sprintf("%s e", eventTableName)). Join(fmt.Sprintf("%s lcm ON (e.ledger_sequence = lcm.sequence)", ledgerCloseMetaTableName)). Where(sq.GtOrEq{"e.id": cursorRange.Start.String()}). - Where(sq.Lt{"e.id": cursorRange.End.String()}) + Where(sq.Lt{"e.id": cursorRange.End.String()}). + OrderBy("e.id ASC") if len(contractIds) > 0 { rowQ = rowQ.Where(sq.Eq{"e.contract_id": contractIds}) diff --git a/cmd/soroban-rpc/internal/events/events.go b/cmd/soroban-rpc/internal/events/events.go deleted file mode 100644 index 931d69a0..00000000 --- a/cmd/soroban-rpc/internal/events/events.go +++ /dev/null @@ -1,100 +0,0 @@ -package events - -import ( - "errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/stellar/go/xdr" - "sync" - - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" -) - -type event struct { - diagnosticEventXDR []byte - txIndex uint32 - eventIndex uint32 - txHash *xdr.Hash // intentionally stored as a pointer to save memory (amortized as soon as there are two events in a transaction) -} - -func (e event) cursor(ledgerSeq uint32) Cursor { - return Cursor{ - Ledger: ledgerSeq, - Tx: e.txIndex, - Event: e.eventIndex, - } -} - -// MemoryStore is an in-memory store of soroban events. -type MemoryStore struct { - // networkPassphrase is an immutable string containing the - // Stellar network passphrase. - // Accessing networkPassphrase does not need to be protected - // by the lock - networkPassphrase string - // lock protects the mutable fields below - lock sync.RWMutex - eventsByLedger *ledgerbucketwindow.LedgerBucketWindow[[]event] - eventsDurationMetric *prometheus.SummaryVec - eventCountMetric prometheus.Summary -} - -// Range defines a [Start, End) interval of Soroban events. -type Range struct { - // Start defines the (inclusive) start of the range. - Start Cursor - // ClampStart indicates whether Start should be clamped up - // to the earliest ledger available if Start is too low. - ClampStart bool - // End defines the (exclusive) end of the range. - End Cursor - // ClampEnd indicates whether End should be clamped down - // to the latest ledger available if End is too high. - ClampEnd bool -} - -type ScanFunction func(xdr.DiagnosticEvent, Cursor, int64, *xdr.Hash) bool - -// Scan applies f on all the events occurring in the given range. -// The events are processed in sorted ascending Cursor order. -// If f returns false, the scan terminates early (f will not be applied on -// remaining events in the range). Note that a read lock is held for the -// entire duration of the Scan function so f should be written in a way -// to minimize latency. -func (m *MemoryStore) Scan(eventRange Range, f ScanFunction) (lastLedgerInWindow uint32, err error) { - return -} - -// validateRange checks if the range falls within the bounds -// of the events in the memory store. -// validateRange should be called with the read lock. -func (m *MemoryStore) validateRange(eventRange *Range) error { - if m.eventsByLedger.Len() == 0 { - return errors.New("event store is empty") - } - firstBucket := m.eventsByLedger.Get(0) - min := Cursor{Ledger: firstBucket.LedgerSeq} - if eventRange.Start.Cmp(min) < 0 { - if eventRange.ClampStart { - eventRange.Start = min - } else { - return errors.New("start is before oldest ledger") - } - } - max := Cursor{Ledger: min.Ledger + m.eventsByLedger.Len()} - if eventRange.Start.Cmp(max) >= 0 { - return errors.New("start is after newest ledger") - } - if eventRange.End.Cmp(max) > 0 { - if eventRange.ClampEnd { - eventRange.End = max - } else { - return errors.New("end is after latest ledger") - } - } - - if eventRange.Start.Cmp(eventRange.End) >= 0 { - return errors.New("start is not before end") - } - - return nil -} From a3f89872408aa1b691b51c29e694fac57c93f312 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Thu, 20 Jun 2024 12:28:23 -0700 Subject: [PATCH 15/63] remove event memory store from latest merge --- cmd/soroban-rpc/internal/daemon/daemon.go | 16 +- cmd/soroban-rpc/internal/db/db.go | 14 +- cmd/soroban-rpc/internal/db/ledger_test.go | 1 - cmd/soroban-rpc/internal/events/events.go | 0 .../internal/events/events_test.go | 407 ------------------ 5 files changed, 9 insertions(+), 429 deletions(-) delete mode 100644 cmd/soroban-rpc/internal/events/events.go delete mode 100644 cmd/soroban-rpc/internal/events/events_test.go diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index a80124a4..7e029d33 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -200,7 +200,7 @@ func MustNew(cfg *config.Config, logger *supportlog.Entry) *Daemon { }, metricsRegistry), } - feewindows, eventStore := daemon.mustInitializeStorage(cfg) + feewindows := daemon.mustInitializeStorage(cfg) onIngestionRetry := func(err error, dur time.Duration) { logger.WithError(err).Error("could not run ingestion. Retrying") @@ -299,12 +299,8 @@ func MustNew(cfg *config.Config, logger *supportlog.Entry) *Daemon { } // mustInitializeStorage initializes the storage using what was on the DB -func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindows, *events.MemoryStore) { - eventStore := events.NewMemoryStore( - d, - cfg.NetworkPassphrase, - cfg.EventLedgerRetentionWindow, - ) +func (d *Daemon) mustInitializeStorage(cfg *config.Config) *feewindow.FeeWindows { + feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase) readTxMetaCtx, cancelReadTxMeta := context.WithTimeout(context.Background(), cfg.IngestionTimeout) @@ -330,9 +326,7 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow "seq": currentSeq, }).Debug("still initializing in-memory store") } - if err := eventStore.IngestEvents(txmeta); err != nil { - d.logger.WithError(err).Fatal("could not initialize event memory store") - } + if err := feewindows.IngestFees(txmeta); err != nil { d.logger.WithError(err).Fatal("could not initialize fee stats") } @@ -358,7 +352,7 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) (*feewindow.FeeWindow }).Info("finished initializing in-memory store") } - return feewindows, eventStore + return feewindows } func (d *Daemon) Run() { diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go index 8007f062..3ff55517 100644 --- a/cmd/soroban-rpc/internal/db/db.go +++ b/cmd/soroban-rpc/internal/db/db.go @@ -11,6 +11,7 @@ import ( "github.com/prometheus/client_golang/prometheus" migrate "github.com/rubenv/sql-migrate" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "path" "strconv" "sync" @@ -375,18 +376,11 @@ func runSQLMigrations(db *sql.DB, dialect string) error { func NewTestDB(tb testing.TB) *DB { tmp := tb.TempDir() - dbPath := path.Join(tmp, "test-db.sqlite") + dbPath := path.Join(tmp, "db.sqlite") db, err := OpenSQLiteDB(dbPath) - if err != nil { - assert.NoError(tb, db.Close()) - } + require.NoError(tb, err) tb.Cleanup(func() { assert.NoError(tb, db.Close()) }) - return &DB{ - SessionInterface: db, - cache: dbCache{ - ledgerEntries: newTransactionalCache(), - }, - } + return db } diff --git a/cmd/soroban-rpc/internal/db/ledger_test.go b/cmd/soroban-rpc/internal/db/ledger_test.go index b484e630..2578dcf8 100644 --- a/cmd/soroban-rpc/internal/db/ledger_test.go +++ b/cmd/soroban-rpc/internal/db/ledger_test.go @@ -5,7 +5,6 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "github.com/stellar/go/network" "github.com/stellar/go/support/log" diff --git a/cmd/soroban-rpc/internal/events/events.go b/cmd/soroban-rpc/internal/events/events.go deleted file mode 100644 index e69de29b..00000000 diff --git a/cmd/soroban-rpc/internal/events/events_test.go b/cmd/soroban-rpc/internal/events/events_test.go deleted file mode 100644 index c5fda34c..00000000 --- a/cmd/soroban-rpc/internal/events/events_test.go +++ /dev/null @@ -1,407 +0,0 @@ -package events - -import ( - "bytes" - "testing" - - "github.com/prometheus/client_golang/prometheus" - dto "github.com/prometheus/client_model/go" - "github.com/stellar/go/xdr" - "github.com/stretchr/testify/require" - - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" -) - -var ( - ledger5CloseTime = ledgerCloseTime(5) - ledger5Events = []event{ - newEvent(1, 0, 100), - newEvent(1, 1, 200), - newEvent(2, 0, 300), - newEvent(2, 1, 400), - } - ledger6CloseTime = ledgerCloseTime(6) - ledger6Events []event = nil - ledger7CloseTime = ledgerCloseTime(7) - ledger7Events = []event{ - newEvent(1, 0, 500), - } - ledger8CloseTime = ledgerCloseTime(8) - ledger8Events = []event{ - newEvent(1, 0, 600), - newEvent(2, 0, 700), - newEvent(2, 1, 800), - newEvent(2, 2, 900), - newEvent(2, 3, 1000), - } -) - -func ledgerCloseTime(seq uint32) int64 { - return int64(seq)*25 + 100 -} - -func newEvent(txIndex, eventIndex, val uint32) event { - v := xdr.Uint32(val) - - e := xdr.DiagnosticEvent{ - InSuccessfulContractCall: true, - Event: xdr.ContractEvent{ - Type: xdr.ContractEventTypeSystem, - Body: xdr.ContractEventBody{ - V: 0, - V0: &xdr.ContractEventV0{ - Data: xdr.ScVal{ - Type: xdr.ScValTypeScvU32, - U32: &v, - }, - }, - }, - }, - } - diagnosticEventXDR, err := e.MarshalBinary() - if err != nil { - panic(err) - } - return event{ - diagnosticEventXDR: diagnosticEventXDR, - txIndex: txIndex, - eventIndex: eventIndex, - } -} - -func (e event) equals(other event) bool { - return e.txIndex == other.txIndex && - e.eventIndex == other.eventIndex && - bytes.Equal(e.diagnosticEventXDR, other.diagnosticEventXDR) -} - -func eventsAreEqual(t *testing.T, a, b []event) { - require.Equal(t, len(a), len(b)) - for i := range a { - require.True(t, a[i].equals(b[i])) - } -} - -func TestScanRangeValidation(t *testing.T) { - m := NewMemoryStore(interfaces.MakeNoOpDeamon(), "unit-tests", 4) - assertNoCalls := func(xdr.DiagnosticEvent, Cursor, int64, *xdr.Hash) bool { - t.Fatalf("unexpected call") - return true - } - _, err := m.Scan(Range{ - Start: MinCursor, - ClampStart: true, - End: MaxCursor, - ClampEnd: true, - }, assertNoCalls) - require.EqualError(t, err, "event store is empty") - - m = createStore(t) - - for _, testCase := range []struct { - input Range - err string - }{ - { - Range{ - Start: MinCursor, - ClampStart: false, - End: MaxCursor, - ClampEnd: true, - }, - "start is before oldest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 4}, - ClampStart: false, - End: MaxCursor, - ClampEnd: true, - }, - "start is before oldest ledger", - }, - { - Range{ - Start: MinCursor, - ClampStart: true, - End: MaxCursor, - ClampEnd: false, - }, - "end is after latest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 5}, - ClampStart: true, - End: Cursor{Ledger: 10}, - ClampEnd: false, - }, - "end is after latest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 10}, - ClampStart: true, - End: Cursor{Ledger: 3}, - ClampEnd: true, - }, - "start is after newest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 10}, - ClampStart: false, - End: Cursor{Ledger: 3}, - ClampEnd: false, - }, - "start is after newest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 9}, - ClampStart: false, - End: Cursor{Ledger: 10}, - ClampEnd: true, - }, - "start is after newest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 9}, - ClampStart: false, - End: Cursor{Ledger: 10}, - ClampEnd: false, - }, - "start is after newest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 2}, - ClampStart: true, - End: Cursor{Ledger: 3}, - ClampEnd: false, - }, - "start is not before end", - }, - { - Range{ - Start: Cursor{Ledger: 2}, - ClampStart: false, - End: Cursor{Ledger: 3}, - ClampEnd: false, - }, - "start is before oldest ledger", - }, - { - Range{ - Start: Cursor{Ledger: 6}, - ClampStart: false, - End: Cursor{Ledger: 6}, - ClampEnd: false, - }, - "start is not before end", - }, - } { - _, err := m.Scan(testCase.input, assertNoCalls) - require.EqualError(t, err, testCase.err, testCase.input) - } -} - -func createStore(t *testing.T) *MemoryStore { - m := NewMemoryStore(interfaces.MakeNoOpDeamon(), "unit-tests", 4) - m.eventsByLedger.Append(ledgerbucketwindow.LedgerBucket[[]event]{ - LedgerSeq: 5, - LedgerCloseTimestamp: ledger5CloseTime, - BucketContent: ledger5Events, - }) - m.eventsByLedger.Append(ledgerbucketwindow.LedgerBucket[[]event]{ - LedgerSeq: 6, - LedgerCloseTimestamp: ledger6CloseTime, - BucketContent: nil, - }) - m.eventsByLedger.Append(ledgerbucketwindow.LedgerBucket[[]event]{ - LedgerSeq: 7, - LedgerCloseTimestamp: ledger7CloseTime, - BucketContent: ledger7Events, - }) - m.eventsByLedger.Append(ledgerbucketwindow.LedgerBucket[[]event]{ - LedgerSeq: 8, - LedgerCloseTimestamp: ledger8CloseTime, - BucketContent: ledger8Events, - }) - - return m -} - -func concat(slices ...[]event) []event { - var result []event - for _, slice := range slices { - result = append(result, slice...) - } - return result -} - -func getMetricValue(metric prometheus.Metric) *dto.Metric { - value := &dto.Metric{} - err := metric.Write(value) - if err != nil { - panic(err) - } - return value -} - -func TestScan(t *testing.T) { - genEquivalentInputs := func(input Range) []Range { - results := []Range{input} - if !input.ClampStart { - rangeCopy := input - rangeCopy.ClampStart = true - results = append(results, rangeCopy) - } - if !input.ClampEnd { - rangeCopy := input - rangeCopy.ClampEnd = true - results = append(results, rangeCopy) - } - if !input.ClampStart && !input.ClampEnd { - rangeCopy := input - rangeCopy.ClampStart = true - rangeCopy.ClampEnd = true - results = append(results, rangeCopy) - } - return results - } - - for _, testCase := range []struct { - input Range - expected []event - }{ - { - Range{ - Start: MinCursor, - ClampStart: true, - End: MaxCursor, - ClampEnd: true, - }, - concat(ledger5Events, ledger6Events, ledger7Events, ledger8Events), - }, - { - Range{ - Start: Cursor{Ledger: 5}, - ClampStart: false, - End: Cursor{Ledger: 9}, - ClampEnd: false, - }, - concat(ledger5Events, ledger6Events, ledger7Events, ledger8Events), - }, - { - Range{ - Start: Cursor{Ledger: 5, Tx: 2}, - ClampStart: false, - End: Cursor{Ledger: 9}, - ClampEnd: false, - }, - concat(ledger5Events[2:], ledger6Events, ledger7Events, ledger8Events), - }, - { - Range{ - Start: Cursor{Ledger: 5, Tx: 3}, - ClampStart: false, - End: MaxCursor, - ClampEnd: true, - }, - concat(ledger6Events, ledger7Events, ledger8Events), - }, - { - Range{ - Start: Cursor{Ledger: 6}, - ClampStart: false, - End: MaxCursor, - ClampEnd: true, - }, - concat(ledger7Events, ledger8Events), - }, - { - Range{ - Start: Cursor{Ledger: 6, Tx: 1}, - ClampStart: false, - End: MaxCursor, - ClampEnd: true, - }, - concat(ledger7Events, ledger8Events), - }, - { - Range{ - Start: Cursor{Ledger: 8, Tx: 2, Event: 3}, - ClampStart: false, - End: MaxCursor, - ClampEnd: true, - }, - ledger8Events[len(ledger8Events)-1:], - }, - { - Range{ - Start: Cursor{Ledger: 8, Tx: 2, Event: 3}, - ClampStart: false, - End: Cursor{Ledger: 9}, - ClampEnd: false, - }, - ledger8Events[len(ledger8Events)-1:], - }, - { - Range{ - Start: Cursor{Ledger: 5}, - ClampStart: false, - End: Cursor{Ledger: 7}, - ClampEnd: false, - }, - concat(ledger5Events, ledger6Events), - }, - { - Range{ - Start: Cursor{Ledger: 5, Tx: 2}, - ClampStart: false, - End: Cursor{Ledger: 8, Tx: 2}, - ClampEnd: false, - }, - concat(ledger5Events[2:], ledger6Events, ledger7Events, ledger8Events[:1]), - }, - } { - for _, input := range genEquivalentInputs(testCase.input) { - m := createStore(t) - var events []event - iterateAll := true - f := func(contractEvent xdr.DiagnosticEvent, cursor Cursor, ledgerCloseTimestamp int64, hash *xdr.Hash) bool { - require.Equal(t, ledgerCloseTime(cursor.Ledger), ledgerCloseTimestamp) - diagnosticEventXDR, err := contractEvent.MarshalBinary() - require.NoError(t, err) - events = append(events, event{ - diagnosticEventXDR: diagnosticEventXDR, - txIndex: cursor.Tx, - eventIndex: cursor.Event, - txHash: hash, - }) - return iterateAll - } - latest, err := m.Scan(input, f) - require.NoError(t, err) - require.Equal(t, uint32(8), latest) - eventsAreEqual(t, testCase.expected, events) - metric, err := m.eventsDurationMetric.MetricVec.GetMetricWith(prometheus.Labels{ - "operation": "scan", - }) - require.NoError(t, err) - require.Equal(t, uint64(1), getMetricValue(metric).GetSummary().GetSampleCount()) - if len(events) > 0 { - events = nil - iterateAll = false - latest, err := m.Scan(input, f) - require.NoError(t, err) - require.Equal(t, uint64(2), getMetricValue(metric).GetSummary().GetSampleCount()) - require.Equal(t, uint32(8), latest) - eventsAreEqual(t, []event{testCase.expected[0]}, events) - } - } - } -} From 1a03730bc42e94687fa73fc94804d8e652f9870a Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Thu, 20 Jun 2024 14:21:05 -0700 Subject: [PATCH 16/63] Fix lint issues part 1 --- cmd/soroban-rpc/internal/daemon/daemon.go | 2 +- cmd/soroban-rpc/internal/db/event.go | 33 ++++++++++--------- cmd/soroban-rpc/internal/db/mocks.go | 8 ++--- .../internal/db/transaction_test.go | 5 +-- .../internal/methods/get_events.go | 14 ++++---- 5 files changed, 33 insertions(+), 29 deletions(-) diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index 7e029d33..363a5e8b 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -206,7 +206,7 @@ func MustNew(cfg *config.Config, logger *supportlog.Entry) *Daemon { logger.WithError(err).Error("could not run ingestion. Retrying") } - // Take the larger of (event retention, tx retention) and then the smaller + // Take the largest of (event retention, tx retention) and then the smallest // of (tx retention, default event retention) if event retention wasn't // specified, for some reason...? maxRetentionWindow := ordered.Max(cfg.EventLedgerRetentionWindow, cfg.TransactionLedgerRetentionWindow) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 42f09351..45c5aa3c 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -3,6 +3,9 @@ package db import ( "context" "fmt" + "io" + "time" + sq "github.com/Masterminds/squirrel" "github.com/prometheus/client_golang/prometheus" "github.com/stellar/go/ingest" @@ -11,8 +14,6 @@ import ( "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" - "io" - "time" ) const eventTableName = "events" @@ -25,7 +26,7 @@ type EventWriter interface { // EventReader has all the public methods to fetch events from DB type EventReader interface { GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIds []string, f ScanFunction) error - //GetLedgerRange(ctx context.Context) error + // GetLedgerRange(ctx context.Context) error } type eventHandler struct { @@ -44,7 +45,7 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { txCount := lcm.CountTransactions() if eventHandler.stmtCache == nil { - return errors.New("EventWriter incorrectly initialized without stmtCache") + return fmt.Errorf("EventWriter incorrectly initialized without stmtCache") } else if txCount == 0 { return nil } @@ -52,9 +53,9 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { var txReader *ingest.LedgerTransactionReader txReader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) if err != nil { - return errors.Wrapf(err, - "failed to open transaction reader for ledger %d", - lcm.LedgerSequence()) + return fmt.Errorf( + "failed to open transaction reader for ledger %d: %w ", + lcm.LedgerSequence(), err) } defer func() { closeErr := txReader.Close() @@ -130,7 +131,6 @@ func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWi // If f returns false, the scan terminates early (f will not be applied on // remaining events in the range). func (eventHandler *eventHandler) GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIds []string, f ScanFunction) error { - start := time.Now() var rows []struct { @@ -141,8 +141,8 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, cursorRange eve rowQ := sq. Select("e.id", "e.application_order", "lcm.meta"). - From(fmt.Sprintf("%s e", eventTableName)). - Join(fmt.Sprintf("%s lcm ON (e.ledger_sequence = lcm.sequence)", ledgerCloseMetaTableName)). + From(eventTableName + " e"). + Join(ledgerCloseMetaTableName + "lcm ON (e.ledger_sequence = lcm.sequence)"). Where(sq.GtOrEq{"e.id": cursorRange.Start.String()}). Where(sq.Lt{"e.id": cursorRange.End.String()}). OrderBy("e.id ASC") @@ -152,7 +152,7 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, cursorRange eve } if err := eventHandler.db.Select(ctx, &rows, rowQ); err != nil { - return errors.Wrapf(err, "db read failed for start ledger cursor= %v contractIds= %v", cursorRange.Start.String(), contractIds) + return fmt.Errorf("db read failed for start ledger cursor= %v contractIds= %v: %w", cursorRange.Start.String(), contractIds, err) } else if len(rows) < 1 { return errors.New("No LCM found with requested event filters") } @@ -160,10 +160,13 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, cursorRange eve for _, row := range rows { eventCursorId, txIndex, lcm := row.EventCursorId, row.TxIndex, row.Lcm reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) - reader.Seek(txIndex - 1) + if err != nil { + return fmt.Errorf("failed to index to tx %d in ledger %d: %w", txIndex, lcm.LedgerSequence(), err) + } + err = reader.Seek(txIndex - 1) if err != nil { - return errors.Wrapf(err, "failed to index to tx %d in ledger %d", txIndex, lcm.LedgerSequence()) + return fmt.Errorf("failed to index to tx %d in ledger %d: %w", txIndex, lcm.LedgerSequence(), err) } ledgerCloseTime := lcm.LedgerCloseTime() @@ -172,12 +175,12 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, cursorRange eve diagEvents, diagErr := ledgerTx.GetDiagnosticEvents() if diagErr != nil { - return errors.Wrapf(err, "db read failed for Event Id %s", eventCursorId) + return fmt.Errorf("db read failed for Event Id %s: %w", eventCursorId, err) } // Find events based on filter passed in function f for eventIndex, event := range diagEvents { - cur := events.Cursor{lcm.LedgerSequence(), uint32(txIndex), 0, uint32(eventIndex)} + cur := events.Cursor{Ledger: lcm.LedgerSequence(), Tx: uint32(txIndex), Event: uint32(eventIndex)} if f != nil && !f(event, cur, ledgerCloseTime, &transactionHash) { return nil } diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index e01a2173..5b9ee6d8 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -2,9 +2,10 @@ package db import ( "context" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "io" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" + "github.com/prometheus/client_golang/prometheus" "github.com/stellar/go/ingest" @@ -60,7 +61,7 @@ func (eventHandler *mockEventHandler) GetEvents(ctx context.Context, cursorRange } for _, event := range diagEvents { - if !f(event, events.Cursor{0, 0, 0, 0}, 0, &ledgerTx.Result.TransactionHash) { + if !f(event, events.Cursor{}, 0, &ledgerTx.Result.TransactionHash) { return nil } } @@ -192,8 +193,7 @@ func (m *mockLedgerReader) StreamAllLedgers(ctx context.Context, f StreamLedgerF return nil } -type mockEventReader struct { -} +type mockEventReader struct{} func NewMockEventReader() { } diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 1bc38d23..8fe89696 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -3,10 +3,11 @@ package db import ( "context" "encoding/hex" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "math/rand" "testing" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" + "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -29,7 +30,6 @@ func TestTransactionNotFound(t *testing.T) { } func txMetaWithEvents(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { - meta := txMeta(acctSeq, successful) contractIDBytes, _ := hex.DecodeString("df06d62447fd25da07c0135eed7557e5a5497ee7d15b7fe345bd47e191d8f577") @@ -102,6 +102,7 @@ func TestTransactionFound(t *testing.T) { cursorRange := events.CursorRange{Start: start, End: end} err = eventReader.GetEvents(ctx, cursorRange, nil, nil) + require.NoError(t, err) // check all 200 cases for _, lcm := range lcms { diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 19f38eea..8cfaeebd 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -4,11 +4,12 @@ import ( "context" "encoding/json" "fmt" + "strings" + "time" + "github.com/stellar/go/support/log" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" - "strings" - "time" "github.com/creachadair/jrpc2" @@ -107,7 +108,7 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { } for i, filter := range g.Filters { if err := filter.Valid(); err != nil { - return errors.Wrapf(err, "filter %d invalid", i+1) + return fmt.Errorf("filter %d invalid: %w", i+1) } } @@ -162,7 +163,7 @@ func (e *EventFilter) Valid() error { } for i, topic := range e.Topics { if err := topic.Valid(); err != nil { - return errors.Wrapf(err, "topic %d invalid", i+1) + return fmt.Errorf("topic %d invalid: %w", i+1, err) } } return nil @@ -220,7 +221,7 @@ func (t *TopicFilter) Valid() error { } for i, segment := range *t { if err := segment.Valid(); err != nil { - return errors.Wrapf(err, "segment %d invalid", i+1) + return fmt.Errorf("segment %d invalid: %w", i+1, err) } } return nil @@ -372,7 +373,6 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } err := h.dbReader.GetEvents(ctx, cursorRange, contractIds, f) - if err != nil { return GetEventsResponse{}, &jrpc2.Error{ Code: jrpc2.InvalidRequest, @@ -393,7 +393,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } results = append(results, info) } - //TODO (prit): Refactor latest ledger code !! + // TODO (prit): Refactor latest ledger code !! return GetEventsResponse{ LatestLedger: 0, Events: results, From 6551dc1606045ee02935e9be98cef0ddc00741c8 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Thu, 20 Jun 2024 18:23:59 -0700 Subject: [PATCH 17/63] Fix lint issues part 2 --- cmd/soroban-rpc/internal/db/event.go | 30 ++++++---- cmd/soroban-rpc/internal/db/mocks.go | 56 ++++++------------- .../internal/methods/get_events.go | 28 +++++++--- 3 files changed, 56 insertions(+), 58 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 45c5aa3c..6440f1a0 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -10,7 +10,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/stellar/go/ingest" "github.com/stellar/go/support/db" - "github.com/stellar/go/support/errors" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" @@ -25,7 +24,7 @@ type EventWriter interface { // EventReader has all the public methods to fetch events from DB type EventReader interface { - GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIds []string, f ScanFunction) error + GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIDs []string, f ScanFunction) error // GetLedgerRange(ctx context.Context) error } @@ -130,11 +129,16 @@ func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWi // The events are returned in sorted ascending Cursor order. // If f returns false, the scan terminates early (f will not be applied on // remaining events in the range). -func (eventHandler *eventHandler) GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIds []string, f ScanFunction) error { +func (eventHandler *eventHandler) GetEvents( + ctx context.Context, + cursorRange events.CursorRange, + contractIDs []string, + f ScanFunction, +) error { start := time.Now() var rows []struct { - EventCursorId string `db:"id"` + EventCursorID string `db:"id"` TxIndex int `db:"application_order"` Lcm xdr.LedgerCloseMeta `db:"meta"` } @@ -147,18 +151,22 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, cursorRange eve Where(sq.Lt{"e.id": cursorRange.End.String()}). OrderBy("e.id ASC") - if len(contractIds) > 0 { - rowQ = rowQ.Where(sq.Eq{"e.contract_id": contractIds}) + if len(contractIDs) > 0 { + rowQ = rowQ.Where(sq.Eq{"e.contract_id": contractIDs}) } if err := eventHandler.db.Select(ctx, &rows, rowQ); err != nil { - return fmt.Errorf("db read failed for start ledger cursor= %v contractIds= %v: %w", cursorRange.Start.String(), contractIds, err) + return fmt.Errorf( + "db read failed for start ledger cursor= %v contractIDs= %v: %w", + cursorRange.Start.String(), + contractIDs, + err) } else if len(rows) < 1 { - return errors.New("No LCM found with requested event filters") + return fmt.Errorf("no LCM found with requested event filters") } for _, row := range rows { - eventCursorId, txIndex, lcm := row.EventCursorId, row.TxIndex, row.Lcm + eventCursorID, txIndex, lcm := row.EventCursorID, row.TxIndex, row.Lcm reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) if err != nil { return fmt.Errorf("failed to index to tx %d in ledger %d: %w", txIndex, lcm.LedgerSequence(), err) @@ -175,7 +183,7 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, cursorRange eve diagEvents, diagErr := ledgerTx.GetDiagnosticEvents() if diagErr != nil { - return fmt.Errorf("db read failed for Event Id %s: %w", eventCursorId, err) + return fmt.Errorf("db read failed for Event Id %s: %w", eventCursorID, err) } // Find events based on filter passed in function f @@ -191,7 +199,7 @@ func (eventHandler *eventHandler) GetEvents(ctx context.Context, cursorRange eve WithField("startLedgerSequence", cursorRange.Start.Ledger). WithField("endLedgerSequence", cursorRange.End.Ledger). WithField("duration", time.Since(start)). - Debugf("Fetched and decoded all the events with filters - contractIds: %v ", contractIds) + Debugf("Fetched and decoded all the events with filters - contractIDs: %v ", contractIDs) return nil } diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index 5b9ee6d8..91c364d5 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -33,41 +33,30 @@ func NewMockTransactionStore(passphrase string) *mockTransactionHandler { } type mockEventHandler struct { - passphrase string - ledgerRange ledgerbucketwindow.LedgerRange - contractIdToMeta map[string]ingest.LedgerTransaction - eventTypeToTx map[int]ingest.LedgerTransaction - eventIdToTx map[string]ingest.LedgerTransaction - ledgerSeqToMeta map[uint32]*xdr.LedgerCloseMeta + passphrase string + ledgerRange ledgerbucketwindow.LedgerRange + contractIDToTx map[string]ingest.LedgerTransaction + eventTypeToTx map[int]ingest.LedgerTransaction + eventIdToTx map[string]ingest.LedgerTransaction + ledgerSeqToMeta map[uint32]*xdr.LedgerCloseMeta } func NewMockEventStore(passphrase string) *mockEventHandler { return &mockEventHandler{ - passphrase: passphrase, - contractIdToMeta: make(map[string]ingest.LedgerTransaction), - eventTypeToTx: make(map[int]ingest.LedgerTransaction), - eventIdToTx: make(map[string]ingest.LedgerTransaction), - ledgerSeqToMeta: make(map[uint32]*xdr.LedgerCloseMeta), + passphrase: passphrase, + contractIDToTx: make(map[string]ingest.LedgerTransaction), + eventTypeToTx: make(map[int]ingest.LedgerTransaction), + eventIdToTx: make(map[string]ingest.LedgerTransaction), + ledgerSeqToMeta: make(map[uint32]*xdr.LedgerCloseMeta), } } -func (eventHandler *mockEventHandler) GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIds []string, f ScanFunction) error { - if contractIds != nil { - for _, contractId := range contractIds { - ledgerTx, ok := eventHandler.contractIdToMeta[contractId] - if ok { - diagEvents, diagErr := ledgerTx.GetDiagnosticEvents() - if diagErr != nil { - } - - for _, event := range diagEvents { - if !f(event, events.Cursor{}, 0, &ledgerTx.Result.TransactionHash) { - return nil - } - } - } - } - } +func (eventHandler *mockEventHandler) GetEvents( + ctx context.Context, + cursorRange events.CursorRange, + contractIds []string, + f ScanFunction, +) error { return nil } @@ -96,7 +85,7 @@ func (eventHandler *mockEventHandler) IngestEvents(lcm xdr.LedgerCloseMeta) erro var contractId []byte if e.Event.ContractId != nil { contractId = e.Event.ContractId[:] - eventHandler.contractIdToMeta[string(contractId)] = tx + eventHandler.contractIDToTx[string(contractId)] = tx } id := events.Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() eventHandler.eventTypeToTx[int(e.Event.Type)] = tx @@ -193,15 +182,6 @@ func (m *mockLedgerReader) StreamAllLedgers(ctx context.Context, f StreamLedgerF return nil } -type mockEventReader struct{} - -func NewMockEventReader() { -} - -func (m *mockEventReader) GetEvents(ctx context.Context, startLedgerSequence int, eventTypes []int, contractIds []string, f ScanFunction) error { - return nil -} - var ( _ TransactionReader = &mockTransactionHandler{} _ TransactionWriter = &mockTransactionHandler{} diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 8cfaeebd..e32bbfaf 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -104,11 +104,11 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { // Validate filters if len(g.Filters) > 5 { - return errors.New("maximum 5 filters per request") + return fmt.Errorf("maximum 5 filters per request") } for i, filter := range g.Filters { if err := filter.Valid(); err != nil { - return fmt.Errorf("filter %d invalid: %w", i+1) + return fmt.Errorf("filter %d invalid: %w", i+1, err) } } @@ -314,7 +314,7 @@ type eventsRPCHandler struct { networkPassphrase string } -func combineContractIds(filters []EventFilter) []string { +func combineContractIDs(filters []EventFilter) []string { contractIDSet := make(map[string]struct{}) for _, filter := range filters { @@ -338,7 +338,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } } - start := events.Cursor{Ledger: uint32(request.StartLedger)} + start := events.Cursor{Ledger: request.StartLedger} limit := h.defaultLimit if request.Pagination != nil { if request.Pagination.Cursor != nil { @@ -351,7 +351,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques limit = request.Pagination.Limit } } - end := events.Cursor{Ledger: uint32(request.StartLedger + ledgerbucketwindow.OneDayOfLedgers)} + end := events.Cursor{Ledger: request.StartLedger + ledgerbucketwindow.OneDayOfLedgers} cursorRange := events.CursorRange{Start: start, End: end} type entry struct { @@ -362,7 +362,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } var found []entry - contractIds := combineContractIds(request.Filters) + contractIDs := combineContractIDs(request.Filters) // Scan function to apply filters f := func(event xdr.DiagnosticEvent, cursor events.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { @@ -372,7 +372,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques return uint(len(found)) < limit } - err := h.dbReader.GetEvents(ctx, cursorRange, contractIds, f) + err := h.dbReader.GetEvents(ctx, cursorRange, contractIDs, f) if err != nil { return GetEventsResponse{}, &jrpc2.Error{ Code: jrpc2.InvalidRequest, @@ -400,7 +400,12 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques }, nil } -func eventInfoForEvent(event xdr.DiagnosticEvent, cursor events.Cursor, ledgerClosedAt string, txHash string) (EventInfo, error) { +func eventInfoForEvent( + event xdr.DiagnosticEvent, + cursor events.Cursor, + ledgerClosedAt string, + txHash string, +) (EventInfo, error) { v0, ok := event.Event.Body.GetV0() if !ok { return EventInfo{}, errors.New("unknown event version") @@ -445,7 +450,12 @@ func eventInfoForEvent(event xdr.DiagnosticEvent, cursor events.Cursor, ledgerCl } // NewGetEventsHandler returns a json rpc handler to fetch and filter events -func NewGetEventsHandler(logger *log.Entry, dbReader db.EventReader, maxLimit, defaultLimit uint, networkPassphrase string) jrpc2.Handler { +func NewGetEventsHandler(logger *log.Entry, + dbReader db.EventReader, + maxLimit uint, + defaultLimit uint, + networkPassphrase string, +) jrpc2.Handler { eventsHandler := eventsRPCHandler{ dbReader: dbReader, maxLimit: maxLimit, From 7ac1ece4886150d460d96cd85a305fe94a1314b7 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Fri, 21 Jun 2024 09:27:46 -0700 Subject: [PATCH 18/63] Fix more lint and add ledger range code for events --- cmd/soroban-rpc/internal/db/event.go | 76 ++++++++++++++++++- cmd/soroban-rpc/internal/db/mocks.go | 56 +++++++------- .../internal/methods/get_events.go | 15 +++- 3 files changed, 114 insertions(+), 33 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 6440f1a0..4a41ab3c 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -6,6 +6,8 @@ import ( "io" "time" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" + sq "github.com/Masterminds/squirrel" "github.com/prometheus/client_golang/prometheus" "github.com/stellar/go/ingest" @@ -25,7 +27,7 @@ type EventWriter interface { // EventReader has all the public methods to fetch events from DB type EventReader interface { GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIDs []string, f ScanFunction) error - // GetLedgerRange(ctx context.Context) error + GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) } type eventHandler struct { @@ -146,7 +148,7 @@ func (eventHandler *eventHandler) GetEvents( rowQ := sq. Select("e.id", "e.application_order", "lcm.meta"). From(eventTableName + " e"). - Join(ledgerCloseMetaTableName + "lcm ON (e.ledger_sequence = lcm.sequence)"). + Join(ledgerCloseMetaTableName + " lcm ON (e.ledger_sequence = lcm.sequence)"). Where(sq.GtOrEq{"e.id": cursorRange.Start.String()}). Where(sq.Lt{"e.id": cursorRange.End.String()}). OrderBy("e.id ASC") @@ -203,3 +205,73 @@ func (eventHandler *eventHandler) GetEvents( return nil } + +// GetLedgerRange returns the min/max ledger sequence numbers from the events table +// TODO: Once we unify all the retention window we would keep only one GetLedgerRange logic + +func (eventHandler *eventHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { + var ledgerRange ledgerbucketwindow.LedgerRange + + // + // We use subqueries alongside a UNION ALL stitch in order to select the min + // and max from the ledger table in a single query and get around sqlite's + // limitations with parentheses (see https://stackoverflow.com/a/22609948). + // + // Queries to get the minimum and maximum ledger sequence from the transactions table + minLedgerSeqQ := sq. + Select("m1.ledger_sequence"). + FromSelect( + sq. + Select("ledger_sequence"). + From(eventTableName). + OrderBy("ledger_sequence ASC"). + Limit(1), + "m1", + ) + maxLedgerSeqQ, args, err := sq. + Select("m2.ledger_sequence"). + FromSelect( + sq. + Select("ledger_sequence"). + From(eventTableName). + OrderBy("ledger_sequence DESC"). + Limit(1), + "m2", + ).ToSql() + if err != nil { + return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) + } + + // Combine the min and max ledger sequence queries using UNION ALL + eventMinMaxLedgersQ, _, err := minLedgerSeqQ.Suffix("UNION ALL "+maxLedgerSeqQ, args...).ToSql() + if err != nil { + return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) + } + + // Final query to join ledger_close_meta table and the sequence numbers we got from eventMinMaxLedgersQ + finalSQL := sq. + Select("lcm.meta"). + From(ledgerCloseMetaTableName + " as lcm"). + JoinClause(fmt.Sprintf("JOIN (%s) as seqs ON lcm.sequence == seqs.ledger_sequence", eventMinMaxLedgersQ)) + + var lcms []xdr.LedgerCloseMeta + if err = eventHandler.db.Select(ctx, &lcms, finalSQL); err != nil { + return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) + } else if len(lcms) < 2 { + // There is almost certainly a row, but we want to avoid a race condition + // with ingestion as well as support test cases from an empty DB, so we need + // to sanity check that there is in fact a result. Note that no ledgers in + // the database isn't an error, it's just an empty range. + return ledgerRange, nil + } + + lcm1, lcm2 := lcms[0], lcms[1] + ledgerRange.FirstLedger.Sequence = lcm1.LedgerSequence() + ledgerRange.FirstLedger.CloseTime = lcm1.LedgerCloseTime() + ledgerRange.LastLedger.Sequence = lcm2.LedgerSequence() + ledgerRange.LastLedger.CloseTime = lcm2.LedgerCloseTime() + + eventHandler.log.Debugf("Database ledger range: [%d, %d]", + ledgerRange.FirstLedger.Sequence, ledgerRange.LastLedger.Sequence) + return ledgerRange, nil +} diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index 91c364d5..02811495 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -14,7 +14,7 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) -type mockTransactionHandler struct { +type MockTransactionHandler struct { passphrase string ledgerRange ledgerbucketwindow.LedgerRange @@ -23,8 +23,8 @@ type mockTransactionHandler struct { ledgerSeqToMeta map[uint32]*xdr.LedgerCloseMeta } -func NewMockTransactionStore(passphrase string) *mockTransactionHandler { - return &mockTransactionHandler{ +func NewMockTransactionStore(passphrase string) *MockTransactionHandler { + return &MockTransactionHandler{ passphrase: passphrase, txs: make(map[string]ingest.LedgerTransaction), txHashToMeta: make(map[string]*xdr.LedgerCloseMeta), @@ -32,35 +32,35 @@ func NewMockTransactionStore(passphrase string) *mockTransactionHandler { } } -type mockEventHandler struct { +type MockEventHandler struct { passphrase string ledgerRange ledgerbucketwindow.LedgerRange contractIDToTx map[string]ingest.LedgerTransaction eventTypeToTx map[int]ingest.LedgerTransaction - eventIdToTx map[string]ingest.LedgerTransaction + eventIDToTx map[string]ingest.LedgerTransaction ledgerSeqToMeta map[uint32]*xdr.LedgerCloseMeta } -func NewMockEventStore(passphrase string) *mockEventHandler { - return &mockEventHandler{ +func NewMockEventStore(passphrase string) *MockEventHandler { + return &MockEventHandler{ passphrase: passphrase, contractIDToTx: make(map[string]ingest.LedgerTransaction), eventTypeToTx: make(map[int]ingest.LedgerTransaction), - eventIdToTx: make(map[string]ingest.LedgerTransaction), + eventIDToTx: make(map[string]ingest.LedgerTransaction), ledgerSeqToMeta: make(map[uint32]*xdr.LedgerCloseMeta), } } -func (eventHandler *mockEventHandler) GetEvents( +func (eventHandler *MockEventHandler) GetEvents( ctx context.Context, cursorRange events.CursorRange, - contractIds []string, + contractIDs []string, f ScanFunction, ) error { return nil } -func (eventHandler *mockEventHandler) IngestEvents(lcm xdr.LedgerCloseMeta) error { +func (eventHandler *MockEventHandler) IngestEvents(lcm xdr.LedgerCloseMeta) error { eventHandler.ledgerSeqToMeta[lcm.LedgerSequence()] = &lcm reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) @@ -82,14 +82,14 @@ func (eventHandler *mockEventHandler) IngestEvents(lcm xdr.LedgerCloseMeta) erro } for index, e := range txEvents { - var contractId []byte + var contractID []byte if e.Event.ContractId != nil { - contractId = e.Event.ContractId[:] - eventHandler.contractIDToTx[string(contractId)] = tx + contractID = e.Event.ContractId[:] + eventHandler.contractIDToTx[string(contractID)] = tx } id := events.Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() eventHandler.eventTypeToTx[int(e.Event.Type)] = tx - eventHandler.eventIdToTx[id] = tx + eventHandler.eventIDToTx[id] = tx } } @@ -107,7 +107,7 @@ func (eventHandler *mockEventHandler) IngestEvents(lcm xdr.LedgerCloseMeta) erro return nil } -func (txn *mockTransactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) error { +func (txn *MockTransactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) error { txn.ledgerSeqToMeta[lcm.LedgerSequence()] = &lcm reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(txn.passphrase, lcm) @@ -143,11 +143,11 @@ func (txn *mockTransactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) e } // GetLedgerRange pulls the min/max ledger sequence numbers from the database. -func (txn *mockTransactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { +func (txn *MockTransactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { return txn.ledgerRange, nil } -func (txn *mockTransactionHandler) GetTransaction(ctx context.Context, hash xdr.Hash) ( +func (txn *MockTransactionHandler) GetTransaction(ctx context.Context, hash xdr.Hash) ( Transaction, ledgerbucketwindow.LedgerRange, error, ) { if tx, ok := txn.txs[hash.HexString()]; !ok { @@ -158,19 +158,19 @@ func (txn *mockTransactionHandler) GetTransaction(ctx context.Context, hash xdr. } } -func (txn *mockTransactionHandler) RegisterMetrics(_, _ prometheus.Observer) {} +func (txn *MockTransactionHandler) RegisterMetrics(_, _ prometheus.Observer) {} -type mockLedgerReader struct { - txn mockTransactionHandler +type MockLedgerReader struct { + txn MockTransactionHandler } -func NewMockLedgerReader(txn *mockTransactionHandler) *mockLedgerReader { - return &mockLedgerReader{ +func NewMockLedgerReader(txn *MockTransactionHandler) *MockLedgerReader { + return &MockLedgerReader{ txn: *txn, } } -func (m *mockLedgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) { +func (m *MockLedgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) { lcm, ok := m.txn.ledgerSeqToMeta[sequence] if !ok { return xdr.LedgerCloseMeta{}, false, nil @@ -178,12 +178,12 @@ func (m *mockLedgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr. return *lcm, true, nil } -func (m *mockLedgerReader) StreamAllLedgers(ctx context.Context, f StreamLedgerFn) error { +func (m *MockLedgerReader) StreamAllLedgers(ctx context.Context, f StreamLedgerFn) error { return nil } var ( - _ TransactionReader = &mockTransactionHandler{} - _ TransactionWriter = &mockTransactionHandler{} - _ LedgerReader = &mockLedgerReader{} + _ TransactionReader = &MockTransactionHandler{} + _ TransactionWriter = &MockTransactionHandler{} + _ LedgerReader = &MockLedgerReader{} ) diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index e32bbfaf..16ba06d2 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -331,6 +331,7 @@ func combineContractIDs(filters []EventFilter) []string { } func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsRequest) (GetEventsResponse, error) { + if err := request.Valid(h.maxLimit); err != nil { return GetEventsResponse{}, &jrpc2.Error{ Code: jrpc2.InvalidParams, @@ -338,6 +339,14 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } } + ledgerRange, err := h.dbReader.GetLedgerRange(ctx) + if err != nil { + return GetEventsResponse{}, &jrpc2.Error{ + Code: jrpc2.InternalError, + Message: err.Error(), + } + } + start := events.Cursor{Ledger: request.StartLedger} limit := h.defaultLimit if request.Pagination != nil { @@ -372,7 +381,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques return uint(len(found)) < limit } - err := h.dbReader.GetEvents(ctx, cursorRange, contractIDs, f) + err = h.dbReader.GetEvents(ctx, cursorRange, contractIDs, f) if err != nil { return GetEventsResponse{}, &jrpc2.Error{ Code: jrpc2.InvalidRequest, @@ -393,9 +402,9 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } results = append(results, info) } - // TODO (prit): Refactor latest ledger code !! + return GetEventsResponse{ - LatestLedger: 0, + LatestLedger: ledgerRange.LastLedger.Sequence, Events: results, }, nil } From 768e657354d3467e0a348fff62000b78529d818c Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Fri, 21 Jun 2024 10:50:31 -0700 Subject: [PATCH 19/63] Update eventHandler mock --- cmd/soroban-rpc/internal/daemon/daemon.go | 7 +++++-- cmd/soroban-rpc/internal/db/mocks.go | 4 ++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index 363a5e8b..63d6bc56 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -300,8 +300,11 @@ func MustNew(cfg *config.Config, logger *supportlog.Entry) *Daemon { // mustInitializeStorage initializes the storage using what was on the DB func (d *Daemon) mustInitializeStorage(cfg *config.Config) *feewindow.FeeWindows { - - feewindows := feewindow.NewFeeWindows(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase) + feewindows := feewindow.NewFeeWindows( + cfg.ClassicFeeStatsLedgerRetentionWindow, + cfg.SorobanFeeStatsLedgerRetentionWindow, + cfg.NetworkPassphrase, + ) readTxMetaCtx, cancelReadTxMeta := context.WithTimeout(context.Background(), cfg.IngestionTimeout) defer cancelReadTxMeta() diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index 02811495..0fa07d69 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -60,6 +60,10 @@ func (eventHandler *MockEventHandler) GetEvents( return nil } +func (eventHandler *MockEventHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { + return ledgerbucketwindow.LedgerRange{}, nil +} + func (eventHandler *MockEventHandler) IngestEvents(lcm xdr.LedgerCloseMeta) error { eventHandler.ledgerSeqToMeta[lcm.LedgerSequence()] = &lcm From d3c913c06644c43f089f263c4931e479dedabe67 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 24 Jun 2024 11:12:53 -0700 Subject: [PATCH 20/63] Fix 2 major errors in tests: nil pointer reference and unknown hash --- .../internal/db/transaction_test.go | 2 +- .../internal/methods/get_events_test.go | 58 ++++++++++++++++--- 2 files changed, 50 insertions(+), 10 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 8fe89696..44b28f29 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -162,7 +162,7 @@ func BenchmarkTransactionFetch(b *testing.B) { func txHash(acctSeq uint32) xdr.Hash { envelope := txEnvelope(acctSeq) - hash, err := network.HashTransactionInEnvelope(envelope, passphrase) + hash, err := network.HashTransactionInEnvelope(envelope, "passphrase") if err != nil { panic(err) } diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index fcc02fe5..5b520492 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -4,8 +4,12 @@ import ( "context" "encoding/json" "fmt" + "github.com/sirupsen/logrus" + "github.com/stellar/go/support/log" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" + "github.com/stretchr/testify/require" "strings" "testing" "time" @@ -18,6 +22,8 @@ import ( "github.com/stellar/go/xdr" ) +var passphrase = "passphrase" + func TestEventTypeSetMatches(t *testing.T) { var defaultSet eventTypeSet @@ -573,9 +579,18 @@ func TestGetEvents(t *testing.T) { }) t.Run("no filtering returns all", func(t *testing.T) { - contractID := xdr.Hash([32]byte{}) - store := db.NewMockEventStore("passphrase") + dbx := db.NewTestDB(t) + ctx := context.TODO() + log := log.DefaultLogger + log.SetLevel(logrus.TraceLevel) + + writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() + store := db.NewEventReader(log, dbx, passphrase) + + contractID := xdr.Hash([32]byte{}) var txMeta []xdr.TransactionMeta for i := 0; i < 10; i++ { txMeta = append(txMeta, transactionMetaWithEvents( @@ -592,8 +607,11 @@ func TestGetEvents(t *testing.T) { ), )) } + ledgerCloseMeta := ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...) - assert.NoError(t, store.IngestEvents(ledgerCloseMeta)) + require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") + assert.NoError(t, eventW.InsertEvents(ledgerCloseMeta)) + require.NoError(t, write.Commit(1)) handler := eventsRPCHandler{ dbReader: store, @@ -635,17 +653,26 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by contract id", func(t *testing.T) { - store := db.NewMockEventStore("passphrase") + + dbx := db.NewTestDB(t) + ctx := context.TODO() + log := log.DefaultLogger + log.SetLevel(logrus.TraceLevel) + + writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() + store := db.NewEventReader(log, dbx, passphrase) var txMeta []xdr.TransactionMeta - contractIds := []xdr.Hash{ + contractIDs := []xdr.Hash{ xdr.Hash([32]byte{}), xdr.Hash([32]byte{1}), } for i := 0; i < 5; i++ { txMeta = append(txMeta, transactionMetaWithEvents( contractEvent( - contractIds[i%len(contractIds)], + contractIDs[i%len(contractIDs)], xdr.ScVec{xdr.ScVal{ Type: xdr.ScValTypeScvSymbol, Sym: &counter, @@ -657,7 +684,13 @@ func TestGetEvents(t *testing.T) { ), )) } - assert.NoError(t, store.IngestEvents(ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...))) + + ledgerCloseMeta := ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...) + require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") + assert.NoError(t, eventW.InsertEvents(ledgerCloseMeta)) + require.NoError(t, write.Commit(1)) + + //assert.NoError(t, store.IngestEvents(ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...))) handler := eventsRPCHandler{ dbReader: store, @@ -667,7 +700,7 @@ func TestGetEvents(t *testing.T) { results, err := handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, Filters: []EventFilter{ - {ContractIDs: []string{strkey.MustEncode(strkey.VersionByteContract, contractIds[0][:])}}, + {ContractIDs: []string{strkey.MustEncode(strkey.VersionByteContract, contractIDs[0][:])}}, }, }) assert.NoError(t, err) @@ -1102,7 +1135,7 @@ func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ... }, }, } - txHash, err := network.HashTransactionInEnvelope(envelope, "unit-tests") + txHash, err := network.HashTransactionInEnvelope(envelope, "passphrase") if err != nil { panic(err) } @@ -1111,6 +1144,7 @@ func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ... TxApplyProcessing: item, Result: xdr.TransactionResultPair{ TransactionHash: txHash, + Result: transactionResult(true), }, }) components := []xdr.TxSetComponent{ @@ -1154,12 +1188,18 @@ func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ... } func transactionMetaWithEvents(events ...xdr.ContractEvent) xdr.TransactionMeta { + counter := xdr.ScSymbol("COUNTER") + return xdr.TransactionMeta{ V: 3, Operations: &[]xdr.OperationMeta{}, V3: &xdr.TransactionMetaV3{ SorobanMeta: &xdr.SorobanTransactionMeta{ Events: events, + ReturnValue: xdr.ScVal{ + Type: xdr.ScValTypeScvSymbol, + Sym: &counter, + }, }, }, } From 432688313ae8d2e10c44eed37482b2f840dfc0cc Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 24 Jun 2024 15:07:31 -0700 Subject: [PATCH 21/63] Fix contract Id filter logic and add cursor set to avoid duplicates --- cmd/soroban-rpc/internal/db/event.go | 7 +- .../internal/db/sqlmigrations/03_events.sql | 2 +- .../internal/methods/get_events.go | 8 ++ .../internal/methods/get_events_test.go | 79 +++++++++++++++---- 4 files changed, 78 insertions(+), 18 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 4a41ab3c..8122cecf 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -3,6 +3,7 @@ package db import ( "context" "fmt" + "github.com/stellar/go/strkey" "io" "time" @@ -93,12 +94,12 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { Columns("id", "ledger_sequence", "application_order", "contract_id", "event_type") for index, e := range txEvents { - var contractId []byte + var contractID string if e.Event.ContractId != nil { - contractId = e.Event.ContractId[:] + contractID = strkey.MustEncode(strkey.VersionByteContract, (*e.Event.ContractId)[:]) } id := events.Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() - query = query.Values(id, lcm.LedgerSequence(), tx.Index, contractId, int(e.Event.Type)) + query = query.Values(id, lcm.LedgerSequence(), tx.Index, contractID, int(e.Event.Type)) } _, err = query.RunWith(eventHandler.stmtCache).Exec() diff --git a/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql b/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql index d8040575..7a81d0af 100644 --- a/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql +++ b/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql @@ -5,7 +5,7 @@ CREATE TABLE events( id TEXT PRIMARY KEY, ledger_sequence INTEGER NOT NULL, application_order INTEGER NOT NULL, - contract_id BLOB, + contract_id TEXT, event_type INTEGER NOT NULL ); diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 16ba06d2..e7bc0aa1 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -370,12 +370,20 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques txHash *xdr.Hash } var found []entry + cursorSet := make(map[string]struct{}) contractIDs := combineContractIDs(request.Filters) // Scan function to apply filters f := func(event xdr.DiagnosticEvent, cursor events.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { + + cursorID := cursor.String() + if _, exists := cursorSet[cursorID]; exists { + return true + } + if request.Matches(event) && cursor.Cmp(start) >= 0 { + cursorSet[cursorID] = struct{}{} found = append(found, entry{cursor, ledgerCloseTimestamp, event, txHash}) } return uint(len(found)) < limit diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 5b520492..5e1dfc39 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -687,10 +687,8 @@ func TestGetEvents(t *testing.T) { ledgerCloseMeta := ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...) require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") - assert.NoError(t, eventW.InsertEvents(ledgerCloseMeta)) - require.NoError(t, write.Commit(1)) - - //assert.NoError(t, store.IngestEvents(ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...))) + require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") + require.NoError(t, write.Commit(2)) handler := eventsRPCHandler{ dbReader: store, @@ -719,7 +717,15 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by topic", func(t *testing.T) { - store := db.NewMockEventStore("passphrase") + dbx := db.NewTestDB(t) + ctx := context.TODO() + log := log.DefaultLogger + log.SetLevel(logrus.TraceLevel) + + writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() + store := db.NewEventReader(log, dbx, passphrase) var txMeta []xdr.TransactionMeta contractID := xdr.Hash([32]byte{}) @@ -738,7 +744,10 @@ func TestGetEvents(t *testing.T) { )) } ledgerCloseMeta := ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...) - assert.NoError(t, store.IngestEvents(ledgerCloseMeta)) + + require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") + require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") + require.NoError(t, write.Commit(1)) number := xdr.Uint64(4) handler := eventsRPCHandler{ @@ -784,7 +793,16 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by both contract id and topic", func(t *testing.T) { - store := db.NewMockEventStore("passphrase") + + dbx := db.NewTestDB(t) + ctx := context.TODO() + log := log.DefaultLogger + log.SetLevel(logrus.TraceLevel) + + writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() + store := db.NewEventReader(log, dbx, passphrase) contractID := xdr.Hash([32]byte{}) otherContractID := xdr.Hash([32]byte{1}) @@ -834,7 +852,10 @@ func TestGetEvents(t *testing.T) { ), } ledgerCloseMeta := ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...) - assert.NoError(t, store.IngestEvents(ledgerCloseMeta)) + + require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") + require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") + require.NoError(t, write.Commit(1)) handler := eventsRPCHandler{ dbReader: store, @@ -881,7 +902,15 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by event type", func(t *testing.T) { - store := db.NewMockEventStore("passphrase") + dbx := db.NewTestDB(t) + ctx := context.TODO() + log := log.DefaultLogger + log.SetLevel(logrus.TraceLevel) + + writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() + store := db.NewEventReader(log, dbx, passphrase) contractID := xdr.Hash([32]byte{}) txMeta := []xdr.TransactionMeta{ @@ -910,7 +939,9 @@ func TestGetEvents(t *testing.T) { ), } ledgerCloseMeta := ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...) - assert.NoError(t, store.IngestEvents(ledgerCloseMeta)) + require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") + require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") + require.NoError(t, write.Commit(1)) handler := eventsRPCHandler{ dbReader: store, @@ -944,7 +975,15 @@ func TestGetEvents(t *testing.T) { }) t.Run("with limit", func(t *testing.T) { - store := db.NewMockEventStore("passphrase") + dbx := db.NewTestDB(t) + ctx := context.TODO() + log := log.DefaultLogger + log.SetLevel(logrus.TraceLevel) + + writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() + store := db.NewEventReader(log, dbx, passphrase) contractID := xdr.Hash([32]byte{}) var txMeta []xdr.TransactionMeta @@ -961,7 +1000,9 @@ func TestGetEvents(t *testing.T) { )) } ledgerCloseMeta := ledgerCloseMetaWithEvents(1, now.Unix(), txMeta...) - assert.NoError(t, store.IngestEvents(ledgerCloseMeta)) + require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") + require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") + require.NoError(t, write.Commit(1)) handler := eventsRPCHandler{ dbReader: store, @@ -1002,7 +1043,15 @@ func TestGetEvents(t *testing.T) { }) t.Run("with cursor", func(t *testing.T) { - store := db.NewMockEventStore("passphrase") + dbx := db.NewTestDB(t) + ctx := context.TODO() + log := log.DefaultLogger + log.SetLevel(logrus.TraceLevel) + + writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() + store := db.NewEventReader(log, dbx, passphrase) contractID := xdr.Hash([32]byte{}) datas := []xdr.ScSymbol{ @@ -1047,7 +1096,9 @@ func TestGetEvents(t *testing.T) { ), } ledgerCloseMeta := ledgerCloseMetaWithEvents(5, now.Unix(), txMeta...) - assert.NoError(t, store.IngestEvents(ledgerCloseMeta)) + require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") + require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") + require.NoError(t, write.Commit(1)) id := &events.Cursor{Ledger: 5, Tx: 1, Op: 0, Event: 0} handler := eventsRPCHandler{ From a4f3d6ab8ca20654578b14f4e163b40bfe83a4b1 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 24 Jun 2024 16:48:33 -0700 Subject: [PATCH 22/63] Validate requested start ledger with stored ledger range --- cmd/soroban-rpc/internal/db/event.go | 8 +++-- .../internal/methods/get_events.go | 10 +++++- .../internal/methods/get_events_test.go | 36 +++++++++---------- 3 files changed, 33 insertions(+), 21 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 8122cecf..88351458 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -165,7 +165,11 @@ func (eventHandler *eventHandler) GetEvents( contractIDs, err) } else if len(rows) < 1 { - return fmt.Errorf("no LCM found with requested event filters") + eventHandler.log.Infof("No events found for start ledger cursor= %v contractIDs= %v", + cursorRange.Start.String(), + contractIDs, + ) + return nil } for _, row := range rows { @@ -192,7 +196,7 @@ func (eventHandler *eventHandler) GetEvents( // Find events based on filter passed in function f for eventIndex, event := range diagEvents { cur := events.Cursor{Ledger: lcm.LedgerSequence(), Tx: uint32(txIndex), Event: uint32(eventIndex)} - if f != nil && !f(event, cur, ledgerCloseTime, &transactionHash) { + if !f(event, cur, ledgerCloseTime, &transactionHash) { return nil } } diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index e7bc0aa1..1ef0a7c3 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -363,6 +363,14 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques end := events.Cursor{Ledger: request.StartLedger + ledgerbucketwindow.OneDayOfLedgers} cursorRange := events.CursorRange{Start: start, End: end} + // Check if requested start ledger is within stored ledger range: + if start.Ledger < ledgerRange.FirstLedger.Sequence || start.Ledger > ledgerRange.LastLedger.Sequence { + return GetEventsResponse{}, &jrpc2.Error{ + Code: jrpc2.InvalidRequest, + Message: fmt.Sprintf("startLedger must be within the ledger range: %d - %d", ledgerRange.FirstLedger.Sequence, ledgerRange.LastLedger.Sequence), + } + } + type entry struct { cursor events.Cursor ledgerCloseTimestamp int64 @@ -397,7 +405,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } } - var results []EventInfo + results := []EventInfo{} for _, entry := range found { info, err := eventInfoForEvent( entry.event, diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 5e1dfc39..553824b7 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -530,22 +530,18 @@ func TestGetEvents(t *testing.T) { counterXdr, err := xdr.MarshalBase64(counterScVal) assert.NoError(t, err) - t.Run("empty", func(t *testing.T) { - store := db.NewMockEventStore("passphrase") - handler := eventsRPCHandler{ - dbReader: store, - maxLimit: 10000, - defaultLimit: 100, - } - _, err = handler.getEvents(context.TODO(), GetEventsRequest{ - StartLedger: 1, - }) - assert.EqualError(t, err, "[-32600] event store is empty") - }) - t.Run("startLedger validation", func(t *testing.T) { contractID := xdr.Hash([32]byte{}) - store := db.NewMockEventStore("passphrase") + dbx := db.NewTestDB(t) + ctx := context.TODO() + log := log.DefaultLogger + log.SetLevel(logrus.TraceLevel) + + writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() + store := db.NewEventReader(log, dbx, passphrase) + var txMeta []xdr.TransactionMeta txMeta = append(txMeta, transactionMetaWithEvents( contractEvent( @@ -560,7 +556,11 @@ func TestGetEvents(t *testing.T) { }, ), )) - assert.NoError(t, store.IngestEvents(ledgerCloseMetaWithEvents(2, now.Unix(), txMeta...))) + + ledgerCloseMeta := ledgerCloseMetaWithEvents(2, now.Unix(), txMeta...) + require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") + assert.NoError(t, eventW.InsertEvents(ledgerCloseMeta)) + require.NoError(t, write.Commit(2)) handler := eventsRPCHandler{ dbReader: store, @@ -570,12 +570,12 @@ func TestGetEvents(t *testing.T) { _, err = handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, }) - assert.EqualError(t, err, "[-32600] start is before oldest ledger") + assert.EqualError(t, err, "[-32600] startLedger must be within the ledger range: 2 - 2") _, err = handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 3, }) - assert.EqualError(t, err, "[-32600] start is after newest ledger") + assert.EqualError(t, err, "[-32600] startLedger must be within the ledger range: 2 - 2") }) t.Run("no filtering returns all", func(t *testing.T) { @@ -1098,7 +1098,7 @@ func TestGetEvents(t *testing.T) { ledgerCloseMeta := ledgerCloseMetaWithEvents(5, now.Unix(), txMeta...) require.NoError(t, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") - require.NoError(t, write.Commit(1)) + require.NoError(t, write.Commit(4)) id := &events.Cursor{Ledger: 5, Tx: 1, Op: 0, Event: 0} handler := eventsRPCHandler{ From 2c35f60cde41ccee9ff441eeee0ec8787b054ebe Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 24 Jun 2024 17:10:25 -0700 Subject: [PATCH 23/63] Fix lint error part 4 --- cmd/soroban-rpc/internal/db/mocks.go | 81 ------------------- .../internal/db/transaction_test.go | 14 ++-- .../internal/methods/get_events_test.go | 17 +++- 3 files changed, 22 insertions(+), 90 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index 0fa07d69..8108f086 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -4,8 +4,6 @@ import ( "context" "io" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" - "github.com/prometheus/client_golang/prometheus" "github.com/stellar/go/ingest" @@ -32,85 +30,6 @@ func NewMockTransactionStore(passphrase string) *MockTransactionHandler { } } -type MockEventHandler struct { - passphrase string - ledgerRange ledgerbucketwindow.LedgerRange - contractIDToTx map[string]ingest.LedgerTransaction - eventTypeToTx map[int]ingest.LedgerTransaction - eventIDToTx map[string]ingest.LedgerTransaction - ledgerSeqToMeta map[uint32]*xdr.LedgerCloseMeta -} - -func NewMockEventStore(passphrase string) *MockEventHandler { - return &MockEventHandler{ - passphrase: passphrase, - contractIDToTx: make(map[string]ingest.LedgerTransaction), - eventTypeToTx: make(map[int]ingest.LedgerTransaction), - eventIDToTx: make(map[string]ingest.LedgerTransaction), - ledgerSeqToMeta: make(map[uint32]*xdr.LedgerCloseMeta), - } -} - -func (eventHandler *MockEventHandler) GetEvents( - ctx context.Context, - cursorRange events.CursorRange, - contractIDs []string, - f ScanFunction, -) error { - return nil -} - -func (eventHandler *MockEventHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { - return ledgerbucketwindow.LedgerRange{}, nil -} - -func (eventHandler *MockEventHandler) IngestEvents(lcm xdr.LedgerCloseMeta) error { - eventHandler.ledgerSeqToMeta[lcm.LedgerSequence()] = &lcm - - reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) - if err != nil { - return err - } - - for { - tx, err := reader.Read() - if err == io.EOF { - break - } else if err != nil { - return err - } - - txEvents, err := tx.GetDiagnosticEvents() - if err != nil { - return err - } - - for index, e := range txEvents { - var contractID []byte - if e.Event.ContractId != nil { - contractID = e.Event.ContractId[:] - eventHandler.contractIDToTx[string(contractID)] = tx - } - id := events.Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() - eventHandler.eventTypeToTx[int(e.Event.Type)] = tx - eventHandler.eventIDToTx[id] = tx - } - } - - if lcmSeq := lcm.LedgerSequence(); lcmSeq < eventHandler.ledgerRange.FirstLedger.Sequence || - eventHandler.ledgerRange.FirstLedger.Sequence == 0 { - eventHandler.ledgerRange.FirstLedger.Sequence = lcmSeq - eventHandler.ledgerRange.FirstLedger.CloseTime = lcm.LedgerCloseTime() - } - - if lcmSeq := lcm.LedgerSequence(); lcmSeq > eventHandler.ledgerRange.LastLedger.Sequence { - eventHandler.ledgerRange.LastLedger.Sequence = lcmSeq - eventHandler.ledgerRange.LastLedger.CloseTime = lcm.LedgerCloseTime() - } - - return nil -} - func (txn *MockTransactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) error { txn.ledgerSeqToMeta[lcm.LedgerSequence()] = &lcm diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 44b28f29..efcb432c 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -29,8 +29,8 @@ func TestTransactionNotFound(t *testing.T) { require.Error(t, err, ErrNoTransaction) } -func txMetaWithEvents(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { - meta := txMeta(acctSeq, successful) +func txMetaWithEvents(acctSeq uint32) xdr.LedgerCloseMeta { + meta := txMeta(acctSeq, true) contractIDBytes, _ := hex.DecodeString("df06d62447fd25da07c0135eed7557e5a5497ee7d15b7fe345bd47e191d8f577") var contractID xdr.Hash @@ -77,10 +77,10 @@ func TestTransactionFound(t *testing.T) { require.NoError(t, err) lcms := []xdr.LedgerCloseMeta{ - txMetaWithEvents(1234, true), - txMetaWithEvents(1235, true), - txMetaWithEvents(1236, true), - txMetaWithEvents(1237, true), + txMetaWithEvents(1234), + txMetaWithEvents(1235), + txMetaWithEvents(1236), + txMetaWithEvents(1237), } eventW := write.EventWriter() ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() @@ -162,7 +162,7 @@ func BenchmarkTransactionFetch(b *testing.B) { func txHash(acctSeq uint32) xdr.Hash { envelope := txEnvelope(acctSeq) - hash, err := network.HashTransactionInEnvelope(envelope, "passphrase") + hash, err := network.HashTransactionInEnvelope(envelope, passphrase) if err != nil { panic(err) } diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 553824b7..801f7a4a 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -539,6 +539,7 @@ func TestGetEvents(t *testing.T) { writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) write, err := writer.NewTx(ctx) + require.NoError(t, err) ledgerW, eventW := write.LedgerWriter(), write.EventWriter() store := db.NewEventReader(log, dbx, passphrase) @@ -570,12 +571,12 @@ func TestGetEvents(t *testing.T) { _, err = handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, }) - assert.EqualError(t, err, "[-32600] startLedger must be within the ledger range: 2 - 2") + require.EqualError(t, err, "[-32600] startLedger must be within the ledger range: 2 - 2") _, err = handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 3, }) - assert.EqualError(t, err, "[-32600] startLedger must be within the ledger range: 2 - 2") + require.EqualError(t, err, "[-32600] startLedger must be within the ledger range: 2 - 2") }) t.Run("no filtering returns all", func(t *testing.T) { @@ -587,6 +588,8 @@ func TestGetEvents(t *testing.T) { writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) write, err := writer.NewTx(ctx) + require.NoError(t, err) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() store := db.NewEventReader(log, dbx, passphrase) @@ -661,6 +664,8 @@ func TestGetEvents(t *testing.T) { writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) write, err := writer.NewTx(ctx) + require.NoError(t, err) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() store := db.NewEventReader(log, dbx, passphrase) @@ -724,6 +729,8 @@ func TestGetEvents(t *testing.T) { writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) write, err := writer.NewTx(ctx) + require.NoError(t, err) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() store := db.NewEventReader(log, dbx, passphrase) @@ -801,6 +808,8 @@ func TestGetEvents(t *testing.T) { writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) write, err := writer.NewTx(ctx) + require.NoError(t, err) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() store := db.NewEventReader(log, dbx, passphrase) @@ -982,6 +991,8 @@ func TestGetEvents(t *testing.T) { writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) write, err := writer.NewTx(ctx) + require.NoError(t, err) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() store := db.NewEventReader(log, dbx, passphrase) @@ -1050,6 +1061,8 @@ func TestGetEvents(t *testing.T) { writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) write, err := writer.NewTx(ctx) + require.NoError(t, err) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() store := db.NewEventReader(log, dbx, passphrase) From 9783797d8a23b41114ba73bbde62c4bd8354ffbc Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 24 Jun 2024 22:19:40 -0700 Subject: [PATCH 24/63] Add migration for events table --- cmd/soroban-rpc/internal/db/event.go | 50 ++++++++++++++++++++++++ cmd/soroban-rpc/internal/db/migration.go | 17 +++++++- 2 files changed, 65 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 88351458..f08d7431 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -280,3 +280,53 @@ func (eventHandler *eventHandler) GetLedgerRange(ctx context.Context) (ledgerbuc ledgerRange.FirstLedger.Sequence, ledgerRange.LastLedger.Sequence) return ledgerRange, nil } + +type eventTableMigration struct { + firstLedger uint32 + lastLedger uint32 + writer EventWriter +} + +func (e *eventTableMigration) ApplicableRange() *LedgerSeqRange { + return &LedgerSeqRange{ + firstLedgerSeq: e.firstLedger, + lastLedgerSeq: e.lastLedger, + } +} + +func (e *eventTableMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error { + return e.writer.InsertEvents(meta) +} + +func newEventTableMigration( + ctx context.Context, + logger *log.Entry, + retentionWindow uint32, + passphrase string, +) migrationApplierFactory { + return migrationApplierFactoryF(func(db *DB, latestLedger uint32) (MigrationApplier, error) { + firstLedgerToMigrate := uint32(2) + writer := &eventHandler{ + log: logger, + db: db, + stmtCache: sq.NewStmtCache(db.GetTx()), + passphrase: passphrase, + } + if latestLedger > retentionWindow { + firstLedgerToMigrate = latestLedger - retentionWindow + } + + // Truncate the table, since it may contain data, causing insert conflicts later on. + _, err := db.Exec(ctx, sq.Delete(transactionTableName).Where(sq.Lt{"ledger_sequence": firstLedgerToMigrate})) + + if err != nil { + return nil, fmt.Errorf("couldn't truncate the table %q: %w", transactionTableName, err) + } + migration := eventTableMigration{ + firstLedger: firstLedgerToMigrate, + lastLedger: latestLedger, + writer: writer, + } + return &migration, nil + }) +} diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go index 74a88f87..b91bb3c0 100644 --- a/cmd/soroban-rpc/internal/db/migration.go +++ b/cmd/soroban-rpc/internal/db/migration.go @@ -184,10 +184,23 @@ func (g *guardedMigration) Rollback(ctx context.Context) error { func BuildMigrations(ctx context.Context, logger *log.Entry, db *DB, cfg *config.Config) (Migration, error) { migrationName := "TransactionsTable" factory := newTransactionTableMigration(ctx, logger.WithField("migration", migrationName), cfg.TransactionLedgerRetentionWindow, cfg.NetworkPassphrase) - m, err := newGuardedDataMigration(ctx, migrationName, factory, db) + m1, err := newGuardedDataMigration(ctx, migrationName, factory, db) if err != nil { return nil, fmt.Errorf("creating guarded transaction migration: %w", err) } // Add other migrations here - return multiMigration{m}, nil + + eventMigrationName := "EventsTable" + eventFactory := newEventTableMigration( + ctx, + logger.WithField("migration", eventMigrationName), + cfg.EventLedgerRetentionWindow, + cfg.NetworkPassphrase, + ) + m2, err := newGuardedDataMigration(ctx, eventMigrationName, eventFactory, db) + if err != nil { + return nil, fmt.Errorf("creating guarded transaction migration: %w", err) + } + + return multiMigration{m1, m2}, nil } From 73ca5007e9d483cc93e8736f4f291100c6de5e8f Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 24 Jun 2024 23:02:55 -0700 Subject: [PATCH 25/63] Fix lint error part 5 --- cmd/soroban-rpc/internal/db/event.go | 6 +++++- cmd/soroban-rpc/internal/db/mocks.go | 8 ++++---- cmd/soroban-rpc/internal/methods/get_events.go | 4 ++-- cmd/soroban-rpc/internal/methods/get_events_test.go | 1 + 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index f08d7431..2644bcb6 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -2,6 +2,7 @@ package db import ( "context" + "errors" "fmt" "github.com/stellar/go/strkey" "io" @@ -47,7 +48,7 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { txCount := lcm.CountTransactions() if eventHandler.stmtCache == nil { - return fmt.Errorf("EventWriter incorrectly initialized without stmtCache") + return errors.New("EventWriter incorrectly initialized without stmtCache") } else if txCount == 0 { return nil } @@ -186,6 +187,9 @@ func (eventHandler *eventHandler) GetEvents( ledgerCloseTime := lcm.LedgerCloseTime() ledgerTx, err := reader.Read() + if err != nil { + return fmt.Errorf("failed reading tx: %w", err) + } transactionHash := ledgerTx.Result.TransactionHash diagEvents, diagErr := ledgerTx.GetDiagnosticEvents() diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index 8108f086..7f206d90 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -66,11 +66,11 @@ func (txn *MockTransactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) e } // GetLedgerRange pulls the min/max ledger sequence numbers from the database. -func (txn *MockTransactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { +func (txn *MockTransactionHandler) GetLedgerRange(_ context.Context) (ledgerbucketwindow.LedgerRange, error) { return txn.ledgerRange, nil } -func (txn *MockTransactionHandler) GetTransaction(ctx context.Context, hash xdr.Hash) ( +func (txn *MockTransactionHandler) GetTransaction(_ context.Context, hash xdr.Hash) ( Transaction, ledgerbucketwindow.LedgerRange, error, ) { if tx, ok := txn.txs[hash.HexString()]; !ok { @@ -93,7 +93,7 @@ func NewMockLedgerReader(txn *MockTransactionHandler) *MockLedgerReader { } } -func (m *MockLedgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) { +func (m *MockLedgerReader) GetLedger(_ context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) { lcm, ok := m.txn.ledgerSeqToMeta[sequence] if !ok { return xdr.LedgerCloseMeta{}, false, nil @@ -101,7 +101,7 @@ func (m *MockLedgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr. return *lcm, true, nil } -func (m *MockLedgerReader) StreamAllLedgers(ctx context.Context, f StreamLedgerFn) error { +func (m *MockLedgerReader) StreamAllLedgers(_ context.Context, f StreamLedgerFn) error { return nil } diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 1ef0a7c3..2acb7fed 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -104,7 +104,7 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { // Validate filters if len(g.Filters) > 5 { - return fmt.Errorf("maximum 5 filters per request") + return errors.New("maximum 5 filters per request") } for i, filter := range g.Filters { if err := filter.Valid(); err != nil { @@ -363,7 +363,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques end := events.Cursor{Ledger: request.StartLedger + ledgerbucketwindow.OneDayOfLedgers} cursorRange := events.CursorRange{Start: start, End: end} - // Check if requested start ledger is within stored ledger range: + // Check if requested start ledger is within stored ledger range if start.Ledger < ledgerRange.FirstLedger.Sequence || start.Ledger > ledgerRange.LastLedger.Sequence { return GetEventsResponse{}, &jrpc2.Error{ Code: jrpc2.InvalidRequest, diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 801f7a4a..7fda6a0a 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -918,6 +918,7 @@ func TestGetEvents(t *testing.T) { writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) write, err := writer.NewTx(ctx) + require.NoError(t, err) ledgerW, eventW := write.LedgerWriter(), write.EventWriter() store := db.NewEventReader(log, dbx, passphrase) From 3b9ce3495e0b77fbc11d7e2ad0c623cfd93f282a Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 24 Jun 2024 23:10:29 -0700 Subject: [PATCH 26/63] Fix lint error part 6 --- cmd/soroban-rpc/internal/db/event.go | 6 +++--- cmd/soroban-rpc/internal/db/mocks.go | 2 +- cmd/soroban-rpc/internal/methods/get_events.go | 2 -- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 2644bcb6..7668b503 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -4,10 +4,11 @@ import ( "context" "errors" "fmt" - "github.com/stellar/go/strkey" "io" "time" + "github.com/stellar/go/strkey" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" sq "github.com/Masterminds/squirrel" @@ -298,7 +299,7 @@ func (e *eventTableMigration) ApplicableRange() *LedgerSeqRange { } } -func (e *eventTableMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error { +func (e *eventTableMigration) Apply(_ context.Context, meta xdr.LedgerCloseMeta) error { return e.writer.InsertEvents(meta) } @@ -322,7 +323,6 @@ func newEventTableMigration( // Truncate the table, since it may contain data, causing insert conflicts later on. _, err := db.Exec(ctx, sq.Delete(transactionTableName).Where(sq.Lt{"ledger_sequence": firstLedgerToMigrate})) - if err != nil { return nil, fmt.Errorf("couldn't truncate the table %q: %w", transactionTableName, err) } diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index 7f206d90..5e81d623 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -101,7 +101,7 @@ func (m *MockLedgerReader) GetLedger(_ context.Context, sequence uint32) (xdr.Le return *lcm, true, nil } -func (m *MockLedgerReader) StreamAllLedgers(_ context.Context, f StreamLedgerFn) error { +func (m *MockLedgerReader) StreamAllLedgers(_ context.Context, _ StreamLedgerFn) error { return nil } diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 2acb7fed..cfca9a47 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -331,7 +331,6 @@ func combineContractIDs(filters []EventFilter) []string { } func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsRequest) (GetEventsResponse, error) { - if err := request.Valid(h.maxLimit); err != nil { return GetEventsResponse{}, &jrpc2.Error{ Code: jrpc2.InvalidParams, @@ -384,7 +383,6 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques // Scan function to apply filters f := func(event xdr.DiagnosticEvent, cursor events.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { - cursorID := cursor.String() if _, exists := cursorSet[cursorID]; exists { return true From 814b6ae3292fb3bec9f4bcd931253c6b2ac064bb Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 25 Jun 2024 15:43:58 -0700 Subject: [PATCH 27/63] Address review comments pt1 --- cmd/soroban-rpc/internal/db/event.go | 19 +++++++++---------- cmd/soroban-rpc/internal/db/migration.go | 2 -- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 7668b503..abbcff0a 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -113,7 +113,12 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { return nil } -type ScanFunction func(xdr.DiagnosticEvent, events.Cursor, int64, *xdr.Hash) bool +type ScanFunction func( + event xdr.DiagnosticEvent, + cursor events.Cursor, + ledgerCloseTimestamp int64, + txHash *xdr.Hash, +) bool // trimEvents removes all Events which fall outside the ledger retention window. func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWindow uint32) error { @@ -167,7 +172,7 @@ func (eventHandler *eventHandler) GetEvents( contractIDs, err) } else if len(rows) < 1 { - eventHandler.log.Infof("No events found for start ledger cursor= %v contractIDs= %v", + eventHandler.log.Debugf("No events found for start ledger cursor= %v contractIDs= %v", cursorRange.Start.String(), contractIDs, ) @@ -178,7 +183,7 @@ func (eventHandler *eventHandler) GetEvents( eventCursorID, txIndex, lcm := row.EventCursorID, row.TxIndex, row.Lcm reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) if err != nil { - return fmt.Errorf("failed to index to tx %d in ledger %d: %w", txIndex, lcm.LedgerSequence(), err) + return fmt.Errorf("failed to create ledger reader from LCM: %w", err) } err = reader.Seek(txIndex - 1) @@ -217,7 +222,7 @@ func (eventHandler *eventHandler) GetEvents( } // GetLedgerRange returns the min/max ledger sequence numbers from the events table -// TODO: Once we unify all the retention window we would keep only one GetLedgerRange logic +// TODO: Once we unify all the retention windows we should keep only one GetLedgerRange logic func (eventHandler *eventHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { var ledgerRange ledgerbucketwindow.LedgerRange @@ -304,7 +309,6 @@ func (e *eventTableMigration) Apply(_ context.Context, meta xdr.LedgerCloseMeta) } func newEventTableMigration( - ctx context.Context, logger *log.Entry, retentionWindow uint32, passphrase string, @@ -321,11 +325,6 @@ func newEventTableMigration( firstLedgerToMigrate = latestLedger - retentionWindow } - // Truncate the table, since it may contain data, causing insert conflicts later on. - _, err := db.Exec(ctx, sq.Delete(transactionTableName).Where(sq.Lt{"ledger_sequence": firstLedgerToMigrate})) - if err != nil { - return nil, fmt.Errorf("couldn't truncate the table %q: %w", transactionTableName, err) - } migration := eventTableMigration{ firstLedger: firstLedgerToMigrate, lastLedger: latestLedger, diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go index b91bb3c0..0fe742c0 100644 --- a/cmd/soroban-rpc/internal/db/migration.go +++ b/cmd/soroban-rpc/internal/db/migration.go @@ -188,11 +188,9 @@ func BuildMigrations(ctx context.Context, logger *log.Entry, db *DB, cfg *config if err != nil { return nil, fmt.Errorf("creating guarded transaction migration: %w", err) } - // Add other migrations here eventMigrationName := "EventsTable" eventFactory := newEventTableMigration( - ctx, logger.WithField("migration", eventMigrationName), cfg.EventLedgerRetentionWindow, cfg.NetworkPassphrase, From f3c9461c6c223e441908dcd2112428dbd463873c Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 25 Jun 2024 23:42:06 -0700 Subject: [PATCH 28/63] Make contract id a blob type --- cmd/soroban-rpc/internal/db/event.go | 10 ++++----- .../internal/db/sqlmigrations/03_events.sql | 2 +- .../internal/methods/get_events.go | 21 ++++++++++++++----- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index abbcff0a..e79367df 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -7,8 +7,6 @@ import ( "io" "time" - "github.com/stellar/go/strkey" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" sq "github.com/Masterminds/squirrel" @@ -29,7 +27,7 @@ type EventWriter interface { // EventReader has all the public methods to fetch events from DB type EventReader interface { - GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIDs []string, f ScanFunction) error + GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIDs [][]byte, f ScanFunction) error GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) } @@ -96,9 +94,9 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { Columns("id", "ledger_sequence", "application_order", "contract_id", "event_type") for index, e := range txEvents { - var contractID string + var contractID []byte if e.Event.ContractId != nil { - contractID = strkey.MustEncode(strkey.VersionByteContract, (*e.Event.ContractId)[:]) + contractID = e.Event.ContractId[:] } id := events.Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() query = query.Values(id, lcm.LedgerSequence(), tx.Index, contractID, int(e.Event.Type)) @@ -142,7 +140,7 @@ func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWi func (eventHandler *eventHandler) GetEvents( ctx context.Context, cursorRange events.CursorRange, - contractIDs []string, + contractIDs [][]byte, f ScanFunction, ) error { start := time.Now() diff --git a/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql b/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql index 7a81d0af..5e698bc9 100644 --- a/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql +++ b/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql @@ -5,7 +5,7 @@ CREATE TABLE events( id TEXT PRIMARY KEY, ledger_sequence INTEGER NOT NULL, application_order INTEGER NOT NULL, - contract_id TEXT, + contract_id BLOB(32), event_type INTEGER NOT NULL ); diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index cfca9a47..cd683441 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -314,7 +314,7 @@ type eventsRPCHandler struct { networkPassphrase string } -func combineContractIDs(filters []EventFilter) []string { +func combineContractIDs(filters []EventFilter) ([][]byte, error) { contractIDSet := make(map[string]struct{}) for _, filter := range filters { @@ -323,11 +323,16 @@ func combineContractIDs(filters []EventFilter) []string { } } - contractIDs := make([]string, 0, len(contractIDSet)) + contractIDs := make([][]byte, 0, len(contractIDSet)) for contractID := range contractIDSet { - contractIDs = append(contractIDs, contractID) + id, err := strkey.Decode(strkey.VersionByteContract, contractID) + if err != nil { + return nil, fmt.Errorf("contract ID %v invalid", contractID) + } + + contractIDs = append(contractIDs, id) } - return contractIDs + return contractIDs, nil } func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsRequest) (GetEventsResponse, error) { @@ -379,7 +384,13 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques var found []entry cursorSet := make(map[string]struct{}) - contractIDs := combineContractIDs(request.Filters) + contractIDs, err := combineContractIDs(request.Filters) + if err != nil { + return GetEventsResponse{}, &jrpc2.Error{ + Code: jrpc2.InvalidParams, + Message: err.Error(), + } + } // Scan function to apply filters f := func(event xdr.DiagnosticEvent, cursor events.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { From 5709a8f6416b54f73fc94e9be1195c391570d6c1 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 25 Jun 2024 23:48:31 -0700 Subject: [PATCH 29/63] Remove events package and move cursor.go to db package --- .../internal/{events => db}/cursor.go | 2 +- .../internal/{events => db}/cursor_test.go | 2 +- cmd/soroban-rpc/internal/db/event.go | 11 +++--- .../internal/db/transaction_test.go | 8 ++-- .../internal/methods/get_events.go | 18 ++++----- .../internal/methods/get_events_test.go | 37 +++++++++---------- 6 files changed, 35 insertions(+), 43 deletions(-) rename cmd/soroban-rpc/internal/{events => db}/cursor.go (99%) rename cmd/soroban-rpc/internal/{events => db}/cursor_test.go (99%) diff --git a/cmd/soroban-rpc/internal/events/cursor.go b/cmd/soroban-rpc/internal/db/cursor.go similarity index 99% rename from cmd/soroban-rpc/internal/events/cursor.go rename to cmd/soroban-rpc/internal/db/cursor.go index 290ea457..7d009df6 100644 --- a/cmd/soroban-rpc/internal/events/cursor.go +++ b/cmd/soroban-rpc/internal/db/cursor.go @@ -1,4 +1,4 @@ -package events +package db import ( "encoding/json" diff --git a/cmd/soroban-rpc/internal/events/cursor_test.go b/cmd/soroban-rpc/internal/db/cursor_test.go similarity index 99% rename from cmd/soroban-rpc/internal/events/cursor_test.go rename to cmd/soroban-rpc/internal/db/cursor_test.go index 6dfe1e58..b081a98b 100644 --- a/cmd/soroban-rpc/internal/events/cursor_test.go +++ b/cmd/soroban-rpc/internal/db/cursor_test.go @@ -1,4 +1,4 @@ -package events +package db import ( "encoding/json" diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index e79367df..62a63200 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -15,7 +15,6 @@ import ( "github.com/stellar/go/support/db" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" ) const eventTableName = "events" @@ -27,7 +26,7 @@ type EventWriter interface { // EventReader has all the public methods to fetch events from DB type EventReader interface { - GetEvents(ctx context.Context, cursorRange events.CursorRange, contractIDs [][]byte, f ScanFunction) error + GetEvents(ctx context.Context, cursorRange CursorRange, contractIDs [][]byte, f ScanFunction) error GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) } @@ -98,7 +97,7 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { if e.Event.ContractId != nil { contractID = e.Event.ContractId[:] } - id := events.Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() + id := Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() query = query.Values(id, lcm.LedgerSequence(), tx.Index, contractID, int(e.Event.Type)) } @@ -113,7 +112,7 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { type ScanFunction func( event xdr.DiagnosticEvent, - cursor events.Cursor, + cursor Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash, ) bool @@ -139,7 +138,7 @@ func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWi // remaining events in the range). func (eventHandler *eventHandler) GetEvents( ctx context.Context, - cursorRange events.CursorRange, + cursorRange CursorRange, contractIDs [][]byte, f ScanFunction, ) error { @@ -203,7 +202,7 @@ func (eventHandler *eventHandler) GetEvents( // Find events based on filter passed in function f for eventIndex, event := range diagEvents { - cur := events.Cursor{Ledger: lcm.LedgerSequence(), Tx: uint32(txIndex), Event: uint32(eventIndex)} + cur := Cursor{Ledger: lcm.LedgerSequence(), Tx: uint32(txIndex), Event: uint32(eventIndex)} if !f(event, cur, ledgerCloseTime, &transactionHash) { return nil } diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index efcb432c..7d50834e 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -6,8 +6,6 @@ import ( "math/rand" "testing" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" - "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -97,9 +95,9 @@ func TestTransactionFound(t *testing.T) { require.Error(t, err, ErrNoTransaction) eventReader := NewEventReader(log, db, passphrase) - start := events.Cursor{Ledger: 1} - end := events.Cursor{Ledger: 1000} - cursorRange := events.CursorRange{Start: start, End: end} + start := Cursor{Ledger: 1} + end := Cursor{Ledger: 1000} + cursorRange := CursorRange{Start: start, End: end} err = eventReader.GetEvents(ctx, cursorRange, nil, nil) require.NoError(t, err) diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index cd683441..3c4a0c97 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -16,8 +16,6 @@ import ( "github.com/stellar/go/strkey" "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" - - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" ) type eventTypeSet map[string]interface{} @@ -297,8 +295,8 @@ func (s *SegmentFilter) UnmarshalJSON(p []byte) error { } type PaginationOptions struct { - Cursor *events.Cursor `json:"cursor,omitempty"` - Limit uint `json:"limit,omitempty"` + Cursor *db.Cursor `json:"cursor,omitempty"` + Limit uint `json:"limit,omitempty"` } type GetEventsResponse struct { @@ -351,7 +349,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } } - start := events.Cursor{Ledger: request.StartLedger} + start := db.Cursor{Ledger: request.StartLedger} limit := h.defaultLimit if request.Pagination != nil { if request.Pagination.Cursor != nil { @@ -364,8 +362,8 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques limit = request.Pagination.Limit } } - end := events.Cursor{Ledger: request.StartLedger + ledgerbucketwindow.OneDayOfLedgers} - cursorRange := events.CursorRange{Start: start, End: end} + end := db.Cursor{Ledger: request.StartLedger + ledgerbucketwindow.OneDayOfLedgers} + cursorRange := db.CursorRange{Start: start, End: end} // Check if requested start ledger is within stored ledger range if start.Ledger < ledgerRange.FirstLedger.Sequence || start.Ledger > ledgerRange.LastLedger.Sequence { @@ -376,7 +374,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } type entry struct { - cursor events.Cursor + cursor db.Cursor ledgerCloseTimestamp int64 event xdr.DiagnosticEvent txHash *xdr.Hash @@ -393,7 +391,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } // Scan function to apply filters - f := func(event xdr.DiagnosticEvent, cursor events.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { + f := func(event xdr.DiagnosticEvent, cursor db.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { cursorID := cursor.String() if _, exists := cursorSet[cursorID]; exists { return true @@ -436,7 +434,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques func eventInfoForEvent( event xdr.DiagnosticEvent, - cursor events.Cursor, + cursor db.Cursor, ledgerClosedAt string, txHash string, ) (EventInfo, error) { diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 7fda6a0a..3ec7b4bf 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -4,15 +4,15 @@ import ( "context" "encoding/json" "fmt" + "strings" + "testing" + "time" + "github.com/sirupsen/logrus" "github.com/stellar/go/support/log" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/events" "github.com/stretchr/testify/require" - "strings" - "testing" - "time" "github.com/stretchr/testify/assert" @@ -416,7 +416,7 @@ func TestGetEventsRequestValid(t *testing.T) { assert.EqualError(t, (&GetEventsRequest{ StartLedger: 1, Filters: []EventFilter{}, - Pagination: &PaginationOptions{Cursor: &events.Cursor{}}, + Pagination: &PaginationOptions{Cursor: &db.Cursor{}}, }).Valid(1000), "startLedger and cursor cannot both be set") assert.NoError(t, (&GetEventsRequest{ @@ -580,7 +580,6 @@ func TestGetEvents(t *testing.T) { }) t.Run("no filtering returns all", func(t *testing.T) { - dbx := db.NewTestDB(t) ctx := context.TODO() log := log.DefaultLogger @@ -628,7 +627,7 @@ func TestGetEvents(t *testing.T) { var expected []EventInfo for i := range txMeta { - id := events.Cursor{ + id := db.Cursor{ Ledger: 1, Tx: uint32(i + 1), Op: 0, @@ -656,7 +655,6 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by contract id", func(t *testing.T) { - dbx := db.NewTestDB(t) ctx := context.TODO() log := log.DefaultLogger @@ -710,9 +708,9 @@ func TestGetEvents(t *testing.T) { assert.Equal(t, uint32(1), results.LatestLedger) expectedIds := []string{ - events.Cursor{Ledger: 1, Tx: 1, Op: 0, Event: 0}.String(), - events.Cursor{Ledger: 1, Tx: 3, Op: 0, Event: 0}.String(), - events.Cursor{Ledger: 1, Tx: 5, Op: 0, Event: 0}.String(), + db.Cursor{Ledger: 1, Tx: 1, Op: 0, Event: 0}.String(), + db.Cursor{Ledger: 1, Tx: 3, Op: 0, Event: 0}.String(), + db.Cursor{Ledger: 1, Tx: 5, Op: 0, Event: 0}.String(), } eventIds := []string{} for _, event := range results.Events { @@ -775,7 +773,7 @@ func TestGetEvents(t *testing.T) { }) assert.NoError(t, err) - id := events.Cursor{Ledger: 1, Tx: 5, Op: 0, Event: 0}.String() + id := db.Cursor{Ledger: 1, Tx: 5, Op: 0, Event: 0}.String() assert.NoError(t, err) value, err := xdr.MarshalBase64(xdr.ScVal{ Type: xdr.ScValTypeScvU64, @@ -800,7 +798,6 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by both contract id and topic", func(t *testing.T) { - dbx := db.NewTestDB(t) ctx := context.TODO() log := log.DefaultLogger @@ -887,7 +884,7 @@ func TestGetEvents(t *testing.T) { }) assert.NoError(t, err) - id := events.Cursor{Ledger: 1, Tx: 4, Op: 0, Event: 0}.String() + id := db.Cursor{Ledger: 1, Tx: 4, Op: 0, Event: 0}.String() value, err := xdr.MarshalBase64(xdr.ScVal{ Type: xdr.ScValTypeScvU64, U64: &number, @@ -966,7 +963,7 @@ func TestGetEvents(t *testing.T) { }) assert.NoError(t, err) - id := events.Cursor{Ledger: 1, Tx: 1, Op: 0, Event: 1}.String() + id := db.Cursor{Ledger: 1, Tx: 1, Op: 0, Event: 1}.String() expected := []EventInfo{ { EventType: EventTypeSystem, @@ -1030,7 +1027,7 @@ func TestGetEvents(t *testing.T) { var expected []EventInfo for i := 0; i < 10; i++ { - id := events.Cursor{ + id := db.Cursor{ Ledger: 1, Tx: uint32(i + 1), Op: 0, @@ -1114,7 +1111,7 @@ func TestGetEvents(t *testing.T) { require.NoError(t, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") require.NoError(t, write.Commit(4)) - id := &events.Cursor{Ledger: 5, Tx: 1, Op: 0, Event: 0} + id := &db.Cursor{Ledger: 5, Tx: 1, Op: 0, Event: 0} handler := eventsRPCHandler{ dbReader: store, maxLimit: 10000, @@ -1130,8 +1127,8 @@ func TestGetEvents(t *testing.T) { var expected []EventInfo expectedIDs := []string{ - events.Cursor{Ledger: 5, Tx: 1, Op: 0, Event: 1}.String(), - events.Cursor{Ledger: 5, Tx: 2, Op: 0, Event: 0}.String(), + db.Cursor{Ledger: 5, Tx: 1, Op: 0, Event: 1}.String(), + db.Cursor{Ledger: 5, Tx: 2, Op: 0, Event: 0}.String(), } symbols := datas[1:3] for i, id := range expectedIDs { @@ -1154,7 +1151,7 @@ func TestGetEvents(t *testing.T) { results, err = handler.getEvents(context.TODO(), GetEventsRequest{ Pagination: &PaginationOptions{ - Cursor: &events.Cursor{Ledger: 5, Tx: 2, Op: 0, Event: 1}, + Cursor: &db.Cursor{Ledger: 5, Tx: 2, Op: 0, Event: 1}, Limit: 2, }, }) From a28b57f4873ba79bf418e2e98e621cffc791fd4e Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Wed, 26 Jun 2024 16:19:38 -0700 Subject: [PATCH 30/63] Fix lint error part 6 --- cmd/soroban-rpc/internal/db/db.go | 20 ++-------- cmd/soroban-rpc/internal/db/event.go | 5 ++- cmd/soroban-rpc/internal/db/event_test.go | 13 ++++--- cmd/soroban-rpc/internal/db/ledger_test.go | 13 +++++++ .../internal/methods/get_events.go | 8 ++-- .../internal/methods/get_events_test.go | 38 ++++++++++++------- 6 files changed, 56 insertions(+), 41 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go index 3ff55517..ccdf5c14 100644 --- a/cmd/soroban-rpc/internal/db/db.go +++ b/cmd/soroban-rpc/internal/db/db.go @@ -6,16 +6,13 @@ import ( "embed" "errors" "fmt" + "strconv" + "sync" + sq "github.com/Masterminds/squirrel" _ "github.com/mattn/go-sqlite3" "github.com/prometheus/client_golang/prometheus" migrate "github.com/rubenv/sql-migrate" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "path" - "strconv" - "sync" - "testing" "github.com/stellar/go/support/db" "github.com/stellar/go/support/log" @@ -373,14 +370,3 @@ func runSQLMigrations(db *sql.DB, dialect string) error { _, err := migrate.ExecMax(db, dialect, m, migrate.Up, 0) return err } - -func NewTestDB(tb testing.TB) *DB { - tmp := tb.TempDir() - dbPath := path.Join(tmp, "db.sqlite") - db, err := OpenSQLiteDB(dbPath) - require.NoError(tb, err) - tb.Cleanup(func() { - assert.NoError(tb, db.Close()) - }) - return db -} diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 62a63200..f1c1ad2f 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -7,14 +7,15 @@ import ( "io" "time" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" - sq "github.com/Masterminds/squirrel" "github.com/prometheus/client_golang/prometheus" + "github.com/stellar/go/ingest" "github.com/stellar/go/support/db" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" + + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) const eventTableName = "events" diff --git a/cmd/soroban-rpc/internal/db/event_test.go b/cmd/soroban-rpc/internal/db/event_test.go index eb272ddb..60ad439a 100644 --- a/cmd/soroban-rpc/internal/db/event_test.go +++ b/cmd/soroban-rpc/internal/db/event_test.go @@ -2,16 +2,19 @@ package db import ( "context" + "testing" + "time" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stellar/go/keypair" "github.com/stellar/go/network" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "testing" - "time" ) func transactionMetaWithEvents(events ...xdr.ContractEvent) xdr.TransactionMeta { @@ -163,5 +166,5 @@ func TestInsertEvents(t *testing.T) { err = eventW.InsertEvents(ledgerCloseMeta) assert.NoError(t, err) - //TODO: Call getEvents and validate events data. + // TODO: Call getEvents and validate events data. } diff --git a/cmd/soroban-rpc/internal/db/ledger_test.go b/cmd/soroban-rpc/internal/db/ledger_test.go index 2578dcf8..a40bc255 100644 --- a/cmd/soroban-rpc/internal/db/ledger_test.go +++ b/cmd/soroban-rpc/internal/db/ledger_test.go @@ -2,9 +2,11 @@ package db import ( "context" + "path" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stellar/go/network" "github.com/stellar/go/support/log" @@ -103,3 +105,14 @@ func TestLedgers(t *testing.T) { assertLedgerRange(t, reader, 8, 12) } + +func NewTestDB(tb testing.TB) *DB { + tmp := tb.TempDir() + dbPath := path.Join(tmp, "db.sqlite") + db, err := OpenSQLiteDB(dbPath) + require.NoError(tb, err) + tb.Cleanup(func() { + assert.NoError(tb, db.Close()) + }) + return db +} diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 3c4a0c97..216fb12f 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -7,15 +7,15 @@ import ( "strings" "time" - "github.com/stellar/go/support/log" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" - "github.com/creachadair/jrpc2" "github.com/stellar/go/strkey" "github.com/stellar/go/support/errors" + "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" + + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) type eventTypeSet map[string]interface{} diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 3ec7b4bf..de47941a 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -4,22 +4,23 @@ import ( "context" "encoding/json" "fmt" + "path" "strings" "testing" "time" "github.com/sirupsen/logrus" - "github.com/stellar/go/support/log" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" - "github.com/stretchr/testify/require" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stellar/go/keypair" "github.com/stellar/go/network" "github.com/stellar/go/strkey" + "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" + + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" ) var passphrase = "passphrase" @@ -532,7 +533,7 @@ func TestGetEvents(t *testing.T) { t.Run("startLedger validation", func(t *testing.T) { contractID := xdr.Hash([32]byte{}) - dbx := db.NewTestDB(t) + dbx := newTestDB(t) ctx := context.TODO() log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) @@ -580,7 +581,7 @@ func TestGetEvents(t *testing.T) { }) t.Run("no filtering returns all", func(t *testing.T) { - dbx := db.NewTestDB(t) + dbx := newTestDB(t) ctx := context.TODO() log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) @@ -655,7 +656,7 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by contract id", func(t *testing.T) { - dbx := db.NewTestDB(t) + dbx := newTestDB(t) ctx := context.TODO() log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) @@ -720,7 +721,7 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by topic", func(t *testing.T) { - dbx := db.NewTestDB(t) + dbx := newTestDB(t) ctx := context.TODO() log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) @@ -798,7 +799,7 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by both contract id and topic", func(t *testing.T) { - dbx := db.NewTestDB(t) + dbx := newTestDB(t) ctx := context.TODO() log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) @@ -908,7 +909,7 @@ func TestGetEvents(t *testing.T) { }) t.Run("filtering by event type", func(t *testing.T) { - dbx := db.NewTestDB(t) + dbx := newTestDB(t) ctx := context.TODO() log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) @@ -982,7 +983,7 @@ func TestGetEvents(t *testing.T) { }) t.Run("with limit", func(t *testing.T) { - dbx := db.NewTestDB(t) + dbx := newTestDB(t) ctx := context.TODO() log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) @@ -1052,7 +1053,7 @@ func TestGetEvents(t *testing.T) { }) t.Run("with cursor", func(t *testing.T) { - dbx := db.NewTestDB(t) + dbx := newTestDB(t) ctx := context.TODO() log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) @@ -1308,3 +1309,14 @@ func diagnosticEvent(contractID xdr.Hash, topic []xdr.ScVal, body xdr.ScVal) xdr }, } } + +func newTestDB(tb testing.TB) *db.DB { + tmp := tb.TempDir() + dbPath := path.Join(tmp, "db.sqlite") + db, err := db.OpenSQLiteDB(dbPath) + require.NoError(tb, err) + tb.Cleanup(func() { + assert.NoError(tb, db.Close()) + }) + return db +} From 952c0e39bf7ddcbcb02a1ff6da06913bacefa549 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Wed, 26 Jun 2024 16:58:41 -0700 Subject: [PATCH 31/63] Fix lint error part 7 --- cmd/soroban-rpc/internal/db/event.go | 76 ++----------------- cmd/soroban-rpc/internal/jsonrpc.go | 11 ++- .../internal/methods/get_events.go | 8 +- 3 files changed, 23 insertions(+), 72 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index f1c1ad2f..a673d427 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -18,7 +18,10 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) -const eventTableName = "events" +const ( + eventTableName = "events" + firstLedger = uint32(2) +) // EventWriter is used during ingestion of events from LCM to DB type EventWriter interface { @@ -220,73 +223,10 @@ func (eventHandler *eventHandler) GetEvents( } // GetLedgerRange returns the min/max ledger sequence numbers from the events table -// TODO: Once we unify all the retention windows we should keep only one GetLedgerRange logic - -func (eventHandler *eventHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { - var ledgerRange ledgerbucketwindow.LedgerRange - - // - // We use subqueries alongside a UNION ALL stitch in order to select the min - // and max from the ledger table in a single query and get around sqlite's - // limitations with parentheses (see https://stackoverflow.com/a/22609948). - // - // Queries to get the minimum and maximum ledger sequence from the transactions table - minLedgerSeqQ := sq. - Select("m1.ledger_sequence"). - FromSelect( - sq. - Select("ledger_sequence"). - From(eventTableName). - OrderBy("ledger_sequence ASC"). - Limit(1), - "m1", - ) - maxLedgerSeqQ, args, err := sq. - Select("m2.ledger_sequence"). - FromSelect( - sq. - Select("ledger_sequence"). - From(eventTableName). - OrderBy("ledger_sequence DESC"). - Limit(1), - "m2", - ).ToSql() - if err != nil { - return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) - } - - // Combine the min and max ledger sequence queries using UNION ALL - eventMinMaxLedgersQ, _, err := minLedgerSeqQ.Suffix("UNION ALL "+maxLedgerSeqQ, args...).ToSql() - if err != nil { - return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) - } - - // Final query to join ledger_close_meta table and the sequence numbers we got from eventMinMaxLedgersQ - finalSQL := sq. - Select("lcm.meta"). - From(ledgerCloseMetaTableName + " as lcm"). - JoinClause(fmt.Sprintf("JOIN (%s) as seqs ON lcm.sequence == seqs.ledger_sequence", eventMinMaxLedgersQ)) - - var lcms []xdr.LedgerCloseMeta - if err = eventHandler.db.Select(ctx, &lcms, finalSQL); err != nil { - return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) - } else if len(lcms) < 2 { - // There is almost certainly a row, but we want to avoid a race condition - // with ingestion as well as support test cases from an empty DB, so we need - // to sanity check that there is in fact a result. Note that no ledgers in - // the database isn't an error, it's just an empty range. - return ledgerRange, nil - } - - lcm1, lcm2 := lcms[0], lcms[1] - ledgerRange.FirstLedger.Sequence = lcm1.LedgerSequence() - ledgerRange.FirstLedger.CloseTime = lcm1.LedgerCloseTime() - ledgerRange.LastLedger.Sequence = lcm2.LedgerSequence() - ledgerRange.LastLedger.CloseTime = lcm2.LedgerCloseTime() - eventHandler.log.Debugf("Database ledger range: [%d, %d]", - ledgerRange.FirstLedger.Sequence, ledgerRange.LastLedger.Sequence) - return ledgerRange, nil +func (eventHandler *eventHandler) GetLedgerRange(_ context.Context) (ledgerbucketwindow.LedgerRange, error) { + // TODO: Once we unify all the retention windows use common GetLedgerRange from LedgerReader + return ledgerbucketwindow.LedgerRange{}, nil } type eventTableMigration struct { @@ -312,7 +252,7 @@ func newEventTableMigration( passphrase string, ) migrationApplierFactory { return migrationApplierFactoryF(func(db *DB, latestLedger uint32) (MigrationApplier, error) { - firstLedgerToMigrate := uint32(2) + firstLedgerToMigrate := firstLedger writer := &eventHandler{ log: logger, db: db, diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index 2ea4968e..65b15c62 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -157,8 +157,15 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { requestDurationLimit: cfg.MaxGetHealthExecutionDuration, }, { - methodName: "getEvents", - underlyingHandler: methods.NewGetEventsHandler(params.Logger, params.EventReader, cfg.MaxEventsLimit, cfg.DefaultEventsLimit, cfg.NetworkPassphrase), + methodName: "getEvents", + underlyingHandler: methods.NewGetEventsHandler( + params.Logger, + params.EventReader, + cfg.MaxEventsLimit, + cfg.DefaultEventsLimit, + cfg.NetworkPassphrase, + ), + longName: "get_events", queueLimit: cfg.RequestBacklogGetEventsQueueLimit, requestDurationLimit: cfg.MaxGetEventsExecutionDuration, diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 216fb12f..d8027d10 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -368,8 +368,12 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques // Check if requested start ledger is within stored ledger range if start.Ledger < ledgerRange.FirstLedger.Sequence || start.Ledger > ledgerRange.LastLedger.Sequence { return GetEventsResponse{}, &jrpc2.Error{ - Code: jrpc2.InvalidRequest, - Message: fmt.Sprintf("startLedger must be within the ledger range: %d - %d", ledgerRange.FirstLedger.Sequence, ledgerRange.LastLedger.Sequence), + Code: jrpc2.InvalidRequest, + Message: fmt.Sprintf( + "startLedger must be within the ledger range: %d - %d", + ledgerRange.FirstLedger.Sequence, + ledgerRange.LastLedger.Sequence, + ), } } From 263e6f7ab2af624cafe63cdd4092b9351669c02f Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Fri, 28 Jun 2024 12:08:49 -0700 Subject: [PATCH 32/63] Optimize migration code --- cmd/soroban-rpc/internal/daemon/daemon.go | 22 ++++-- cmd/soroban-rpc/internal/db/event.go | 71 +++++++++++++++++-- cmd/soroban-rpc/internal/db/ledger.go | 30 ++++++++ cmd/soroban-rpc/internal/db/migration.go | 24 +++++-- cmd/soroban-rpc/internal/db/mocks.go | 4 ++ cmd/soroban-rpc/internal/db/transaction.go | 4 +- .../methods/get_latest_ledger_test.go | 4 ++ 7 files changed, 139 insertions(+), 20 deletions(-) diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index 63d6bc56..f3bcfd58 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -314,10 +314,19 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) *feewindow.FeeWindows if err != nil { d.logger.WithError(err).Fatal("could not build migrations") } - // NOTE: We could optimize this to avoid unnecessary ingestion calls - // (the range of txmetas can be larger than the individual store retention windows) - // but it's probably not worth the pain. - err = db.NewLedgerReader(d.db).StreamAllLedgers(readTxMetaCtx, func(txmeta xdr.LedgerCloseMeta) error { + + // Merge migrations range and fee stats range to get the applicable range + latestLedger, err := db.NewLedgerEntryReader(d.db).GetLatestLedgerSequence(readTxMetaCtx) + if err != nil { + d.logger.WithError(err).Fatal("failed to get latest ledger sequence: %w", err) + } + + maxFeeRetentionWindow := max(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow) + ledgerSeqRange := &db.LedgerSeqRange{FirstLedgerSeq: latestLedger - maxFeeRetentionWindow, LastLedgerSeq: latestLedger} + applicableRange := dataMigrations.ApplicableRange() + ledgerSeqRange = ledgerSeqRange.Merge(applicableRange) + + err = db.NewLedgerReader(d.db).StreamLedgerRange(readTxMetaCtx, ledgerSeqRange.FirstLedgerSeq, ledgerSeqRange.LastLedgerSeq, func(txmeta xdr.LedgerCloseMeta) error { currentSeq = txmeta.LedgerSequence() if initialSeq == 0 { initialSeq = currentSeq @@ -333,9 +342,8 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) *feewindow.FeeWindows if err := feewindows.IngestFees(txmeta); err != nil { d.logger.WithError(err).Fatal("could not initialize fee stats") } - // TODO: clean up once we remove the in-memory storage. - // (we should only stream over the required range) - if r := dataMigrations.ApplicableRange(); r.IsLedgerIncluded(currentSeq) { + + if applicableRange.IsLedgerIncluded(currentSeq) { if err := dataMigrations.Apply(readTxMetaCtx, txmeta); err != nil { d.logger.WithError(err).Fatal("could not run migrations") } diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index a673d427..3a1d6a8e 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -224,9 +224,72 @@ func (eventHandler *eventHandler) GetEvents( // GetLedgerRange returns the min/max ledger sequence numbers from the events table -func (eventHandler *eventHandler) GetLedgerRange(_ context.Context) (ledgerbucketwindow.LedgerRange, error) { +func (eventHandler *eventHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { // TODO: Once we unify all the retention windows use common GetLedgerRange from LedgerReader - return ledgerbucketwindow.LedgerRange{}, nil + var ledgerRange ledgerbucketwindow.LedgerRange + + // + // We use subqueries alongside a UNION ALL stitch in order to select the min + // and max from the ledger table in a single query and get around sqlite's + // limitations with parentheses (see https://stackoverflow.com/a/22609948). + // + // Queries to get the minimum and maximum ledger sequence from the transactions table + minLedgerSeqQ := sq. + Select("m1.ledger_sequence"). + FromSelect( + sq. + Select("ledger_sequence"). + From(eventTableName). + OrderBy("ledger_sequence ASC"). + Limit(1), + "m1", + ) + maxLedgerSeqQ, args, err := sq. + Select("m2.ledger_sequence"). + FromSelect( + sq. + Select("ledger_sequence"). + From(eventTableName). + OrderBy("ledger_sequence DESC"). + Limit(1), + "m2", + ).ToSql() + if err != nil { + return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) + } + + // Combine the min and max ledger sequence queries using UNION ALL + eventMinMaxLedgersQ, _, err := minLedgerSeqQ.Suffix("UNION ALL "+maxLedgerSeqQ, args...).ToSql() + if err != nil { + return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) + } + + // Final query to join ledger_close_meta table and the sequence numbers we got from eventMinMaxLedgersQ + finalSQL := sq. + Select("lcm.meta"). + From(ledgerCloseMetaTableName + " as lcm"). + JoinClause(fmt.Sprintf("JOIN (%s) as seqs ON lcm.sequence == seqs.ledger_sequence", eventMinMaxLedgersQ)) + + var lcms []xdr.LedgerCloseMeta + if err = eventHandler.db.Select(ctx, &lcms, finalSQL); err != nil { + return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) + } else if len(lcms) < 2 { + // There is almost certainly a row, but we want to avoid a race condition + // with ingestion as well as support test cases from an empty DB, so we need + // to sanity check that there is in fact a result. Note that no ledgers in + // the database isn't an error, it's just an empty range. + return ledgerRange, nil + } + + lcm1, lcm2 := lcms[0], lcms[1] + ledgerRange.FirstLedger.Sequence = lcm1.LedgerSequence() + ledgerRange.FirstLedger.CloseTime = lcm1.LedgerCloseTime() + ledgerRange.LastLedger.Sequence = lcm2.LedgerSequence() + ledgerRange.LastLedger.CloseTime = lcm2.LedgerCloseTime() + + eventHandler.log.Debugf("Database ledger range: [%d, %d]", + ledgerRange.FirstLedger.Sequence, ledgerRange.LastLedger.Sequence) + return ledgerRange, nil } type eventTableMigration struct { @@ -237,8 +300,8 @@ type eventTableMigration struct { func (e *eventTableMigration) ApplicableRange() *LedgerSeqRange { return &LedgerSeqRange{ - firstLedgerSeq: e.firstLedger, - lastLedgerSeq: e.lastLedger, + FirstLedgerSeq: e.firstLedger, + LastLedgerSeq: e.lastLedger, } } diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index 97887281..c8b65846 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -18,6 +18,7 @@ type StreamLedgerFn func(xdr.LedgerCloseMeta) error type LedgerReader interface { GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) StreamAllLedgers(ctx context.Context, f StreamLedgerFn) error + StreamLedgerRange(ctx context.Context, startLedger uint32, endLedger uint32, f StreamLedgerFn) error } type LedgerWriter interface { @@ -52,6 +53,35 @@ func (r ledgerReader) StreamAllLedgers(ctx context.Context, f StreamLedgerFn) er return q.Err() } +// StreamLedgerRange runs f over inclusive (startLedger, endLedger) (until f errors or signals it's done). +func (r ledgerReader) StreamLedgerRange( + ctx context.Context, + startLedger uint32, + endLedger uint32, + f StreamLedgerFn, +) error { + sql := sq.Select("meta").From(ledgerCloseMetaTableName). + Where(sq.GtOrEq{"sequence": startLedger}). + Where(sq.LtOrEq{"sequence": endLedger}). + OrderBy("sequence asc") + + q, err := r.db.Query(ctx, sql) + if err != nil { + return err + } + defer q.Close() + for q.Next() { + var closeMeta xdr.LedgerCloseMeta + if err = q.Scan(&closeMeta); err != nil { + return err + } + if err = f(closeMeta); err != nil { + return err + } + } + return q.Err() +} + // GetLedger fetches a single ledger from the db. func (r ledgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) { sql := sq.Select("meta").From(ledgerCloseMetaTableName).Where(sq.Eq{"sequence": sequence}) diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go index 0fe742c0..5461040c 100644 --- a/cmd/soroban-rpc/internal/db/migration.go +++ b/cmd/soroban-rpc/internal/db/migration.go @@ -12,15 +12,15 @@ import ( ) type LedgerSeqRange struct { - firstLedgerSeq uint32 - lastLedgerSeq uint32 + FirstLedgerSeq uint32 + LastLedgerSeq uint32 } func (mlr *LedgerSeqRange) IsLedgerIncluded(ledgerSeq uint32) bool { if mlr == nil { return false } - return ledgerSeq >= mlr.firstLedgerSeq && ledgerSeq <= mlr.lastLedgerSeq + return ledgerSeq >= mlr.FirstLedgerSeq && ledgerSeq <= mlr.LastLedgerSeq } func (mlr *LedgerSeqRange) Merge(other *LedgerSeqRange) *LedgerSeqRange { @@ -33,8 +33,8 @@ func (mlr *LedgerSeqRange) Merge(other *LedgerSeqRange) *LedgerSeqRange { // TODO: using min/max can result in a much larger range than needed, // as an optimization, we should probably use a sequence of ranges instead. return &LedgerSeqRange{ - firstLedgerSeq: min(mlr.firstLedgerSeq, other.firstLedgerSeq), - lastLedgerSeq: max(mlr.lastLedgerSeq, other.lastLedgerSeq), + FirstLedgerSeq: min(mlr.FirstLedgerSeq, other.FirstLedgerSeq), + LastLedgerSeq: max(mlr.LastLedgerSeq, other.LastLedgerSeq), } } @@ -182,12 +182,21 @@ func (g *guardedMigration) Rollback(ctx context.Context) error { } func BuildMigrations(ctx context.Context, logger *log.Entry, db *DB, cfg *config.Config) (Migration, error) { + var migrations []Migration + migrationName := "TransactionsTable" - factory := newTransactionTableMigration(ctx, logger.WithField("migration", migrationName), cfg.TransactionLedgerRetentionWindow, cfg.NetworkPassphrase) + factory := newTransactionTableMigration( + ctx, + logger.WithField("migration", migrationName), + cfg.TransactionLedgerRetentionWindow, + cfg.NetworkPassphrase, + ) + m1, err := newGuardedDataMigration(ctx, migrationName, factory, db) if err != nil { return nil, fmt.Errorf("creating guarded transaction migration: %w", err) } + migrations = append(migrations, m1) eventMigrationName := "EventsTable" eventFactory := newEventTableMigration( @@ -199,6 +208,7 @@ func BuildMigrations(ctx context.Context, logger *log.Entry, db *DB, cfg *config if err != nil { return nil, fmt.Errorf("creating guarded transaction migration: %w", err) } + migrations = append(migrations, m2) - return multiMigration{m1, m2}, nil + return multiMigration(migrations), nil } diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index 5e81d623..77a7941f 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -105,6 +105,10 @@ func (m *MockLedgerReader) StreamAllLedgers(_ context.Context, _ StreamLedgerFn) return nil } +func (m *MockLedgerReader) StreamLedgerRange(_ context.Context, _ uint32, _ uint32, f StreamLedgerFn) error { + return nil +} + var ( _ TransactionReader = &MockTransactionHandler{} _ TransactionWriter = &MockTransactionHandler{} diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index fdb8b5b9..0088815a 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -316,8 +316,8 @@ type transactionTableMigration struct { func (t *transactionTableMigration) ApplicableRange() *LedgerSeqRange { return &LedgerSeqRange{ - firstLedgerSeq: t.firstLedger, - lastLedgerSeq: t.lastLedger, + FirstLedgerSeq: t.firstLedger, + LastLedgerSeq: t.lastLedger, } } diff --git a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go index 86a8e48a..1970dd07 100644 --- a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go +++ b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go @@ -56,6 +56,10 @@ func (ledgerReader *ConstantLedgerReader) StreamAllLedgers(ctx context.Context, return nil } +func (ledgerReader *ConstantLedgerReader) StreamLedgerRange(ctx context.Context, startLedger uint32, endLedger uint32, f db.StreamLedgerFn) error { + return nil +} + func createLedger(ledgerSequence uint32, protocolVersion uint32, hash byte) xdr.LedgerCloseMeta { return xdr.LedgerCloseMeta{ V: 1, From 88f3dfb92b97eac90a839f9355b6ebb59456c5e1 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 2 Jul 2024 14:44:45 -0700 Subject: [PATCH 33/63] Introduce endLedger and remove Cursor id from schema --- cmd/soroban-rpc/internal/db/event.go | 29 ++++++++++--------- .../internal/db/sqlmigrations/03_events.sql | 2 +- .../internal/methods/get_events.go | 18 +++++++++--- .../internal/methods/get_events_test.go | 2 +- 4 files changed, 31 insertions(+), 20 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 3a1d6a8e..2d17588a 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -94,15 +94,14 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { } query := sq.Insert(eventTableName). - Columns("id", "ledger_sequence", "application_order", "contract_id", "event_type") + Columns("ledger_sequence", "application_order", "contract_id", "event_type") - for index, e := range txEvents { + for _, e := range txEvents { var contractID []byte if e.Event.ContractId != nil { contractID = e.Event.ContractId[:] } - id := Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() - query = query.Values(id, lcm.LedgerSequence(), tx.Index, contractID, int(e.Event.Type)) + query = query.Values(lcm.LedgerSequence(), tx.Index, contractID, int(e.Event.Type)) } _, err = query.RunWith(eventHandler.stmtCache).Exec() @@ -149,18 +148,18 @@ func (eventHandler *eventHandler) GetEvents( start := time.Now() var rows []struct { - EventCursorID string `db:"id"` - TxIndex int `db:"application_order"` - Lcm xdr.LedgerCloseMeta `db:"meta"` + TxIndex int `db:"application_order"` + Lcm xdr.LedgerCloseMeta `db:"meta"` } rowQ := sq. - Select("e.id", "e.application_order", "lcm.meta"). + Select("e.application_order", "lcm.meta"). From(eventTableName + " e"). Join(ledgerCloseMetaTableName + " lcm ON (e.ledger_sequence = lcm.sequence)"). - Where(sq.GtOrEq{"e.id": cursorRange.Start.String()}). - Where(sq.Lt{"e.id": cursorRange.End.String()}). - OrderBy("e.id ASC") + Where(sq.GtOrEq{"e.ledger_sequence": cursorRange.Start.Ledger}). + Where(sq.GtOrEq{"e.application_order": cursorRange.Start.Tx}). + Where(sq.Lt{"e.ledger_sequence": cursorRange.End.Ledger}). + OrderBy("e.ledger_sequence ASC") if len(contractIDs) > 0 { rowQ = rowQ.Where(sq.Eq{"e.contract_id": contractIDs}) @@ -173,15 +172,17 @@ func (eventHandler *eventHandler) GetEvents( contractIDs, err) } else if len(rows) < 1 { - eventHandler.log.Debugf("No events found for start ledger cursor= %v contractIDs= %v", + eventHandler.log.Debugf( + "No events found for ledger range: start ledger cursor= %v - end ledger cursor= %v contractIDs= %v", cursorRange.Start.String(), + cursorRange.End.String(), contractIDs, ) return nil } for _, row := range rows { - eventCursorID, txIndex, lcm := row.EventCursorID, row.TxIndex, row.Lcm + txIndex, lcm := row.TxIndex, row.Lcm reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) if err != nil { return fmt.Errorf("failed to create ledger reader from LCM: %w", err) @@ -201,7 +202,7 @@ func (eventHandler *eventHandler) GetEvents( diagEvents, diagErr := ledgerTx.GetDiagnosticEvents() if diagErr != nil { - return fmt.Errorf("db read failed for Event Id %s: %w", eventCursorID, err) + return fmt.Errorf("couldn't encode transaction DiagnosticEvents: %w", err) } // Find events based on filter passed in function f diff --git a/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql b/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql index 5e698bc9..2ced08d2 100644 --- a/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql +++ b/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql @@ -2,7 +2,6 @@ -- indexing table to find events in ledgers by contract_id CREATE TABLE events( - id TEXT PRIMARY KEY, ledger_sequence INTEGER NOT NULL, application_order INTEGER NOT NULL, contract_id BLOB(32), @@ -11,6 +10,7 @@ CREATE TABLE events( CREATE INDEX idx_ledger_sequence ON events(ledger_sequence); CREATE INDEX idx_contract_id ON events(contract_id); +CREATE INDEX idx_ledger_sequence_application_order ON events (ledger_sequence, application_order); -- +migrate Down drop table events cascade; diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index d8027d10..306a9804 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -15,9 +15,10 @@ import ( "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) +const LedgerScanLimit = 2000 + type eventTypeSet map[string]interface{} func (e eventTypeSet) valid() error { @@ -82,6 +83,7 @@ type EventInfo struct { type GetEventsRequest struct { StartLedger uint32 `json:"startLedger,omitempty"` + EndLedger uint32 `json:"endLedger,omitempty"` Filters []EventFilter `json:"filters"` Pagination *PaginationOptions `json:"pagination,omitempty"` } @@ -90,11 +92,13 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { // Validate start // Validate the paging limit (if it exists) if g.Pagination != nil && g.Pagination.Cursor != nil { - if g.StartLedger != 0 { - return errors.New("startLedger and cursor cannot both be set") + if g.StartLedger != 0 || g.EndLedger != 0 { + return errors.New("startLedger/endLedger and cursor cannot both be set") } } else if g.StartLedger <= 0 { return errors.New("startLedger must be positive") + } else if g.EndLedger < 0 { + return errors.New("endLedger must be positive") } if g.Pagination != nil && g.Pagination.Limit > maxLimit { return fmt.Errorf("limit must not exceed %d", maxLimit) @@ -362,7 +366,13 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques limit = request.Pagination.Limit } } - end := db.Cursor{Ledger: request.StartLedger + ledgerbucketwindow.OneDayOfLedgers} + endLedger := request.StartLedger + LedgerScanLimit + + if request.EndLedger != 0 { + endLedger = min(request.EndLedger, endLedger) + } + + end := db.Cursor{Ledger: endLedger} cursorRange := db.CursorRange{Start: start, End: end} // Check if requested start ledger is within stored ledger range diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index de47941a..1a26eb50 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -418,7 +418,7 @@ func TestGetEventsRequestValid(t *testing.T) { StartLedger: 1, Filters: []EventFilter{}, Pagination: &PaginationOptions{Cursor: &db.Cursor{}}, - }).Valid(1000), "startLedger and cursor cannot both be set") + }).Valid(1000), "startLedger/endLedger and cursor cannot both be set") assert.NoError(t, (&GetEventsRequest{ StartLedger: 1, From eceb76748d1eaef0ec6a3501adb48c3230b0c46a Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 2 Jul 2024 15:28:26 -0700 Subject: [PATCH 34/63] Use LedgerReader to get Ledger Range in events --- cmd/soroban-rpc/internal/db/event.go | 73 ------------------- cmd/soroban-rpc/internal/jsonrpc.go | 2 +- .../internal/methods/get_events.go | 27 +++---- .../internal/methods/get_events_test.go | 8 ++ 4 files changed, 23 insertions(+), 87 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 2d17588a..ff75454f 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -14,8 +14,6 @@ import ( "github.com/stellar/go/support/db" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" - - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) const ( @@ -31,7 +29,6 @@ type EventWriter interface { // EventReader has all the public methods to fetch events from DB type EventReader interface { GetEvents(ctx context.Context, cursorRange CursorRange, contractIDs [][]byte, f ScanFunction) error - GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) } type eventHandler struct { @@ -223,76 +220,6 @@ func (eventHandler *eventHandler) GetEvents( return nil } -// GetLedgerRange returns the min/max ledger sequence numbers from the events table - -func (eventHandler *eventHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { - // TODO: Once we unify all the retention windows use common GetLedgerRange from LedgerReader - var ledgerRange ledgerbucketwindow.LedgerRange - - // - // We use subqueries alongside a UNION ALL stitch in order to select the min - // and max from the ledger table in a single query and get around sqlite's - // limitations with parentheses (see https://stackoverflow.com/a/22609948). - // - // Queries to get the minimum and maximum ledger sequence from the transactions table - minLedgerSeqQ := sq. - Select("m1.ledger_sequence"). - FromSelect( - sq. - Select("ledger_sequence"). - From(eventTableName). - OrderBy("ledger_sequence ASC"). - Limit(1), - "m1", - ) - maxLedgerSeqQ, args, err := sq. - Select("m2.ledger_sequence"). - FromSelect( - sq. - Select("ledger_sequence"). - From(eventTableName). - OrderBy("ledger_sequence DESC"). - Limit(1), - "m2", - ).ToSql() - if err != nil { - return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) - } - - // Combine the min and max ledger sequence queries using UNION ALL - eventMinMaxLedgersQ, _, err := minLedgerSeqQ.Suffix("UNION ALL "+maxLedgerSeqQ, args...).ToSql() - if err != nil { - return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) - } - - // Final query to join ledger_close_meta table and the sequence numbers we got from eventMinMaxLedgersQ - finalSQL := sq. - Select("lcm.meta"). - From(ledgerCloseMetaTableName + " as lcm"). - JoinClause(fmt.Sprintf("JOIN (%s) as seqs ON lcm.sequence == seqs.ledger_sequence", eventMinMaxLedgersQ)) - - var lcms []xdr.LedgerCloseMeta - if err = eventHandler.db.Select(ctx, &lcms, finalSQL); err != nil { - return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) - } else if len(lcms) < 2 { - // There is almost certainly a row, but we want to avoid a race condition - // with ingestion as well as support test cases from an empty DB, so we need - // to sanity check that there is in fact a result. Note that no ledgers in - // the database isn't an error, it's just an empty range. - return ledgerRange, nil - } - - lcm1, lcm2 := lcms[0], lcms[1] - ledgerRange.FirstLedger.Sequence = lcm1.LedgerSequence() - ledgerRange.FirstLedger.CloseTime = lcm1.LedgerCloseTime() - ledgerRange.LastLedger.Sequence = lcm2.LedgerSequence() - ledgerRange.LastLedger.CloseTime = lcm2.LedgerCloseTime() - - eventHandler.log.Debugf("Database ledger range: [%d, %d]", - ledgerRange.FirstLedger.Sequence, ledgerRange.LastLedger.Sequence) - return ledgerRange, nil -} - type eventTableMigration struct { firstLedger uint32 lastLedger uint32 diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index 5ac102e8..e5b65f94 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -165,7 +165,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { params.EventReader, cfg.MaxEventsLimit, cfg.DefaultEventsLimit, - cfg.NetworkPassphrase, + params.LedgerReader, ), longName: "get_events", diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 306a9804..367b2192 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -309,11 +309,11 @@ type GetEventsResponse struct { } type eventsRPCHandler struct { - dbReader db.EventReader - maxLimit uint - defaultLimit uint - logger *log.Entry - networkPassphrase string + dbReader db.EventReader + maxLimit uint + defaultLimit uint + logger *log.Entry + ledgerReader db.LedgerReader } func combineContractIDs(filters []EventFilter) ([][]byte, error) { @@ -345,7 +345,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } } - ledgerRange, err := h.dbReader.GetLedgerRange(ctx) + ledgerRange, err := h.ledgerReader.GetLedgerRange(ctx) if err != nil { return GetEventsResponse{}, &jrpc2.Error{ Code: jrpc2.InternalError, @@ -496,18 +496,19 @@ func eventInfoForEvent( } // NewGetEventsHandler returns a json rpc handler to fetch and filter events -func NewGetEventsHandler(logger *log.Entry, +func NewGetEventsHandler( + logger *log.Entry, dbReader db.EventReader, maxLimit uint, defaultLimit uint, - networkPassphrase string, + ledgerReader db.LedgerReader, ) jrpc2.Handler { eventsHandler := eventsRPCHandler{ - dbReader: dbReader, - maxLimit: maxLimit, - defaultLimit: defaultLimit, - logger: logger, - networkPassphrase: networkPassphrase, + dbReader: dbReader, + maxLimit: maxLimit, + defaultLimit: defaultLimit, + logger: logger, + ledgerReader: ledgerReader, } return NewHandler(func(ctx context.Context, request GetEventsRequest) (GetEventsResponse, error) { return eventsHandler.getEvents(ctx, request) diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 1a26eb50..bed6c6bc 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -568,6 +568,7 @@ func TestGetEvents(t *testing.T) { dbReader: store, maxLimit: 10000, defaultLimit: 100, + ledgerReader: db.NewLedgerReader(dbx), } _, err = handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, @@ -620,6 +621,7 @@ func TestGetEvents(t *testing.T) { dbReader: store, maxLimit: 10000, defaultLimit: 100, + ledgerReader: db.NewLedgerReader(dbx), } results, err := handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, @@ -698,6 +700,7 @@ func TestGetEvents(t *testing.T) { dbReader: store, maxLimit: 10000, defaultLimit: 100, + ledgerReader: db.NewLedgerReader(dbx), } results, err := handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, @@ -760,6 +763,7 @@ func TestGetEvents(t *testing.T) { dbReader: store, maxLimit: 10000, defaultLimit: 100, + ledgerReader: db.NewLedgerReader(dbx), } results, err := handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, @@ -868,6 +872,7 @@ func TestGetEvents(t *testing.T) { dbReader: store, maxLimit: 10000, defaultLimit: 100, + ledgerReader: db.NewLedgerReader(dbx), } results, err := handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, @@ -955,6 +960,7 @@ func TestGetEvents(t *testing.T) { dbReader: store, maxLimit: 10000, defaultLimit: 100, + ledgerReader: db.NewLedgerReader(dbx), } results, err := handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, @@ -1018,6 +1024,7 @@ func TestGetEvents(t *testing.T) { dbReader: store, maxLimit: 10000, defaultLimit: 100, + ledgerReader: db.NewLedgerReader(dbx), } results, err := handler.getEvents(context.TODO(), GetEventsRequest{ StartLedger: 1, @@ -1117,6 +1124,7 @@ func TestGetEvents(t *testing.T) { dbReader: store, maxLimit: 10000, defaultLimit: 100, + ledgerReader: db.NewLedgerReader(dbx), } results, err := handler.getEvents(context.TODO(), GetEventsRequest{ Pagination: &PaginationOptions{ From 115c32ab5b0adfb39fd856f0b999019988941aeb Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Wed, 3 Jul 2024 13:59:12 -0700 Subject: [PATCH 35/63] Fix lint errors --- cmd/soroban-rpc/internal/daemon/daemon.go | 46 ++++++++++--------- cmd/soroban-rpc/internal/db/ledger.go | 1 + cmd/soroban-rpc/internal/db/mocks.go | 2 +- .../internal/methods/get_events.go | 7 ++- .../methods/get_latest_ledger_test.go | 7 ++- 5 files changed, 36 insertions(+), 27 deletions(-) diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index 356a492b..93a12cdc 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -315,30 +315,34 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) *feewindow.FeeWindows applicableRange := dataMigrations.ApplicableRange() ledgerSeqRange = ledgerSeqRange.Merge(applicableRange) - err = db.NewLedgerReader(d.db).StreamLedgerRange(readTxMetaCtx, ledgerSeqRange.FirstLedgerSeq, ledgerSeqRange.LastLedgerSeq, func(txmeta xdr.LedgerCloseMeta) error { - currentSeq = txmeta.LedgerSequence() - if initialSeq == 0 { - initialSeq = currentSeq - d.logger.WithFields(supportlog.F{ - "seq": currentSeq, - }).Info("initializing in-memory store") - } else if (currentSeq-initialSeq)%inMemoryInitializationLedgerLogPeriod == 0 { - d.logger.WithFields(supportlog.F{ - "seq": currentSeq, - }).Debug("still initializing in-memory store") - } + err = db.NewLedgerReader(d.db).StreamLedgerRange( + readTxMetaCtx, + ledgerSeqRange.FirstLedgerSeq, + ledgerSeqRange.LastLedgerSeq, + func(txmeta xdr.LedgerCloseMeta) error { + currentSeq = txmeta.LedgerSequence() + if initialSeq == 0 { + initialSeq = currentSeq + d.logger.WithFields(supportlog.F{ + "seq": currentSeq, + }).Info("initializing in-memory store") + } else if (currentSeq-initialSeq)%inMemoryInitializationLedgerLogPeriod == 0 { + d.logger.WithFields(supportlog.F{ + "seq": currentSeq, + }).Debug("still initializing in-memory store") + } - if err := feewindows.IngestFees(txmeta); err != nil { - d.logger.WithError(err).Fatal("could not initialize fee stats") - } + if err := feewindows.IngestFees(txmeta); err != nil { + d.logger.WithError(err).Fatal("could not initialize fee stats") + } - if applicableRange.IsLedgerIncluded(currentSeq) { - if err := dataMigrations.Apply(readTxMetaCtx, txmeta); err != nil { - d.logger.WithError(err).Fatal("could not run migrations") + if applicableRange.IsLedgerIncluded(currentSeq) { + if err := dataMigrations.Apply(readTxMetaCtx, txmeta); err != nil { + d.logger.WithError(err).Fatal("could not run migrations") + } } - } - return nil - }) + return nil + }) if err != nil { d.logger.WithError(err).Fatal("could not obtain txmeta cache from the database") } diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index b5f77b0c..66c34229 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -5,6 +5,7 @@ import ( "fmt" sq "github.com/Masterminds/squirrel" + "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index 3566b9c5..fee538d9 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -101,7 +101,7 @@ func (m *MockLedgerReader) StreamAllLedgers(_ context.Context, _ StreamLedgerFn) return nil } -func (m *MockLedgerReader) StreamLedgerRange(_ context.Context, _ uint32, _ uint32, f StreamLedgerFn) error { +func (m *MockLedgerReader) StreamLedgerRange(_ context.Context, _ uint32, _ uint32, _ StreamLedgerFn) error { return nil } diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 367b2192..f8e35448 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -93,13 +93,12 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { // Validate the paging limit (if it exists) if g.Pagination != nil && g.Pagination.Cursor != nil { if g.StartLedger != 0 || g.EndLedger != 0 { - return errors.New("startLedger/endLedger and cursor cannot both be set") + return fmt.Errorf("startLedger/endLedger and cursor cannot both be set") } } else if g.StartLedger <= 0 { - return errors.New("startLedger must be positive") - } else if g.EndLedger < 0 { - return errors.New("endLedger must be positive") + return fmt.Errorf("startLedger must be positive") } + if g.Pagination != nil && g.Pagination.Limit > maxLimit { return fmt.Errorf("limit must not exceed %d", maxLimit) } diff --git a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go index 0f74c953..c86e9d9a 100644 --- a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go +++ b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go @@ -64,7 +64,12 @@ func (ledgerReader *ConstantLedgerReader) StreamAllLedgers(_ context.Context, _ return nil } -func (ledgerReader *ConstantLedgerReader) StreamLedgerRange(ctx context.Context, startLedger uint32, endLedger uint32, f db.StreamLedgerFn) error { +func (ledgerReader *ConstantLedgerReader) StreamLedgerRange( + _ context.Context, + _ uint32, + _ uint32, + _ db.StreamLedgerFn, +) error { return nil } From 289aa93820bb0c3e036c436ef97a3a1d98225178 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 16 Jul 2024 11:28:15 -0700 Subject: [PATCH 36/63] Add benchmark for testing various load parameter --- cmd/soroban-rpc/internal/config/options.go | 5 +- .../internal/methods/get_events_test.go | 82 +++++++++++++++++++ 2 files changed, 85 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/config/options.go b/cmd/soroban-rpc/internal/config/options.go index 9d14084a..bd35bf5a 100644 --- a/cmd/soroban-rpc/internal/config/options.go +++ b/cmd/soroban-rpc/internal/config/options.go @@ -17,7 +17,8 @@ import ( const ( // OneDayOfLedgers is (roughly) a 24 hour window of ledgers. - OneDayOfLedgers = 17280 + OneDayOfLedgers = 17280 + SevenDayOfLedgers = OneDayOfLedgers * 7 defaultHTTPEndpoint = "localhost:8000" ) @@ -233,7 +234,7 @@ func (cfg *Config) options() Options { " the default value is %d which corresponds to about 24 hours of history", OneDayOfLedgers), ConfigKey: &cfg.EventLedgerRetentionWindow, - DefaultValue: uint32(OneDayOfLedgers), + DefaultValue: uint32(SevenDayOfLedgers), Validate: positive, }, // TODO: remove diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index bed6c6bc..1b899b88 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -1169,6 +1169,88 @@ func TestGetEvents(t *testing.T) { }) } +func BenchmarkGetEvents(b *testing.B) { + now := time.Now().UTC() + counter := xdr.ScSymbol("COUNTER") + requestedCounter := xdr.ScSymbol("REQUESTED") + dbx := newTestDB(b) + ctx := context.TODO() + log := log.DefaultLogger + log.SetLevel(logrus.TraceLevel) + + writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + require.NoError(b, err) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() + store := db.NewEventReader(log, dbx, passphrase) + + contractID := xdr.Hash([32]byte{}) + + txMeta := []xdr.TransactionMeta{ + transactionMetaWithEvents( + contractEvent( + contractID, + xdr.ScVec{ + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, + }, + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, + ), + systemEvent( + contractID, + xdr.ScVec{ + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, + }, + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, + ), + diagnosticEvent( + contractID, + xdr.ScVec{ + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, + }, + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, + ), + ), + } + for i := 1; i < 100000; i++ { + ledgerCloseMeta := ledgerCloseMetaWithEvents(uint32(i), now.Unix(), txMeta...) + require.NoError(b, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") + require.NoError(b, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") + } + require.NoError(b, write.Commit(1)) + + handler := eventsRPCHandler{ + dbReader: store, + maxLimit: 10000, + defaultLimit: 100, + ledgerReader: db.NewLedgerReader(dbx), + } + + request := GetEventsRequest{ + StartLedger: 1, + Filters: []EventFilter{ + { + ContractIDs: []string{strkey.MustEncode(strkey.VersionByteContract, contractID[:])}, + Topics: []TopicFilter{ + []SegmentFilter{ + {scval: &xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &requestedCounter}}, + }, + }, + }, + }, + } + + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + _, err := handler.getEvents(ctx, request) + if err != nil { + b.Errorf("getEvents failed: %v", err) + } + } + }) +} + func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ...xdr.TransactionMeta) xdr.LedgerCloseMeta { var txProcessing []xdr.TransactionResultMeta var phases []xdr.TransactionPhase From 50ca15206280d5e6aa66228f374723c5114fd0b9 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Wed, 17 Jul 2024 14:10:19 -0700 Subject: [PATCH 37/63] update benchmark --- .../internal/methods/get_events_test.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 1b899b88..ef1bd177 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -1211,7 +1211,7 @@ func BenchmarkGetEvents(b *testing.B) { ), ), } - for i := 1; i < 100000; i++ { + for i := 1; i < 1000000; i++ { ledgerCloseMeta := ledgerCloseMetaWithEvents(uint32(i), now.Unix(), txMeta...) require.NoError(b, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") require.NoError(b, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") @@ -1241,14 +1241,13 @@ func BenchmarkGetEvents(b *testing.B) { b.ResetTimer() - b.RunParallel(func(pb *testing.PB) { - for pb.Next() { - _, err := handler.getEvents(ctx, request) - if err != nil { - b.Errorf("getEvents failed: %v", err) - } + for i := 0; i < b.N; i++ { + _, err := handler.getEvents(ctx, request) + if err != nil { + b.Errorf("getEvents failed: %v", err) } - }) + } + } func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ...xdr.TransactionMeta) xdr.LedgerCloseMeta { From 7b9934b51154888616bb7320866326270fe16a47 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 22 Jul 2024 20:28:40 -0700 Subject: [PATCH 38/63] Comment benchmark events --- cmd/soroban-rpc/internal/db/event.go | 3 +- .../internal/methods/get_events.go | 4 +- .../internal/methods/get_events_test.go | 76 ++++++++++--------- 3 files changed, 43 insertions(+), 40 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index ff75454f..df8a0931 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -170,7 +170,8 @@ func (eventHandler *eventHandler) GetEvents( err) } else if len(rows) < 1 { eventHandler.log.Debugf( - "No events found for ledger range: start ledger cursor= %v - end ledger cursor= %v contractIDs= %v", + "No events found for ledger range: duration= %v start ledger cursor= %v - end ledger cursor= %v contractIDs= %v", + time.Since(start), cursorRange.Start.String(), cursorRange.End.String(), contractIDs, diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index f8e35448..78d09d73 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -17,7 +17,7 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" ) -const LedgerScanLimit = 2000 +const LedgerScanLimit = 4000 type eventTypeSet map[string]interface{} @@ -425,7 +425,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } } - results := []EventInfo{} + results := make([]EventInfo, 0, len(found)) for _, entry := range found { info, err := eventInfoForEvent( entry.event, diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index ef1bd177..c2a8af33 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -1169,55 +1169,56 @@ func TestGetEvents(t *testing.T) { }) } +// TODO:Clean up once benchmarking is done !! func BenchmarkGetEvents(b *testing.B) { - now := time.Now().UTC() - counter := xdr.ScSymbol("COUNTER") + // now := time.Now().UTC() + // counter := xdr.ScSymbol("COUNTER") requestedCounter := xdr.ScSymbol("REQUESTED") dbx := newTestDB(b) ctx := context.TODO() log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) - writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) - write, err := writer.NewTx(ctx) - require.NoError(b, err) - ledgerW, eventW := write.LedgerWriter(), write.EventWriter() + // writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + // write, err := writer.NewTx(ctx) + // require.NoError(b, err) + // ledgerW, eventW := write.LedgerWriter(), write.EventWriter() store := db.NewEventReader(log, dbx, passphrase) - contractID := xdr.Hash([32]byte{}) + // contractID := xdr.Hash([32]byte{1, 2, 3}) - txMeta := []xdr.TransactionMeta{ - transactionMetaWithEvents( - contractEvent( - contractID, - xdr.ScVec{ + /* txMeta := []xdr.TransactionMeta{ + transactionMetaWithEvents( + contractEvent( + contractID, + xdr.ScVec{ + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, + }, xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, - }, - xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, - ), - systemEvent( - contractID, - xdr.ScVec{ + ), + systemEvent( + contractID, + xdr.ScVec{ + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, + }, xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, - }, - xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, - ), - diagnosticEvent( - contractID, - xdr.ScVec{ + ), + diagnosticEvent( + contractID, + xdr.ScVec{ + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, + }, xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, - }, - xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, + ), ), - ), - } - for i := 1; i < 1000000; i++ { - ledgerCloseMeta := ledgerCloseMetaWithEvents(uint32(i), now.Unix(), txMeta...) - require.NoError(b, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") - require.NoError(b, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") - } - require.NoError(b, write.Commit(1)) - + } + for i := 1; i < 1000000; i++ { + ledgerCloseMeta := ledgerCloseMetaWithEvents(uint32(i), now.Unix(), txMeta...) + require.NoError(b, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") + require.NoError(b, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") + } + require.NoError(b, write.Commit(1)) + */ handler := eventsRPCHandler{ dbReader: store, maxLimit: 10000, @@ -1229,7 +1230,8 @@ func BenchmarkGetEvents(b *testing.B) { StartLedger: 1, Filters: []EventFilter{ { - ContractIDs: []string{strkey.MustEncode(strkey.VersionByteContract, contractID[:])}, + // ContractIDs: []string{"CCVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKUD2U"}, + EventType: map[string]interface{}{EventTypeSystem: nil}, Topics: []TopicFilter{ []SegmentFilter{ {scval: &xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &requestedCounter}}, @@ -1240,6 +1242,7 @@ func BenchmarkGetEvents(b *testing.B) { } b.ResetTimer() + b.ReportAllocs() for i := 0; i < b.N; i++ { _, err := handler.getEvents(ctx, request) @@ -1247,7 +1250,6 @@ func BenchmarkGetEvents(b *testing.B) { b.Errorf("getEvents failed: %v", err) } } - } func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ...xdr.TransactionMeta) xdr.LedgerCloseMeta { From 3506b414f8fb89193b51616b4366dd9eff7eb288 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 29 Jul 2024 10:16:25 -0700 Subject: [PATCH 39/63] Reduce allocs pt1 --- cmd/soroban-rpc/internal/config/options.go | 2 +- cmd/soroban-rpc/internal/db/event.go | 19 +++-- .../internal/methods/get_events.go | 26 ++++--- .../internal/methods/get_events_test.go | 72 +++++++++---------- 4 files changed, 62 insertions(+), 57 deletions(-) diff --git a/cmd/soroban-rpc/internal/config/options.go b/cmd/soroban-rpc/internal/config/options.go index 2977f5a8..58df85e2 100644 --- a/cmd/soroban-rpc/internal/config/options.go +++ b/cmd/soroban-rpc/internal/config/options.go @@ -232,7 +232,7 @@ func (cfg *Config) options() Options { " the default value is %d which corresponds to about 24 hours of history", OneDayOfLedgers), ConfigKey: &cfg.EventLedgerRetentionWindow, - DefaultValue: uint32(SevenDayOfLedgers), + DefaultValue: uint32(OneDayOfLedgers), Validate: positive, }, // TODO: remove diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index df8a0931..fd6c5a7e 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -169,13 +169,18 @@ func (eventHandler *eventHandler) GetEvents( contractIDs, err) } else if len(rows) < 1 { - eventHandler.log.Debugf( - "No events found for ledger range: duration= %v start ledger cursor= %v - end ledger cursor= %v contractIDs= %v", - time.Since(start), - cursorRange.Start.String(), - cursorRange.End.String(), - contractIDs, - ) + eventHandler.log. + WithField("duration", time.Since(start)). + WithField("start", cursorRange.Start.String()). + WithField("end", cursorRange.End.String()). + WithField("contracts", contractIDs). + Debugf( + "No events found for ledger range: duration= %v start ledger cursor= %v - end ledger cursor= %v contractIDs= %v", + time.Since(start), + cursorRange.Start.String(), + cursorRange.End.String(), + contractIDs, + ) return nil } diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 78d09d73..d5b38b3e 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -10,6 +10,7 @@ import ( "github.com/creachadair/jrpc2" "github.com/stellar/go/strkey" + "github.com/stellar/go/support/collections/set" "github.com/stellar/go/support/errors" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" @@ -17,7 +18,12 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" ) -const LedgerScanLimit = 4000 +const ( + LedgerScanLimit = 4000 + maxContractIDsLimit = 5 + maxTopicsLimit = 5 + maxFiltersLimit = 5 +) type eventTypeSet map[string]interface{} @@ -104,7 +110,7 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { } // Validate filters - if len(g.Filters) > 5 { + if len(g.Filters) > maxFiltersLimit { return errors.New("maximum 5 filters per request") } for i, filter := range g.Filters { @@ -150,10 +156,10 @@ func (e *EventFilter) Valid() error { if err := e.EventType.valid(); err != nil { return errors.Wrap(err, "filter type invalid") } - if len(e.ContractIDs) > 5 { + if len(e.ContractIDs) > maxContractIDsLimit { return errors.New("maximum 5 contract IDs per filter") } - if len(e.Topics) > 5 { + if len(e.Topics) > maxTopicsLimit { return errors.New("maximum 5 topics per filter") } for i, id := range e.ContractIDs { @@ -316,11 +322,11 @@ type eventsRPCHandler struct { } func combineContractIDs(filters []EventFilter) ([][]byte, error) { - contractIDSet := make(map[string]struct{}) + contractIDSet := set.NewSet[string](maxFiltersLimit * maxContractIDsLimit) for _, filter := range filters { for _, contractID := range filter.ContractIDs { - contractIDSet[contractID] = struct{}{} + contractIDSet.Add(contractID) } } @@ -328,7 +334,7 @@ func combineContractIDs(filters []EventFilter) ([][]byte, error) { for contractID := range contractIDSet { id, err := strkey.Decode(strkey.VersionByteContract, contractID) if err != nil { - return nil, fmt.Errorf("contract ID %v invalid", contractID) + return nil, fmt.Errorf("invalid contract ID %v: ", contractID) } contractIDs = append(contractIDs, id) @@ -392,8 +398,8 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques event xdr.DiagnosticEvent txHash *xdr.Hash } - var found []entry - cursorSet := make(map[string]struct{}) + found := make([]entry, 0, limit) + cursorSet := set.NewSet[string](int(limit)) contractIDs, err := combineContractIDs(request.Filters) if err != nil { @@ -411,7 +417,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } if request.Matches(event) && cursor.Cmp(start) >= 0 { - cursorSet[cursorID] = struct{}{} + cursorSet.Add(cursorID) found = append(found, entry{cursor, ledgerCloseTimestamp, event, txHash}) } return uint(len(found)) < limit diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index c2a8af33..4938612c 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -1171,54 +1171,48 @@ func TestGetEvents(t *testing.T) { // TODO:Clean up once benchmarking is done !! func BenchmarkGetEvents(b *testing.B) { - // now := time.Now().UTC() - // counter := xdr.ScSymbol("COUNTER") + now := time.Now().UTC() + counter := xdr.ScSymbol("COUNTER") requestedCounter := xdr.ScSymbol("REQUESTED") dbx := newTestDB(b) ctx := context.TODO() log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) + contractID := xdr.Hash([32]byte{}) - // writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) - // write, err := writer.NewTx(ctx) - // require.NoError(b, err) - // ledgerW, eventW := write.LedgerWriter(), write.EventWriter() + writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + require.NoError(b, err) + ledgerW, eventW := write.LedgerWriter(), write.EventWriter() store := db.NewEventReader(log, dbx, passphrase) - // contractID := xdr.Hash([32]byte{1, 2, 3}) + // create 25 contract events + var events []xdr.ContractEvent + for i := 0; i < 25; i++ { + + contractEvent := contractEvent( + contractID, + xdr.ScVec{ + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, + }, + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, + ) + events = append(events, contractEvent) + } + + txMeta := []xdr.TransactionMeta{ + transactionMetaWithEvents( + events..., + ), + } + + for i := 1; i < 121000; i++ { + ledgerCloseMeta := ledgerCloseMetaWithEvents(uint32(i), now.Unix(), txMeta...) + require.NoError(b, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") + require.NoError(b, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") + } + require.NoError(b, write.Commit(1)) - /* txMeta := []xdr.TransactionMeta{ - transactionMetaWithEvents( - contractEvent( - contractID, - xdr.ScVec{ - xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, - }, - xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, - ), - systemEvent( - contractID, - xdr.ScVec{ - xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, - }, - xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, - ), - diagnosticEvent( - contractID, - xdr.ScVec{ - xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, - }, - xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, - ), - ), - } - for i := 1; i < 1000000; i++ { - ledgerCloseMeta := ledgerCloseMetaWithEvents(uint32(i), now.Unix(), txMeta...) - require.NoError(b, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") - require.NoError(b, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") - } - require.NoError(b, write.Commit(1)) - */ handler := eventsRPCHandler{ dbReader: store, maxLimit: 10000, From c431913269b57daeb50780e80b8c88cb0d242673 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Thu, 1 Aug 2024 21:18:56 -0700 Subject: [PATCH 40/63] Benchmark with 30 million events --- cmd/soroban-rpc/internal/db/event.go | 2 +- cmd/soroban-rpc/internal/methods/get_events_test.go | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index fd6c5a7e..3eac29b7 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -150,7 +150,7 @@ func (eventHandler *eventHandler) GetEvents( } rowQ := sq. - Select("e.application_order", "lcm.meta"). + Select("DISTINCT e.application_order", "lcm.meta"). From(eventTableName + " e"). Join(ledgerCloseMetaTableName + " lcm ON (e.ledger_sequence = lcm.sequence)"). Where(sq.GtOrEq{"e.ledger_sequence": cursorRange.Start.Ledger}). diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 4938612c..bb96cf0c 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -1186,9 +1186,9 @@ func BenchmarkGetEvents(b *testing.B) { ledgerW, eventW := write.LedgerWriter(), write.EventWriter() store := db.NewEventReader(log, dbx, passphrase) - // create 25 contract events + // create 250 contract events var events []xdr.ContractEvent - for i := 0; i < 25; i++ { + for i := 0; i < 250; i++ { contractEvent := contractEvent( contractID, @@ -1244,6 +1244,7 @@ func BenchmarkGetEvents(b *testing.B) { b.Errorf("getEvents failed: %v", err) } } + } func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ...xdr.TransactionMeta) xdr.LedgerCloseMeta { @@ -1279,7 +1280,7 @@ func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ... V1: &xdr.TransactionV1Envelope{ Tx: xdr.Transaction{ SourceAccount: xdr.MustMuxedAddress(keypair.MustRandom().Address()), - Operations: operations, + //Operations: operations, }, }, } From 818cd20ddf485898ff5f38d6f39019fd425155b3 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Wed, 7 Aug 2024 01:09:17 -0700 Subject: [PATCH 41/63] Refactor getEvents to backed only by Events table --- cmd/soroban-rpc/internal/db/event.go | 170 ++++++++++++------ .../internal/db/sqlmigrations/03_events.sql | 21 ++- .../internal/db/transaction_test.go | 2 +- .../internal/methods/get_events.go | 78 ++++++-- .../internal/methods/get_events_test.go | 106 +++++------ 5 files changed, 254 insertions(+), 123 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 3eac29b7..d32a5e36 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -28,7 +28,14 @@ type EventWriter interface { // EventReader has all the public methods to fetch events from DB type EventReader interface { - GetEvents(ctx context.Context, cursorRange CursorRange, contractIDs [][]byte, f ScanFunction) error + GetEvents( + ctx context.Context, + cursorRange CursorRange, + contractIDs [][]byte, + topics [][]string, + eventTypes []int, + f ScanFunction, + ) error } type eventHandler struct { @@ -81,6 +88,8 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { continue } + transactionHash := tx.Result.TransactionHash[:] + txEvents, err := tx.GetDiagnosticEvents() if err != nil { return err @@ -91,14 +100,48 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { } query := sq.Insert(eventTableName). - Columns("ledger_sequence", "application_order", "contract_id", "event_type") + Columns("id", "contract_id", "event_type", "event_data", "ledger_close_time", "transaction_hash", "topic1", "topic2", "topic3", "topic4") + + for index, e := range txEvents { - for _, e := range txEvents { var contractID []byte if e.Event.ContractId != nil { contractID = e.Event.ContractId[:] } - query = query.Values(lcm.LedgerSequence(), tx.Index, contractID, int(e.Event.Type)) + + id := Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() + eventBlob, err := xdr.MarshalBase64(e) + if err != nil { + return err + } + + v0, ok := e.Event.Body.GetV0() + if !ok { + return errors.New("unknown event version") + } + + // Encode the topics + topicList := make([]string, 4) + for index, segment := range v0.Topics { + seg, err := xdr.MarshalBase64(segment) + if err != nil { + return err + } + topicList[index] = seg + } + + query = query.Values( + id, + contractID, + int(e.Event.Type), + eventBlob, + lcm.LedgerCloseTime(), + transactionHash, + topicList[0], + topicList[1], + topicList[2], + topicList[3], + ) } _, err = query.RunWith(eventHandler.stmtCache).Exec() @@ -140,40 +183,93 @@ func (eventHandler *eventHandler) GetEvents( ctx context.Context, cursorRange CursorRange, contractIDs [][]byte, + topics [][]string, + eventTypes []int, f ScanFunction, ) error { start := time.Now() - var rows []struct { - TxIndex int `db:"application_order"` - Lcm xdr.LedgerCloseMeta `db:"meta"` - } - rowQ := sq. - Select("DISTINCT e.application_order", "lcm.meta"). - From(eventTableName + " e"). - Join(ledgerCloseMetaTableName + " lcm ON (e.ledger_sequence = lcm.sequence)"). - Where(sq.GtOrEq{"e.ledger_sequence": cursorRange.Start.Ledger}). - Where(sq.GtOrEq{"e.application_order": cursorRange.Start.Tx}). - Where(sq.Lt{"e.ledger_sequence": cursorRange.End.Ledger}). - OrderBy("e.ledger_sequence ASC") + Select(" id", "event_data", "transaction_hash", "ledger_close_time"). + From(eventTableName). + Where(sq.GtOrEq{"id": cursorRange.Start.String()}). + Where(sq.Lt{"id": cursorRange.End.String()}). + OrderBy("id ASC") if len(contractIDs) > 0 { - rowQ = rowQ.Where(sq.Eq{"e.contract_id": contractIDs}) + rowQ = rowQ.Where(sq.Eq{"contract_id": contractIDs}) + } + if len(eventTypes) > 0 { + rowQ = rowQ.Where(sq.Eq{"event_type": eventTypes}) } - if err := eventHandler.db.Select(ctx, &rows, rowQ); err != nil { + if len(topics) > 0 { + var orConditions sq.Or + for i, topic := range topics { + if topic == nil { + break + } + orConditions = append(orConditions, sq.Eq{fmt.Sprintf("topic%d", i+1): topic}) + } + if len(orConditions) > 0 { + rowQ = rowQ.Where(orConditions) + } + } + + rows, err := eventHandler.db.Query(ctx, rowQ) + if err != nil { return fmt.Errorf( - "db read failed for start ledger cursor= %v contractIDs= %v: %w", + "db read failed for start ledger cursor= %v end ledger cursor= %v "+ + "contractIDs= %v eventTypes= %v topics= %v : %w", cursorRange.Start.String(), + cursorRange.End.String(), contractIDs, + eventTypes, + topics, err) - } else if len(rows) < 1 { + } + + defer rows.Close() + + foundRows := false + for rows.Next() { + foundRows = true + var row struct { + eventCursorID string `db:"id"` + eventData []byte `db:"event_data"` + transactionHash []byte `db:"transaction_hash"` + ledgerCloseTime int64 `db:"ledger_close_time"` + } + + err = rows.Scan(&row.eventCursorID, &row.eventData, &row.transactionHash, &row.ledgerCloseTime) + + id, eventData, ledgerCloseTime, transactionHash := row.eventCursorID, row.eventData, row.ledgerCloseTime, row.transactionHash + cur, err := ParseCursor(id) + if err != nil { + return fmt.Errorf("failed to parse cursor: %w", err) + } + + var eventXDR xdr.DiagnosticEvent + err = xdr.SafeUnmarshalBase64(string(eventData), &eventXDR) + + if err != nil { + return fmt.Errorf("failed to decode event: %w", err) + } + txHash := xdr.Hash(transactionHash) + if !f(eventXDR, cur, ledgerCloseTime, &txHash) { + return nil + } + + } + + if !foundRows { eventHandler.log. WithField("duration", time.Since(start)). WithField("start", cursorRange.Start.String()). WithField("end", cursorRange.End.String()). WithField("contracts", contractIDs). + WithField("eventTypes", eventTypes). + WithField("Topics", topics). Debugf( "No events found for ledger range: duration= %v start ledger cursor= %v - end ledger cursor= %v contractIDs= %v", time.Since(start), @@ -181,40 +277,6 @@ func (eventHandler *eventHandler) GetEvents( cursorRange.End.String(), contractIDs, ) - return nil - } - - for _, row := range rows { - txIndex, lcm := row.TxIndex, row.Lcm - reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(eventHandler.passphrase, lcm) - if err != nil { - return fmt.Errorf("failed to create ledger reader from LCM: %w", err) - } - - err = reader.Seek(txIndex - 1) - if err != nil { - return fmt.Errorf("failed to index to tx %d in ledger %d: %w", txIndex, lcm.LedgerSequence(), err) - } - - ledgerCloseTime := lcm.LedgerCloseTime() - ledgerTx, err := reader.Read() - if err != nil { - return fmt.Errorf("failed reading tx: %w", err) - } - transactionHash := ledgerTx.Result.TransactionHash - diagEvents, diagErr := ledgerTx.GetDiagnosticEvents() - - if diagErr != nil { - return fmt.Errorf("couldn't encode transaction DiagnosticEvents: %w", err) - } - - // Find events based on filter passed in function f - for eventIndex, event := range diagEvents { - cur := Cursor{Ledger: lcm.LedgerSequence(), Tx: uint32(txIndex), Event: uint32(eventIndex)} - if !f(event, cur, ledgerCloseTime, &transactionHash) { - return nil - } - } } eventHandler.log. diff --git a/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql b/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql index 2ced08d2..106824cf 100644 --- a/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql +++ b/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql @@ -1,16 +1,23 @@ -- +migrate Up -- indexing table to find events in ledgers by contract_id -CREATE TABLE events( - ledger_sequence INTEGER NOT NULL, - application_order INTEGER NOT NULL, +CREATE TABLE events +( + id TEXT PRIMARY KEY, contract_id BLOB(32), - event_type INTEGER NOT NULL + event_type INTEGER NOT NULL, + event_data BLOB NOT NULL, + ledger_close_time INTEGER NOT NULL, + transaction_hash BLOB(32), + topic1 BLOB, + topic2 BLOB, + topic3 BLOB, + topic4 BLOB ); -CREATE INDEX idx_ledger_sequence ON events(ledger_sequence); -CREATE INDEX idx_contract_id ON events(contract_id); -CREATE INDEX idx_ledger_sequence_application_order ON events (ledger_sequence, application_order); +CREATE INDEX idx_contract_id ON events (contract_id); +CREATE INDEX idx_topic1 ON events (topic1); + -- +migrate Down drop table events cascade; diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 147af505..62a940f3 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -99,7 +99,7 @@ func TestTransactionFound(t *testing.T) { end := Cursor{Ledger: 1000} cursorRange := CursorRange{Start: start, End: end} - err = eventReader.GetEvents(ctx, cursorRange, nil, nil) + err = eventReader.GetEvents(ctx, cursorRange, nil, nil, nil, nil) require.NoError(t, err) // check all 200 cases diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index d5b38b3e..374b86c8 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -19,7 +19,7 @@ import ( ) const ( - LedgerScanLimit = 4000 + LedgerScanLimit = 10000 maxContractIDsLimit = 5 maxTopicsLimit = 5 maxFiltersLimit = 5 @@ -66,6 +66,14 @@ func (e eventTypeSet) MarshalJSON() ([]byte, error) { return json.Marshal(strings.Join(keys, ",")) } +func (e eventTypeSet) Keys() []string { + keys := make([]string, 0, len(e)) + for key := range e { + keys = append(keys, key) + } + return keys +} + func (e eventTypeSet) matches(event xdr.ContractEvent) bool { if len(e) == 0 { return true @@ -146,6 +154,12 @@ var eventTypeFromXDR = map[xdr.ContractEventType]string{ xdr.ContractEventTypeDiagnostic: EventTypeDiagnostic, } +var eventTypeXDRFromEventType = map[string]xdr.ContractEventType{ + EventTypeSystem: xdr.ContractEventTypeSystem, + EventTypeContract: xdr.ContractEventTypeContract, + EventTypeDiagnostic: xdr.ContractEventTypeDiagnostic, +} + type EventFilter struct { EventType eventTypeSet `json:"type,omitempty"` ContractIDs []string `json:"contractIds,omitempty"` @@ -341,6 +355,46 @@ func combineContractIDs(filters []EventFilter) ([][]byte, error) { } return contractIDs, nil } +func combineEventTypes(filters []EventFilter) []int { + + eventTypes := make(map[int]bool) + for _, filter := range filters { + for _, eventType := range filter.EventType.Keys() { + eventTypeXDR := eventTypeXDRFromEventType[eventType] + eventTypes[int(eventTypeXDR)] = true + } + } + uniqueEventTypes := make([]int, 0, len(eventTypes)) + for eventType := range eventTypes { + uniqueEventTypes = append(uniqueEventTypes, eventType) + } + return uniqueEventTypes +} + +func combineTopics(filters []EventFilter) ([][]string, error) { + encodedTopicsList := make([][]string, 4) + + for _, filter := range filters { + if len(filter.Topics) == 0 { + return [][]string{}, nil + } + + for _, topicFilter := range filter.Topics { + for i, segmentFilter := range topicFilter { + if segmentFilter.wildcard == nil && segmentFilter.scval != nil { + encodedTopic, err := xdr.MarshalBase64(segmentFilter.scval) + if err != nil { + return [][]string{}, fmt.Errorf("failed to marshal segment: %w", err) + } + encodedTopicsList[i] = append(encodedTopicsList[i], encodedTopic) + } + + } + } + } + + return encodedTopicsList, nil +} func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsRequest) (GetEventsResponse, error) { if err := request.Valid(h.maxLimit); err != nil { @@ -399,7 +453,6 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques txHash *xdr.Hash } found := make([]entry, 0, limit) - cursorSet := set.NewSet[string](int(limit)) contractIDs, err := combineContractIDs(request.Filters) if err != nil { @@ -409,21 +462,26 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } } - // Scan function to apply filters - f := func(event xdr.DiagnosticEvent, cursor db.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { - cursorID := cursor.String() - if _, exists := cursorSet[cursorID]; exists { - return true + topics, err := combineTopics(request.Filters) + if err != nil { + return GetEventsResponse{}, &jrpc2.Error{ + Code: jrpc2.InvalidParams, + Message: err.Error(), } + } + + eventTypes := combineEventTypes(request.Filters) + + // Scan function to apply filters + eventScanFunction := func(event xdr.DiagnosticEvent, cursor db.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { - if request.Matches(event) && cursor.Cmp(start) >= 0 { - cursorSet.Add(cursorID) + if request.Matches(event) { found = append(found, entry{cursor, ledgerCloseTimestamp, event, txHash}) } return uint(len(found)) < limit } - err = h.dbReader.GetEvents(ctx, cursorRange, contractIDs, f) + err = h.dbReader.GetEvents(ctx, cursorRange, contractIDs, topics, eventTypes, eventScanFunction) if err != nil { return GetEventsResponse{}, &jrpc2.Error{ Code: jrpc2.InvalidRequest, diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index bb96cf0c..9549dd40 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "path" + "strconv" "strings" "testing" "time" @@ -1171,42 +1172,34 @@ func TestGetEvents(t *testing.T) { // TODO:Clean up once benchmarking is done !! func BenchmarkGetEvents(b *testing.B) { - now := time.Now().UTC() - counter := xdr.ScSymbol("COUNTER") - requestedCounter := xdr.ScSymbol("REQUESTED") + + var counters [10000]xdr.ScSymbol + for i := 0; i < len(counters); i++ { + counters[i] = xdr.ScSymbol("TEST-COUNTER-" + strconv.Itoa(i+1) + strconv.Itoa(i%1000)) + } + //counter := xdr.ScSymbol("COUNTER") + //requestedCounter := xdr.ScSymbol("REQUESTED") dbx := newTestDB(b) ctx := context.TODO() log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) + store := db.NewEventReader(log, dbx, passphrase) contractID := xdr.Hash([32]byte{}) + now := time.Now().UTC() writer := db.NewReadWriter(log, dbx, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) write, err := writer.NewTx(ctx) require.NoError(b, err) ledgerW, eventW := write.LedgerWriter(), write.EventWriter() - store := db.NewEventReader(log, dbx, passphrase) - // create 250 contract events - var events []xdr.ContractEvent - for i := 0; i < 250; i++ { - - contractEvent := contractEvent( - contractID, - xdr.ScVec{ - xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, - }, - xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counter}, - ) - events = append(events, contractEvent) - } + for i := 1; i < 121000; i++ { - txMeta := []xdr.TransactionMeta{ - transactionMetaWithEvents( - events..., - ), - } + var counters [250]xdr.ScSymbol + for j := 0; j < len(counters); j++ { + counters[j] = xdr.ScSymbol("TEST-COUNTER-" + strconv.Itoa(j+1) + strconv.Itoa(i%1000)) + } - for i := 1; i < 121000; i++ { + txMeta := getTxMeta(contractID, counters) ledgerCloseMeta := ledgerCloseMetaWithEvents(uint32(i), now.Unix(), txMeta...) require.NoError(b, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") require.NoError(b, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") @@ -1220,15 +1213,17 @@ func BenchmarkGetEvents(b *testing.B) { ledgerReader: db.NewLedgerReader(dbx), } + //star := "*" request := GetEventsRequest{ StartLedger: 1, Filters: []EventFilter{ { - // ContractIDs: []string{"CCVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKUD2U"}, - EventType: map[string]interface{}{EventTypeSystem: nil}, + //ContractIDs: []string{"CCVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKUD2U"}, + EventType: map[string]interface{}{EventTypeContract: nil}, Topics: []TopicFilter{ []SegmentFilter{ - {scval: &xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &requestedCounter}}, + {scval: &xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counters[1]}}, + //{wildcard: &star}, }, }, }, @@ -1245,6 +1240,38 @@ func BenchmarkGetEvents(b *testing.B) { } } + totalNs := b.Elapsed() + nsPerOp := totalNs.Nanoseconds() / int64(b.N) + msPerOp := float64(nsPerOp) / 1e6 + fmt.Printf("Benchmark Results:\n") + fmt.Printf("%d ns/op (%.3f ms/op)\n", nsPerOp, msPerOp) + +} + +func getTxMeta(contractID xdr.Hash, counters [250]xdr.ScSymbol) []xdr.TransactionMeta { + // create 250 contract events + var events []xdr.ContractEvent + count := 0 + for i := 0; i < 250; i++ { + + contractEvent := contractEvent( + contractID, + xdr.ScVec{ + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counters[count]}, + }, + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counters[count]}, + ) + events = append(events, contractEvent) + count += 1 + + } + + txMeta := []xdr.TransactionMeta{ + transactionMetaWithEvents( + events..., + ), + } + return txMeta } func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ...xdr.TransactionMeta) xdr.LedgerCloseMeta { @@ -1252,29 +1279,6 @@ func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ... var phases []xdr.TransactionPhase for _, item := range txMeta { - var operations []xdr.Operation - for range item.MustV3().SorobanMeta.Events { - operations = append(operations, - xdr.Operation{ - Body: xdr.OperationBody{ - Type: xdr.OperationTypeInvokeHostFunction, - InvokeHostFunctionOp: &xdr.InvokeHostFunctionOp{ - HostFunction: xdr.HostFunction{ - Type: xdr.HostFunctionTypeHostFunctionTypeInvokeContract, - InvokeContract: &xdr.InvokeContractArgs{ - ContractAddress: xdr.ScAddress{ - Type: xdr.ScAddressTypeScAddressTypeContract, - ContractId: &xdr.Hash{0x1, 0x2}, - }, - FunctionName: "foo", - Args: nil, - }, - }, - Auth: []xdr.SorobanAuthorizationEntry{}, - }, - }, - }) - } envelope := xdr.TransactionEnvelope{ Type: xdr.EnvelopeTypeEnvelopeTypeTx, V1: &xdr.TransactionV1Envelope{ @@ -1397,8 +1401,8 @@ func diagnosticEvent(contractID xdr.Hash, topic []xdr.ScVal, body xdr.ScVal) xdr } func newTestDB(tb testing.TB) *db.DB { - tmp := tb.TempDir() - dbPath := path.Join(tmp, "db.sqlite") + //tmp := tb.TempDir() + dbPath := path.Join("", "dbx.sqlite") db, err := db.OpenSQLiteDB(dbPath) require.NoError(tb, err) tb.Cleanup(func() { From 2c7ffa4618375672b6950c77300e2be6e4264f71 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Wed, 7 Aug 2024 01:16:20 -0700 Subject: [PATCH 42/63] change test db path --- cmd/soroban-rpc/internal/methods/get_events_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 9549dd40..c345eff1 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -1401,8 +1401,8 @@ func diagnosticEvent(contractID xdr.Hash, topic []xdr.ScVal, body xdr.ScVal) xdr } func newTestDB(tb testing.TB) *db.DB { - //tmp := tb.TempDir() - dbPath := path.Join("", "dbx.sqlite") + tmp := tb.TempDir() + dbPath := path.Join(tmp, "dbx.sqlite") db, err := db.OpenSQLiteDB(dbPath) require.NoError(tb, err) tb.Cleanup(func() { From b26913400163853bf21d4b9a1578b6f485020050 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Wed, 7 Aug 2024 15:37:08 -0700 Subject: [PATCH 43/63] Use Binary encoding for saving events into DB --- cmd/soroban-rpc/internal/db/event.go | 5 ++-- .../internal/methods/get_events.go | 6 ++--- .../internal/methods/get_events_test.go | 27 ++++++++----------- 3 files changed, 15 insertions(+), 23 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index d32a5e36..e47a5322 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -110,7 +110,7 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { } id := Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() - eventBlob, err := xdr.MarshalBase64(e) + eventBlob, err := xdr.NewEncodingBuffer().MarshalBinary(&e) if err != nil { return err } @@ -250,8 +250,7 @@ func (eventHandler *eventHandler) GetEvents( } var eventXDR xdr.DiagnosticEvent - err = xdr.SafeUnmarshalBase64(string(eventData), &eventXDR) - + err = xdr.SafeUnmarshal(eventData, &eventXDR) if err != nil { return fmt.Errorf("failed to decode event: %w", err) } diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 374b86c8..249a16bc 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -19,7 +19,7 @@ import ( ) const ( - LedgerScanLimit = 10000 + LedgerScanLimit = 8000 maxContractIDsLimit = 5 maxTopicsLimit = 5 maxFiltersLimit = 5 @@ -355,8 +355,8 @@ func combineContractIDs(filters []EventFilter) ([][]byte, error) { } return contractIDs, nil } -func combineEventTypes(filters []EventFilter) []int { +func combineEventTypes(filters []EventFilter) []int { eventTypes := make(map[int]bool) for _, filter := range filters { for _, eventType := range filter.EventType.Keys() { @@ -388,7 +388,6 @@ func combineTopics(filters []EventFilter) ([][]string, error) { } encodedTopicsList[i] = append(encodedTopicsList[i], encodedTopic) } - } } } @@ -474,7 +473,6 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques // Scan function to apply filters eventScanFunction := func(event xdr.DiagnosticEvent, cursor db.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { - if request.Matches(event) { found = append(found, entry{cursor, ledgerCloseTimestamp, event, txHash}) } diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index c345eff1..5a621384 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -1172,13 +1172,12 @@ func TestGetEvents(t *testing.T) { // TODO:Clean up once benchmarking is done !! func BenchmarkGetEvents(b *testing.B) { - var counters [10000]xdr.ScSymbol for i := 0; i < len(counters); i++ { counters[i] = xdr.ScSymbol("TEST-COUNTER-" + strconv.Itoa(i+1) + strconv.Itoa(i%1000)) } - //counter := xdr.ScSymbol("COUNTER") - //requestedCounter := xdr.ScSymbol("REQUESTED") + // counter := xdr.ScSymbol("COUNTER") + // requestedCounter := xdr.ScSymbol("REQUESTED") dbx := newTestDB(b) ctx := context.TODO() log := log.DefaultLogger @@ -1194,7 +1193,7 @@ func BenchmarkGetEvents(b *testing.B) { for i := 1; i < 121000; i++ { - var counters [250]xdr.ScSymbol + var counters [1000]xdr.ScSymbol for j := 0; j < len(counters); j++ { counters[j] = xdr.ScSymbol("TEST-COUNTER-" + strconv.Itoa(j+1) + strconv.Itoa(i%1000)) } @@ -1213,13 +1212,13 @@ func BenchmarkGetEvents(b *testing.B) { ledgerReader: db.NewLedgerReader(dbx), } - //star := "*" + // star := "*" request := GetEventsRequest{ StartLedger: 1, Filters: []EventFilter{ { - //ContractIDs: []string{"CCVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKUD2U"}, - EventType: map[string]interface{}{EventTypeContract: nil}, + // ContractIDs: []string{"CCVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKUD2U"}, + // EventType: map[string]interface{}{EventTypeContract: nil}, Topics: []TopicFilter{ []SegmentFilter{ {scval: &xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counters[1]}}, @@ -1245,25 +1244,21 @@ func BenchmarkGetEvents(b *testing.B) { msPerOp := float64(nsPerOp) / 1e6 fmt.Printf("Benchmark Results:\n") fmt.Printf("%d ns/op (%.3f ms/op)\n", nsPerOp, msPerOp) - } -func getTxMeta(contractID xdr.Hash, counters [250]xdr.ScSymbol) []xdr.TransactionMeta { +func getTxMeta(contractID xdr.Hash, counters [1000]xdr.ScSymbol) []xdr.TransactionMeta { // create 250 contract events + var events []xdr.ContractEvent - count := 0 for i := 0; i < 250; i++ { - contractEvent := contractEvent( contractID, xdr.ScVec{ - xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counters[count]}, + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counters[i]}, }, - xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counters[count]}, + xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counters[i]}, ) events = append(events, contractEvent) - count += 1 - } txMeta := []xdr.TransactionMeta{ @@ -1284,7 +1279,7 @@ func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ... V1: &xdr.TransactionV1Envelope{ Tx: xdr.Transaction{ SourceAccount: xdr.MustMuxedAddress(keypair.MustRandom().Address()), - //Operations: operations, + // Operations: operations, }, }, } From f5ea54e86ddc60769e608ead98abb0010dee820e Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Wed, 7 Aug 2024 16:00:28 -0700 Subject: [PATCH 44/63] Correct number of topics --- cmd/soroban-rpc/internal/db/event.go | 25 +++++++++++++++---- .../internal/methods/get_events.go | 9 +++++-- 2 files changed, 27 insertions(+), 7 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index e47a5322..0b6e3f43 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -100,7 +100,19 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { } query := sq.Insert(eventTableName). - Columns("id", "contract_id", "event_type", "event_data", "ledger_close_time", "transaction_hash", "topic1", "topic2", "topic3", "topic4") + Columns( + "id", + "contract_id", + "event_type", + "event_data", + "ledger_close_time", + "transaction_hash", + "topic1", + "topic2", + "topic3", + "topic4", + "topic5", + ) for index, e := range txEvents { @@ -121,7 +133,7 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { } // Encode the topics - topicList := make([]string, 4) + topicList := make([]string, 5) for index, segment := range v0.Topics { seg, err := xdr.MarshalBase64(segment) if err != nil { @@ -141,6 +153,7 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { topicList[1], topicList[2], topicList[3], + topicList[4], ) } @@ -242,8 +255,12 @@ func (eventHandler *eventHandler) GetEvents( } err = rows.Scan(&row.eventCursorID, &row.eventData, &row.transactionHash, &row.ledgerCloseTime) + if err != nil { + return fmt.Errorf("failed to scan row: %w", err) + } - id, eventData, ledgerCloseTime, transactionHash := row.eventCursorID, row.eventData, row.ledgerCloseTime, row.transactionHash + id, eventData, ledgerCloseTime := row.eventCursorID, row.eventData, row.ledgerCloseTime + transactionHash := row.transactionHash cur, err := ParseCursor(id) if err != nil { return fmt.Errorf("failed to parse cursor: %w", err) @@ -258,9 +275,7 @@ func (eventHandler *eventHandler) GetEvents( if !f(eventXDR, cur, ledgerCloseTime, &txHash) { return nil } - } - if !foundRows { eventHandler.log. WithField("duration", time.Since(start)). diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 249a16bc..38fcae39 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -372,7 +372,7 @@ func combineEventTypes(filters []EventFilter) []int { } func combineTopics(filters []EventFilter) ([][]string, error) { - encodedTopicsList := make([][]string, 4) + encodedTopicsList := make([][]string, maxTopicsLimit) for _, filter := range filters { if len(filter.Topics) == 0 { @@ -472,7 +472,12 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques eventTypes := combineEventTypes(request.Filters) // Scan function to apply filters - eventScanFunction := func(event xdr.DiagnosticEvent, cursor db.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash) bool { + eventScanFunction := func( + event xdr.DiagnosticEvent, + cursor db.Cursor, + ledgerCloseTimestamp int64, + txHash *xdr.Hash, + ) bool { if request.Matches(event) { found = append(found, entry{cursor, ledgerCloseTimestamp, event, txHash}) } From 12f7790f77f8a35864d1e8db5bd5ddc8e0e9e628 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Wed, 7 Aug 2024 16:13:58 -0700 Subject: [PATCH 45/63] reduce events in benchmark so that tests run --- cmd/soroban-rpc/internal/methods/get_events_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 5a621384..3799ad9a 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -1250,7 +1250,7 @@ func getTxMeta(contractID xdr.Hash, counters [1000]xdr.ScSymbol) []xdr.Transacti // create 250 contract events var events []xdr.ContractEvent - for i := 0; i < 250; i++ { + for i := 0; i < 2; i++ { contractEvent := contractEvent( contractID, xdr.ScVec{ From b9e5f42b573e647cd8aae2dda1d8dd56ece5991c Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Wed, 7 Aug 2024 16:47:33 -0700 Subject: [PATCH 46/63] update events schema --- cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql b/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql index 106824cf..992113f4 100644 --- a/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql +++ b/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql @@ -12,7 +12,8 @@ CREATE TABLE events topic1 BLOB, topic2 BLOB, topic3 BLOB, - topic4 BLOB + topic4 BLOB, + topic5 BLOB ); CREATE INDEX idx_contract_id ON events (contract_id); From 215e6c4fb6e0d94a2e2b0bccf5fc389cde8b4679 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Thu, 8 Aug 2024 13:57:38 -0700 Subject: [PATCH 47/63] Fix topic count --- cmd/soroban-rpc/internal/db/event.go | 2 -- .../internal/db/sqlmigrations/03_events.sql | 3 +-- .../internal/methods/get_events.go | 2 +- .../internal/methods/get_events_test.go | 20 +++++++++---------- 4 files changed, 12 insertions(+), 15 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 0b6e3f43..bcc99a89 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -111,7 +111,6 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { "topic2", "topic3", "topic4", - "topic5", ) for index, e := range txEvents { @@ -153,7 +152,6 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { topicList[1], topicList[2], topicList[3], - topicList[4], ) } diff --git a/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql b/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql index 992113f4..106824cf 100644 --- a/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql +++ b/cmd/soroban-rpc/internal/db/sqlmigrations/03_events.sql @@ -12,8 +12,7 @@ CREATE TABLE events topic1 BLOB, topic2 BLOB, topic3 BLOB, - topic4 BLOB, - topic5 BLOB + topic4 BLOB ); CREATE INDEX idx_contract_id ON events (contract_id); diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 38fcae39..141fbf25 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -372,7 +372,7 @@ func combineEventTypes(filters []EventFilter) []int { } func combineTopics(filters []EventFilter) ([][]string, error) { - encodedTopicsList := make([][]string, maxTopicsLimit) + encodedTopicsList := make([][]string, 4) for _, filter := range filters { if len(filter.Topics) == 0 { diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 3799ad9a..066de84d 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -1172,9 +1172,9 @@ func TestGetEvents(t *testing.T) { // TODO:Clean up once benchmarking is done !! func BenchmarkGetEvents(b *testing.B) { - var counters [10000]xdr.ScSymbol + var counters [1000]xdr.ScSymbol for i := 0; i < len(counters); i++ { - counters[i] = xdr.ScSymbol("TEST-COUNTER-" + strconv.Itoa(i+1) + strconv.Itoa(i%1000)) + counters[i] = xdr.ScSymbol("TEST-COUNTER-" + strconv.Itoa(i+1)) } // counter := xdr.ScSymbol("COUNTER") // requestedCounter := xdr.ScSymbol("REQUESTED") @@ -1193,12 +1193,7 @@ func BenchmarkGetEvents(b *testing.B) { for i := 1; i < 121000; i++ { - var counters [1000]xdr.ScSymbol - for j := 0; j < len(counters); j++ { - counters[j] = xdr.ScSymbol("TEST-COUNTER-" + strconv.Itoa(j+1) + strconv.Itoa(i%1000)) - } - - txMeta := getTxMeta(contractID, counters) + txMeta := getTxMetaWithContractEvents(contractID) ledgerCloseMeta := ledgerCloseMetaWithEvents(uint32(i), now.Unix(), txMeta...) require.NoError(b, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") require.NoError(b, eventW.InsertEvents(ledgerCloseMeta), "ingestion failed for events ") @@ -1246,9 +1241,14 @@ func BenchmarkGetEvents(b *testing.B) { fmt.Printf("%d ns/op (%.3f ms/op)\n", nsPerOp, msPerOp) } -func getTxMeta(contractID xdr.Hash, counters [1000]xdr.ScSymbol) []xdr.TransactionMeta { - // create 250 contract events +func getTxMetaWithContractEvents(contractID xdr.Hash) []xdr.TransactionMeta { + + var counters [1000]xdr.ScSymbol + for j := 0; j < len(counters); j++ { + counters[j] = xdr.ScSymbol("TEST-COUNTER-" + strconv.Itoa(j+1)) + } + // create 2 contract events var events []xdr.ContractEvent for i := 0; i < 2; i++ { contractEvent := contractEvent( From a6d35aec9028cc642fa660c1936dc7eb5fcfd2a7 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Thu, 8 Aug 2024 16:37:13 -0700 Subject: [PATCH 48/63] Fix fetch query to not stop if null topic --- cmd/soroban-rpc/internal/db/event.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index bcc99a89..9c5a478c 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -218,7 +218,7 @@ func (eventHandler *eventHandler) GetEvents( var orConditions sq.Or for i, topic := range topics { if topic == nil { - break + continue } orConditions = append(orConditions, sq.Eq{fmt.Sprintf("topic%d", i+1): topic}) } From 905eaf05b18fe10515fb0c2f0ff0f8e07cd63117 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Thu, 8 Aug 2024 16:44:35 -0700 Subject: [PATCH 49/63] Fix lint pt1 --- cmd/soroban-rpc/internal/methods/get_events.go | 4 ++-- cmd/soroban-rpc/internal/methods/get_events_test.go | 9 ++------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 141fbf25..6b0c217d 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -107,10 +107,10 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { // Validate the paging limit (if it exists) if g.Pagination != nil && g.Pagination.Cursor != nil { if g.StartLedger != 0 || g.EndLedger != 0 { - return fmt.Errorf("startLedger/endLedger and cursor cannot both be set") + return errors.New("startLedger/endLedger and cursor cannot both be set") } } else if g.StartLedger <= 0 { - return fmt.Errorf("startLedger must be positive") + return errors.New("startLedger must be positive") } if g.Pagination != nil && g.Pagination.Limit > maxLimit { diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 066de84d..ddb854a1 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -28,7 +28,6 @@ var passphrase = "passphrase" func TestEventTypeSetMatches(t *testing.T) { var defaultSet eventTypeSet - all := eventTypeSet{} all[EventTypeContract] = nil all[EventTypeDiagnostic] = nil @@ -1191,7 +1190,7 @@ func BenchmarkGetEvents(b *testing.B) { require.NoError(b, err) ledgerW, eventW := write.LedgerWriter(), write.EventWriter() - for i := 1; i < 121000; i++ { + for i := range []int{1, 2, 3} { txMeta := getTxMetaWithContractEvents(contractID) ledgerCloseMeta := ledgerCloseMetaWithEvents(uint32(i), now.Unix(), txMeta...) @@ -1212,12 +1211,9 @@ func BenchmarkGetEvents(b *testing.B) { StartLedger: 1, Filters: []EventFilter{ { - // ContractIDs: []string{"CCVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKVKUD2U"}, - // EventType: map[string]interface{}{EventTypeContract: nil}, Topics: []TopicFilter{ []SegmentFilter{ {scval: &xdr.ScVal{Type: xdr.ScValTypeScvSymbol, Sym: &counters[1]}}, - //{wildcard: &star}, }, }, }, @@ -1248,9 +1244,8 @@ func getTxMetaWithContractEvents(contractID xdr.Hash) []xdr.TransactionMeta { counters[j] = xdr.ScSymbol("TEST-COUNTER-" + strconv.Itoa(j+1)) } - // create 2 contract events var events []xdr.ContractEvent - for i := 0; i < 2; i++ { + for i := 0; i < 10; i++ { contractEvent := contractEvent( contractID, xdr.ScVec{ From 4b8726bc92086b245c44fda088ba472bc74ee82b Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Fri, 9 Aug 2024 12:34:16 -0700 Subject: [PATCH 50/63] Fix trimEvents and lint errors --- cmd/soroban-rpc/internal/db/event.go | 5 +++-- cmd/soroban-rpc/internal/methods/get_events_test.go | 4 +--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 9c5a478c..5c2b29de 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -176,12 +176,13 @@ func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWi if latestLedgerSeq+1 <= retentionWindow { return nil } - cutoff := latestLedgerSeq + 1 - retentionWindow + id := Cursor{Ledger: cutoff}.String() + _, err := sq.StatementBuilder. RunWith(eventHandler.stmtCache). Delete(eventTableName). - Where(sq.Lt{"ledger_sequence": cutoff}). + Where(sq.Lt{"id": id}). Exec() return err } diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index ddb854a1..580ea461 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -1191,7 +1191,6 @@ func BenchmarkGetEvents(b *testing.B) { ledgerW, eventW := write.LedgerWriter(), write.EventWriter() for i := range []int{1, 2, 3} { - txMeta := getTxMetaWithContractEvents(contractID) ledgerCloseMeta := ledgerCloseMetaWithEvents(uint32(i), now.Unix(), txMeta...) require.NoError(b, ledgerW.InsertLedger(ledgerCloseMeta), "ingestion failed for ledger ") @@ -1233,8 +1232,7 @@ func BenchmarkGetEvents(b *testing.B) { totalNs := b.Elapsed() nsPerOp := totalNs.Nanoseconds() / int64(b.N) msPerOp := float64(nsPerOp) / 1e6 - fmt.Printf("Benchmark Results:\n") - fmt.Printf("%d ns/op (%.3f ms/op)\n", nsPerOp, msPerOp) + log.Info("Benchmark Results: (%.3f ms/op) ", msPerOp) } func getTxMetaWithContractEvents(contractID xdr.Hash) []xdr.TransactionMeta { From 8b28d65d7284a1b826d2c0185d3e68972c5238a1 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Fri, 9 Aug 2024 12:41:00 -0700 Subject: [PATCH 51/63] update log info --- cmd/soroban-rpc/internal/methods/get_events_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 580ea461..84186d84 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -1232,7 +1232,7 @@ func BenchmarkGetEvents(b *testing.B) { totalNs := b.Elapsed() nsPerOp := totalNs.Nanoseconds() / int64(b.N) msPerOp := float64(nsPerOp) / 1e6 - log.Info("Benchmark Results: (%.3f ms/op) ", msPerOp) + log.Infof("Benchmark Results: %v ms/op ", msPerOp) } func getTxMetaWithContractEvents(contractID xdr.Hash) []xdr.TransactionMeta { From 051ab4fe0aca86f5de866c905187e35a3f8d8d33 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 12 Aug 2024 15:45:35 -0700 Subject: [PATCH 52/63] Fix more lint errors --- cmd/soroban-rpc/internal/db/event.go | 3 ++- .../internal/methods/get_events.go | 22 ++++++++++--------- .../internal/methods/get_events_test.go | 11 ++++------ 3 files changed, 18 insertions(+), 18 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 5c2b29de..484d4c91 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -132,7 +132,8 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { } // Encode the topics - topicList := make([]string, 5) + maxTopicCount := 4 + topicList := make([]string, maxTopicCount) for index, segment := range v0.Topics { seg, err := xdr.MarshalBase64(segment) if err != nil { diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 6b0c217d..d91c4c80 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -107,10 +107,10 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { // Validate the paging limit (if it exists) if g.Pagination != nil && g.Pagination.Cursor != nil { if g.StartLedger != 0 || g.EndLedger != 0 { - return errors.New("startLedger/endLedger and cursor cannot both be set") + return fmt.Errorf("startLedger/endLedger and cursor cannot both be set") } } else if g.StartLedger <= 0 { - return errors.New("startLedger must be positive") + return fmt.Errorf("startLedger must be positive") } if g.Pagination != nil && g.Pagination.Limit > maxLimit { @@ -119,7 +119,7 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { // Validate filters if len(g.Filters) > maxFiltersLimit { - return errors.New("maximum 5 filters per request") + return fmt.Errorf("maximum 5 filters per request") } for i, filter := range g.Filters { if err := filter.Valid(); err != nil { @@ -154,10 +154,12 @@ var eventTypeFromXDR = map[xdr.ContractEventType]string{ xdr.ContractEventTypeDiagnostic: EventTypeDiagnostic, } -var eventTypeXDRFromEventType = map[string]xdr.ContractEventType{ - EventTypeSystem: xdr.ContractEventTypeSystem, - EventTypeContract: xdr.ContractEventTypeContract, - EventTypeDiagnostic: xdr.ContractEventTypeDiagnostic, +func getEventTypeXDRFromEventType() map[string]xdr.ContractEventType { + return map[string]xdr.ContractEventType{ + EventTypeSystem: xdr.ContractEventTypeSystem, + EventTypeContract: xdr.ContractEventTypeContract, + EventTypeDiagnostic: xdr.ContractEventTypeDiagnostic, + } } type EventFilter struct { @@ -360,7 +362,7 @@ func combineEventTypes(filters []EventFilter) []int { eventTypes := make(map[int]bool) for _, filter := range filters { for _, eventType := range filter.EventType.Keys() { - eventTypeXDR := eventTypeXDRFromEventType[eventType] + eventTypeXDR := getEventTypeXDRFromEventType()[eventType] eventTypes[int(eventTypeXDR)] = true } } @@ -372,7 +374,7 @@ func combineEventTypes(filters []EventFilter) []int { } func combineTopics(filters []EventFilter) ([][]string, error) { - encodedTopicsList := make([][]string, 4) + encodedTopicsList := make([][]string, maxTopicCount) for _, filter := range filters { if len(filter.Topics) == 0 { @@ -529,7 +531,7 @@ func eventInfoForEvent( } // base64-xdr encode the topic - topic := make([]string, 0, 4) + topic := make([]string, 0, maxTopicCount) for _, segment := range v0.Topics { seg, err := xdr.MarshalBase64(segment) if err != nil { diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 84186d84..33d2196c 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -596,7 +596,7 @@ func TestGetEvents(t *testing.T) { contractID := xdr.Hash([32]byte{}) var txMeta []xdr.TransactionMeta - for i := 0; i < 10; i++ { + for range []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} { txMeta = append(txMeta, transactionMetaWithEvents( contractEvent( contractID, @@ -738,7 +738,7 @@ func TestGetEvents(t *testing.T) { var txMeta []xdr.TransactionMeta contractID := xdr.Hash([32]byte{}) - for i := 0; i < 10; i++ { + for i := range []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} { number := xdr.Uint64(i) txMeta = append(txMeta, transactionMetaWithEvents( // Generate a unique topic like /counter/4 for each event so we can check @@ -1169,9 +1169,8 @@ func TestGetEvents(t *testing.T) { }) } -// TODO:Clean up once benchmarking is done !! func BenchmarkGetEvents(b *testing.B) { - var counters [1000]xdr.ScSymbol + var counters [10]xdr.ScSymbol for i := 0; i < len(counters); i++ { counters[i] = xdr.ScSymbol("TEST-COUNTER-" + strconv.Itoa(i+1)) } @@ -1205,7 +1204,6 @@ func BenchmarkGetEvents(b *testing.B) { ledgerReader: db.NewLedgerReader(dbx), } - // star := "*" request := GetEventsRequest{ StartLedger: 1, Filters: []EventFilter{ @@ -1236,14 +1234,13 @@ func BenchmarkGetEvents(b *testing.B) { } func getTxMetaWithContractEvents(contractID xdr.Hash) []xdr.TransactionMeta { - var counters [1000]xdr.ScSymbol for j := 0; j < len(counters); j++ { counters[j] = xdr.ScSymbol("TEST-COUNTER-" + strconv.Itoa(j+1)) } var events []xdr.ContractEvent - for i := 0; i < 10; i++ { + for i := range []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} { contractEvent := contractEvent( contractID, xdr.ScVec{ From f284c9e34ee1c65acda08f380372b86fb41226be Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 12 Aug 2024 16:09:00 -0700 Subject: [PATCH 53/63] Fix more lint errors pt 11 --- cmd/soroban-rpc/internal/methods/get_events.go | 8 ++++---- cmd/soroban-rpc/internal/methods/get_events_test.go | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index d91c4c80..605c24b6 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -107,10 +107,10 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { // Validate the paging limit (if it exists) if g.Pagination != nil && g.Pagination.Cursor != nil { if g.StartLedger != 0 || g.EndLedger != 0 { - return fmt.Errorf("startLedger/endLedger and cursor cannot both be set") + return errors.New("startLedger/endLedger and cursor cannot both be set") //nolint:forbidigo } } else if g.StartLedger <= 0 { - return fmt.Errorf("startLedger must be positive") + return errors.New("startLedger must be positive") } if g.Pagination != nil && g.Pagination.Limit > maxLimit { @@ -119,11 +119,11 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { // Validate filters if len(g.Filters) > maxFiltersLimit { - return fmt.Errorf("maximum 5 filters per request") + return errors.New("maximum 5 filters per request") } for i, filter := range g.Filters { if err := filter.Valid(); err != nil { - return fmt.Errorf("filter %d invalid: %w", i+1, err) + return errors.Wrapf(err, "filter %d invalid: %w", i+1) } } diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index 33d2196c..b52295b0 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -1239,7 +1239,7 @@ func getTxMetaWithContractEvents(contractID xdr.Hash) []xdr.TransactionMeta { counters[j] = xdr.ScSymbol("TEST-COUNTER-" + strconv.Itoa(j+1)) } - var events []xdr.ContractEvent + events := make([]xdr.ContractEvent, 0, 10) for i := range []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} { contractEvent := contractEvent( contractID, From 9c7e870829ee455d4c096f1943ee4517c0843e7a Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 13 Aug 2024 10:57:23 -0700 Subject: [PATCH 54/63] Fix format in error --- cmd/soroban-rpc/internal/methods/get_events.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 605c24b6..fb40beb7 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -123,7 +123,7 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { } for i, filter := range g.Filters { if err := filter.Valid(); err != nil { - return errors.Wrapf(err, "filter %d invalid: %w", i+1) + return errors.Wrapf(err, "filter %d invalid", i+1) //nolint:forbidigo } } From 52bdf2486094d4206059aa07cff8a8399b151901 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 13 Aug 2024 11:57:51 -0700 Subject: [PATCH 55/63] Fix linter error pt 12 --- .../internal/methods/get_events.go | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index fb40beb7..bbdf0c33 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -397,19 +397,24 @@ func combineTopics(filters []EventFilter) ([][]string, error) { return encodedTopicsList, nil } +type entry struct { + cursor db.Cursor + ledgerCloseTimestamp int64 + event xdr.DiagnosticEvent + txHash *xdr.Hash +} + func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsRequest) (GetEventsResponse, error) { if err := request.Valid(h.maxLimit); err != nil { return GetEventsResponse{}, &jrpc2.Error{ - Code: jrpc2.InvalidParams, - Message: err.Error(), + Code: jrpc2.InvalidParams, Message: err.Error(), } } ledgerRange, err := h.ledgerReader.GetLedgerRange(ctx) if err != nil { return GetEventsResponse{}, &jrpc2.Error{ - Code: jrpc2.InternalError, - Message: err.Error(), + Code: jrpc2.InternalError, Message: err.Error(), } } @@ -418,8 +423,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques if request.Pagination != nil { if request.Pagination.Cursor != nil { start = *request.Pagination.Cursor - // increment event index because, when paginating, - // we start with the item right after the cursor + // increment event index because, when paginating, we start with the item right after the cursor start.Event++ } if request.Pagination.Limit > 0 { @@ -435,7 +439,6 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques end := db.Cursor{Ledger: endLedger} cursorRange := db.CursorRange{Start: start, End: end} - // Check if requested start ledger is within stored ledger range if start.Ledger < ledgerRange.FirstLedger.Sequence || start.Ledger > ledgerRange.LastLedger.Sequence { return GetEventsResponse{}, &jrpc2.Error{ Code: jrpc2.InvalidRequest, @@ -447,27 +450,19 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques } } - type entry struct { - cursor db.Cursor - ledgerCloseTimestamp int64 - event xdr.DiagnosticEvent - txHash *xdr.Hash - } found := make([]entry, 0, limit) contractIDs, err := combineContractIDs(request.Filters) if err != nil { return GetEventsResponse{}, &jrpc2.Error{ - Code: jrpc2.InvalidParams, - Message: err.Error(), + Code: jrpc2.InvalidParams, Message: err.Error(), } } topics, err := combineTopics(request.Filters) if err != nil { return GetEventsResponse{}, &jrpc2.Error{ - Code: jrpc2.InvalidParams, - Message: err.Error(), + Code: jrpc2.InvalidParams, Message: err.Error(), } } @@ -475,10 +470,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques // Scan function to apply filters eventScanFunction := func( - event xdr.DiagnosticEvent, - cursor db.Cursor, - ledgerCloseTimestamp int64, - txHash *xdr.Hash, + event xdr.DiagnosticEvent, cursor db.Cursor, ledgerCloseTimestamp int64, txHash *xdr.Hash, ) bool { if request.Matches(event) { found = append(found, entry{cursor, ledgerCloseTimestamp, event, txHash}) @@ -489,8 +481,7 @@ func (h eventsRPCHandler) getEvents(ctx context.Context, request GetEventsReques err = h.dbReader.GetEvents(ctx, cursorRange, contractIDs, topics, eventTypes, eventScanFunction) if err != nil { return GetEventsResponse{}, &jrpc2.Error{ - Code: jrpc2.InvalidRequest, - Message: err.Error(), + Code: jrpc2.InvalidRequest, Message: err.Error(), } } From 99d541108529e6a2bdb88bee05d4d62b7f56f3ac Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 13 Aug 2024 16:04:03 -0700 Subject: [PATCH 56/63] Add nolint for GetEvents as a temp fix. --- cmd/soroban-rpc/internal/db/event.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 484d4c91..6f62d9ff 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -192,6 +192,8 @@ func (eventHandler *eventHandler) trimEvents(latestLedgerSeq uint32, retentionWi // The events are returned in sorted ascending Cursor order. // If f returns false, the scan terminates early (f will not be applied on // remaining events in the range). +// +//nolint:funlen func (eventHandler *eventHandler) GetEvents( ctx context.Context, cursorRange CursorRange, From 7466e5446d50d99df70e0a3cd4ebcd3de2b8dc47 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 16 Aug 2024 13:21:01 -0400 Subject: [PATCH 57/63] Add events table migration (#262) * Fix migrations - 1 * Make migrations sequential - 1 * Make migrations sequential - 2 * Fix failing unittest - 1 * Fix linting errors - 1 * Fix failing integration test - 1 * Remove %w from Fatal strings * refactor migrationApplierFactoryF * Add ledger seq to fatal error string * Add comments - 1 * Fix - 1 * Optimise migrations - 1 * Optimise migrations - 2 * Optimise migrations - 3 * Fix linting - 1 * Fix linting - 2 * Remove dupicate latest ledger fetch code * Rollback db in daemon instead of migration * Remove unused constant * Remove unused constant - 2 * Add rollback() statement * Small change * Abstract transaction and rollback management inside migration code * Fix failing unittest --- cmd/soroban-rpc/internal/daemon/daemon.go | 39 ++--- cmd/soroban-rpc/internal/db/event.go | 27 ++-- cmd/soroban-rpc/internal/db/migration.go | 147 +++++++++--------- cmd/soroban-rpc/internal/db/transaction.go | 30 ++-- .../internal/feewindow/feewindow.go | 14 +- .../internal/ingest/service_test.go | 2 +- 6 files changed, 127 insertions(+), 132 deletions(-) diff --git a/cmd/soroban-rpc/internal/daemon/daemon.go b/cmd/soroban-rpc/internal/daemon/daemon.go index b1983cb8..7d3d0adb 100644 --- a/cmd/soroban-rpc/internal/daemon/daemon.go +++ b/cmd/soroban-rpc/internal/daemon/daemon.go @@ -289,38 +289,43 @@ func MustNew(cfg *config.Config, logger *supportlog.Entry) *Daemon { // mustInitializeStorage initializes the storage using what was on the DB func (d *Daemon) mustInitializeStorage(cfg *config.Config) *feewindow.FeeWindows { - feewindows := feewindow.NewFeeWindows( + feeWindows := feewindow.NewFeeWindows( cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow, cfg.NetworkPassphrase, + d.db, ) readTxMetaCtx, cancelReadTxMeta := context.WithTimeout(context.Background(), cfg.IngestionTimeout) defer cancelReadTxMeta() var initialSeq uint32 var currentSeq uint32 - dataMigrations, err := db.BuildMigrations(readTxMetaCtx, d.logger, d.db, cfg) + applicableRange, err := db.GetMigrationLedgerRange(readTxMetaCtx, d.db, cfg.HistoryRetentionWindow) if err != nil { - d.logger.WithError(err).Fatal("could not build migrations") + d.logger.WithError(err).Fatal("could not get ledger range for migration") } - // Merge migrations range and fee stats range to get the applicable range - latestLedger, err := db.NewLedgerEntryReader(d.db).GetLatestLedgerSequence(readTxMetaCtx) + maxFeeRetentionWindow := max(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow) + feeStatsRange, err := db.GetMigrationLedgerRange(readTxMetaCtx, d.db, maxFeeRetentionWindow) if err != nil { - d.logger.WithError(err).Fatal("failed to get latest ledger sequence: %w", err) + d.logger.WithError(err).Fatal("could not get ledger range for fee stats") } - maxFeeRetentionWindow := max(cfg.ClassicFeeStatsLedgerRetentionWindow, cfg.SorobanFeeStatsLedgerRetentionWindow) - ledgerSeqRange := &db.LedgerSeqRange{FirstLedgerSeq: latestLedger - maxFeeRetentionWindow, LastLedgerSeq: latestLedger} - applicableRange := dataMigrations.ApplicableRange() - ledgerSeqRange = ledgerSeqRange.Merge(applicableRange) + // Combine the ledger range for fees, events and transactions + ledgerSeqRange := feeStatsRange.Merge(applicableRange) + + dataMigrations, err := db.BuildMigrations(readTxMetaCtx, d.logger, d.db, cfg.NetworkPassphrase, ledgerSeqRange) + if err != nil { + d.logger.WithError(err).Fatal("could not build migrations") + } + // Apply migration for events, transactions and fee stats err = db.NewLedgerReader(d.db).StreamLedgerRange( readTxMetaCtx, ledgerSeqRange.FirstLedgerSeq, ledgerSeqRange.LastLedgerSeq, - func(txmeta xdr.LedgerCloseMeta) error { - currentSeq = txmeta.LedgerSequence() + func(txMeta xdr.LedgerCloseMeta) error { + currentSeq = txMeta.LedgerSequence() if initialSeq == 0 { initialSeq = currentSeq d.logger.WithFields(supportlog.F{ @@ -332,14 +337,12 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) *feewindow.FeeWindows }).Debug("still initializing in-memory store") } - if err := feewindows.IngestFees(txmeta); err != nil { + if err = feeWindows.IngestFees(txMeta); err != nil { d.logger.WithError(err).Fatal("could not initialize fee stats") } - if applicableRange.IsLedgerIncluded(currentSeq) { - if err := dataMigrations.Apply(readTxMetaCtx, txmeta); err != nil { - d.logger.WithError(err).Fatal("could not run migrations") - } + if err := dataMigrations.Apply(readTxMetaCtx, txMeta); err != nil { + d.logger.WithError(err).Fatal("could not apply migration for ledger ", currentSeq) } return nil }) @@ -356,7 +359,7 @@ func (d *Daemon) mustInitializeStorage(cfg *config.Config) *feewindow.FeeWindows }).Info("finished initializing in-memory store and applying DB data migrations") } - return feewindows + return feeWindows } func (d *Daemon) Run() { diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 6f62d9ff..01a32192 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -322,26 +322,21 @@ func (e *eventTableMigration) Apply(_ context.Context, meta xdr.LedgerCloseMeta) } func newEventTableMigration( + _ context.Context, logger *log.Entry, - retentionWindow uint32, passphrase string, + ledgerSeqRange *LedgerSeqRange, ) migrationApplierFactory { - return migrationApplierFactoryF(func(db *DB, latestLedger uint32) (MigrationApplier, error) { - firstLedgerToMigrate := firstLedger - writer := &eventHandler{ - log: logger, - db: db, - stmtCache: sq.NewStmtCache(db.GetTx()), - passphrase: passphrase, - } - if latestLedger > retentionWindow { - firstLedgerToMigrate = latestLedger - retentionWindow - } - + return migrationApplierFactoryF(func(db *DB) (MigrationApplier, error) { migration := eventTableMigration{ - firstLedger: firstLedgerToMigrate, - lastLedger: latestLedger, - writer: writer, + firstLedger: ledgerSeqRange.FirstLedgerSeq, + lastLedger: ledgerSeqRange.LastLedgerSeq, + writer: &eventHandler{ + log: logger, + db: db, + stmtCache: sq.NewStmtCache(db.GetTx()), + passphrase: passphrase, + }, } return &migration, nil }) diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go index 4c7c9205..69df9de2 100644 --- a/cmd/soroban-rpc/internal/db/migration.go +++ b/cmd/soroban-rpc/internal/db/migration.go @@ -7,8 +7,11 @@ import ( "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" +) - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config" +const ( + transactionsMigrationName = "TransactionsTable" + eventsMigrationName = "EventsTable" ) type LedgerSeqRange struct { @@ -47,65 +50,59 @@ type MigrationApplier interface { Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error } +type migrationApplierF func(context.Context, *log.Entry, string, *LedgerSeqRange) migrationApplierFactory + type migrationApplierFactory interface { - New(db *DB, latestLedger uint32) (MigrationApplier, error) + New(db *DB) (MigrationApplier, error) } -type migrationApplierFactoryF func(db *DB, latestLedger uint32) (MigrationApplier, error) +type migrationApplierFactoryF func(db *DB) (MigrationApplier, error) -func (m migrationApplierFactoryF) New(db *DB, latestLedger uint32) (MigrationApplier, error) { - return m(db, latestLedger) +func (m migrationApplierFactoryF) New(db *DB) (MigrationApplier, error) { + return m(db) } type Migration interface { MigrationApplier Commit(ctx context.Context) error - Rollback(ctx context.Context) error } -type multiMigration []Migration +type MultiMigration struct { + migrations []Migration + db *DB +} -func (mm multiMigration) ApplicableRange() *LedgerSeqRange { +func (mm MultiMigration) ApplicableRange() *LedgerSeqRange { var result *LedgerSeqRange - for _, m := range mm { + for _, m := range mm.migrations { result = m.ApplicableRange().Merge(result) } return result } -func (mm multiMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error { +func (mm MultiMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error { var err error - for _, m := range mm { + for _, m := range mm.migrations { ledgerSeq := meta.LedgerSequence() if !m.ApplicableRange().IsLedgerIncluded(ledgerSeq) { // The range of a sub-migration can be smaller than the global range. continue } if localErr := m.Apply(ctx, meta); localErr != nil { - err = errors.Join(err, localErr) + err = errors.Join(err, localErr, mm.db.Rollback()) } } return err } -func (mm multiMigration) Commit(ctx context.Context) error { +func (mm MultiMigration) Commit(ctx context.Context) error { var err error - for _, m := range mm { + for _, m := range mm.migrations { if localErr := m.Commit(ctx); localErr != nil { - err = errors.Join(err, localErr) - } - } - return err -} - -func (mm multiMigration) Rollback(ctx context.Context) error { - var err error - for _, m := range mm { - if localErr := m.Rollback(ctx); localErr != nil { - err = errors.Join(err, localErr) + err = errors.Join(err, localErr, mm.db.Rollback()) } } - return err + return mm.db.Commit() } // guardedMigration is a db data migration whose application is guarded by a boolean in the meta table @@ -122,32 +119,18 @@ type guardedMigration struct { func newGuardedDataMigration( ctx context.Context, uniqueMigrationName string, logger *log.Entry, factory migrationApplierFactory, db *DB, ) (Migration, error) { - migrationDB := &DB{ - cache: db.cache, - SessionInterface: db.SessionInterface.Clone(), - } - if err := migrationDB.Begin(ctx); err != nil { - return nil, err - } metaKey := "Migration" + uniqueMigrationName + "Done" - previouslyMigrated, err := getMetaBool(ctx, migrationDB, metaKey) + previouslyMigrated, err := getMetaBool(ctx, db, metaKey) if err != nil && !errors.Is(err, ErrEmptyDB) { - err = errors.Join(err, migrationDB.Rollback()) return nil, err } - latestLedger, err := NewLedgerEntryReader(db).GetLatestLedgerSequence(ctx) - if err != nil && !errors.Is(err, ErrEmptyDB) { - err = errors.Join(err, migrationDB.Rollback()) - return nil, fmt.Errorf("failed to get latest ledger sequence: %w", err) - } - applier, err := factory.New(migrationDB, latestLedger) + applier, err := factory.New(db) if err != nil { - err = errors.Join(err, migrationDB.Rollback()) return nil, err } guardedMigration := &guardedMigration{ guardMetaKey: metaKey, - db: migrationDB, + db: db, migration: applier, alreadyMigrated: previouslyMigrated, logger: logger, @@ -179,46 +162,58 @@ func (g *guardedMigration) Commit(ctx context.Context) error { if g.alreadyMigrated { return nil } - err := setMetaBool(ctx, g.db, g.guardMetaKey, true) - if err != nil { - return errors.Join(err, g.Rollback(ctx)) - } - return g.db.Commit() + return setMetaBool(ctx, g.db, g.guardMetaKey, true) } -func (g *guardedMigration) Rollback(_ context.Context) error { - return g.db.Rollback() +func GetMigrationLedgerRange(ctx context.Context, db *DB, retentionWindow uint32) (*LedgerSeqRange, error) { + firstLedgerToMigrate := firstLedger + latestLedger, err := NewLedgerEntryReader(db).GetLatestLedgerSequence(ctx) + if err != nil && !errors.Is(err, ErrEmptyDB) { + return nil, fmt.Errorf("failed to get latest ledger sequence: %w", err) + } + if latestLedger > retentionWindow { + firstLedgerToMigrate = latestLedger - retentionWindow + } + return &LedgerSeqRange{ + FirstLedgerSeq: firstLedgerToMigrate, + LastLedgerSeq: latestLedger, + }, nil } -func BuildMigrations(ctx context.Context, logger *log.Entry, db *DB, cfg *config.Config) (Migration, error) { - var migrations []Migration - - migrationName := "TransactionsTable" - logger = logger.WithField("migration", migrationName) - factory := newTransactionTableMigration( - ctx, - logger, - cfg.HistoryRetentionWindow, - cfg.NetworkPassphrase, - ) - - m1, err := newGuardedDataMigration(ctx, migrationName, logger, factory, db) +func BuildMigrations(ctx context.Context, logger *log.Entry, db *DB, networkPassphrase string, + ledgerSeqRange *LedgerSeqRange, +) (MultiMigration, error) { + // Start a common db transaction for the entire migration duration + err := db.Begin(ctx) if err != nil { - return nil, fmt.Errorf("creating guarded transaction migration: %w", err) + return MultiMigration{}, errors.Join(err, db.Rollback()) } - migrations = append(migrations, m1) - eventMigrationName := "EventsTable" - eventFactory := newEventTableMigration( - logger.WithField("migration", eventMigrationName), - cfg.HistoryRetentionWindow, - cfg.NetworkPassphrase, - ) - m2, err := newGuardedDataMigration(ctx, eventMigrationName, logger, eventFactory, db) - if err != nil { - return nil, fmt.Errorf("creating guarded transaction migration: %w", err) + migrationNameToFunc := map[string]migrationApplierF{ + transactionsMigrationName: newTransactionTableMigration, + eventsMigrationName: newEventTableMigration, } - migrations = append(migrations, m2) - return multiMigration(migrations), nil + migrations := make([]Migration, 0, len(migrationNameToFunc)) + + for migrationName, migrationFunc := range migrationNameToFunc { + migrationLogger := logger.WithField("migration", migrationName) + factory := migrationFunc( + ctx, + migrationLogger, + networkPassphrase, + ledgerSeqRange, + ) + + guardedM, err := newGuardedDataMigration(ctx, migrationName, migrationLogger, factory, db) + if err != nil { + return MultiMigration{}, errors.Join(fmt.Errorf( + "could not create guarded migration for %s: %w", migrationName, err), db.Rollback()) + } + migrations = append(migrations, guardedM) + } + return MultiMigration{ + migrations: migrations, + db: db, + }, nil } diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index 1ef6818b..931d6715 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -266,20 +266,13 @@ func (t *transactionTableMigration) Apply(_ context.Context, meta xdr.LedgerClos return t.writer.InsertTransactions(meta) } -func newTransactionTableMigration(ctx context.Context, logger *log.Entry, - retentionWindow uint32, passphrase string, +func newTransactionTableMigration( + ctx context.Context, + logger *log.Entry, + passphrase string, + ledgerSeqRange *LedgerSeqRange, ) migrationApplierFactory { - return migrationApplierFactoryF(func(db *DB, latestLedger uint32) (MigrationApplier, error) { - firstLedgerToMigrate := uint32(2) //nolint:mnd - writer := &transactionHandler{ - log: logger, - db: db, - stmtCache: sq.NewStmtCache(db.GetTx()), - passphrase: passphrase, - } - if latestLedger > retentionWindow { - firstLedgerToMigrate = latestLedger - retentionWindow - } + return migrationApplierFactoryF(func(db *DB) (MigrationApplier, error) { // Truncate the table, since it may contain data, causing insert conflicts later on. // (the migration was shipped after the actual transactions table change) _, err := db.Exec(ctx, sq.Delete(transactionTableName)) @@ -287,9 +280,14 @@ func newTransactionTableMigration(ctx context.Context, logger *log.Entry, return nil, fmt.Errorf("couldn't delete table %q: %w", transactionTableName, err) } migration := transactionTableMigration{ - firstLedger: firstLedgerToMigrate, - lastLedger: latestLedger, - writer: writer, + firstLedger: ledgerSeqRange.FirstLedgerSeq, + lastLedger: ledgerSeqRange.LastLedgerSeq, + writer: &transactionHandler{ + log: logger, + db: db, + stmtCache: sq.NewStmtCache(db.GetTx()), + passphrase: passphrase, + }, } return &migration, nil }) diff --git a/cmd/soroban-rpc/internal/feewindow/feewindow.go b/cmd/soroban-rpc/internal/feewindow/feewindow.go index 3d662fbc..b1d40bee 100644 --- a/cmd/soroban-rpc/internal/feewindow/feewindow.go +++ b/cmd/soroban-rpc/internal/feewindow/feewindow.go @@ -2,6 +2,7 @@ package feewindow import ( + "errors" "io" "slices" "sync" @@ -9,6 +10,7 @@ import ( "github.com/stellar/go/ingest" "github.com/stellar/go/xdr" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) @@ -128,20 +130,22 @@ type FeeWindows struct { SorobanInclusionFeeWindow *FeeWindow ClassicFeeWindow *FeeWindow networkPassPhrase string + db *db.DB } -func NewFeeWindows(classicRetention uint32, sorobanRetetion uint32, networkPassPhrase string) *FeeWindows { +func NewFeeWindows(classicRetention uint32, sorobanRetetion uint32, networkPassPhrase string, db *db.DB) *FeeWindows { return &FeeWindows{ SorobanInclusionFeeWindow: NewFeeWindow(sorobanRetetion), ClassicFeeWindow: NewFeeWindow(classicRetention), networkPassPhrase: networkPassPhrase, + db: db, } } func (fw *FeeWindows) IngestFees(meta xdr.LedgerCloseMeta) error { reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(fw.networkPassPhrase, meta) if err != nil { - return err + return errors.Join(err, fw.db.Rollback()) } var sorobanInclusionFees []uint64 var classicFees []uint64 @@ -151,7 +155,7 @@ func (fw *FeeWindows) IngestFees(meta xdr.LedgerCloseMeta) error { break } if err != nil { - return err + return errors.Join(err, fw.db.Rollback()) } feeCharged := uint64(tx.Result.Result.FeeCharged) ops := tx.Envelope.Operations() @@ -182,11 +186,11 @@ func (fw *FeeWindows) IngestFees(meta xdr.LedgerCloseMeta) error { BucketContent: classicFees, } if err := fw.ClassicFeeWindow.AppendLedgerFees(bucket); err != nil { - return err + return errors.Join(err, fw.db.Rollback()) } bucket.BucketContent = sorobanInclusionFees if err := fw.SorobanInclusionFeeWindow.AppendLedgerFees(bucket); err != nil { - return err + return errors.Join(err, fw.db.Rollback()) } return nil } diff --git a/cmd/soroban-rpc/internal/ingest/service_test.go b/cmd/soroban-rpc/internal/ingest/service_test.go index 2eeaf89f..f3f2d523 100644 --- a/cmd/soroban-rpc/internal/ingest/service_test.go +++ b/cmd/soroban-rpc/internal/ingest/service_test.go @@ -67,7 +67,7 @@ func TestIngestion(t *testing.T) { config := Config{ Logger: supportlog.New(), DB: mockDB, - FeeWindows: feewindow.NewFeeWindows(1, 1, network.TestNetworkPassphrase), + FeeWindows: feewindow.NewFeeWindows(1, 1, network.TestNetworkPassphrase, nil), LedgerBackend: mockLedgerBackend, Daemon: daemon, NetworkPassPhrase: network.TestNetworkPassphrase, From 8e33b1ebe7dc05a943f37c641c4bf2097da215e8 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 19 Aug 2024 15:42:18 -0700 Subject: [PATCH 58/63] Address review comments --- cmd/soroban-rpc/internal/methods/get_events.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index bbdf0c33..988519d6 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -107,7 +107,7 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { // Validate the paging limit (if it exists) if g.Pagination != nil && g.Pagination.Cursor != nil { if g.StartLedger != 0 || g.EndLedger != 0 { - return errors.New("startLedger/endLedger and cursor cannot both be set") //nolint:forbidigo + return errors.New("ledger ranges and cursor cannot both be set") //nolint:forbidigo } } else if g.StartLedger <= 0 { return errors.New("startLedger must be positive") @@ -123,7 +123,7 @@ func (g *GetEventsRequest) Valid(maxLimit uint) error { } for i, filter := range g.Filters { if err := filter.Valid(); err != nil { - return errors.Wrapf(err, "filter %d invalid", i+1) //nolint:forbidigo + return fmt.Errorf("filter %d invalid: %w", i+1, err) } } From 7d4742758ca6fd06bf589229b7062780b9f43286 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 19 Aug 2024 18:52:36 -0700 Subject: [PATCH 59/63] Store binary of topics instead of string --- cmd/soroban-rpc/internal/db/event.go | 10 +++++----- cmd/soroban-rpc/internal/methods/get_events.go | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index 01a32192..aa700f18 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -32,7 +32,7 @@ type EventReader interface { ctx context.Context, cursorRange CursorRange, contractIDs [][]byte, - topics [][]string, + topics [][][]byte, eventTypes []int, f ScanFunction, ) error @@ -121,7 +121,7 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { } id := Cursor{Ledger: lcm.LedgerSequence(), Tx: tx.Index, Op: 0, Event: uint32(index)}.String() - eventBlob, err := xdr.NewEncodingBuffer().MarshalBinary(&e) + eventBlob, err := e.MarshalBinary() if err != nil { return err } @@ -133,9 +133,9 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { // Encode the topics maxTopicCount := 4 - topicList := make([]string, maxTopicCount) + topicList := make([][]byte, maxTopicCount) for index, segment := range v0.Topics { - seg, err := xdr.MarshalBase64(segment) + seg, err := segment.MarshalBinary() if err != nil { return err } @@ -198,7 +198,7 @@ func (eventHandler *eventHandler) GetEvents( ctx context.Context, cursorRange CursorRange, contractIDs [][]byte, - topics [][]string, + topics [][][]byte, eventTypes []int, f ScanFunction, ) error { diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index 988519d6..a9e74c0f 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -373,20 +373,20 @@ func combineEventTypes(filters []EventFilter) []int { return uniqueEventTypes } -func combineTopics(filters []EventFilter) ([][]string, error) { - encodedTopicsList := make([][]string, maxTopicCount) +func combineTopics(filters []EventFilter) ([][][]byte, error) { + encodedTopicsList := make([][][]byte, maxTopicCount) for _, filter := range filters { if len(filter.Topics) == 0 { - return [][]string{}, nil + return [][][]byte{}, nil } for _, topicFilter := range filter.Topics { for i, segmentFilter := range topicFilter { if segmentFilter.wildcard == nil && segmentFilter.scval != nil { - encodedTopic, err := xdr.MarshalBase64(segmentFilter.scval) + encodedTopic, err := segmentFilter.scval.MarshalBinary() if err != nil { - return [][]string{}, fmt.Errorf("failed to marshal segment: %w", err) + return [][][]byte{}, fmt.Errorf("failed to marshal segment: %w", err) } encodedTopicsList[i] = append(encodedTopicsList[i], encodedTopic) } From 435b7e04c6a4606d20c11dcecec2200da74e8b09 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 19 Aug 2024 19:08:59 -0700 Subject: [PATCH 60/63] Unify min/max topic count in event.go --- cmd/soroban-rpc/internal/db/event.go | 5 +++-- cmd/soroban-rpc/internal/methods/get_events.go | 13 ++++--------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index aa700f18..bc67f532 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -19,6 +19,8 @@ import ( const ( eventTableName = "events" firstLedger = uint32(2) + MinTopicCount = 1 + MaxTopicCount = 4 ) // EventWriter is used during ingestion of events from LCM to DB @@ -132,8 +134,7 @@ func (eventHandler *eventHandler) InsertEvents(lcm xdr.LedgerCloseMeta) error { } // Encode the topics - maxTopicCount := 4 - topicList := make([][]byte, maxTopicCount) + topicList := make([][]byte, MaxTopicCount) for index, segment := range v0.Topics { seg, err := segment.MarshalBinary() if err != nil { diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index a9e74c0f..b151d8fd 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -230,16 +230,11 @@ func (e *EventFilter) matchesTopics(event xdr.ContractEvent) bool { type TopicFilter []SegmentFilter -const ( - minTopicCount = 1 - maxTopicCount = 4 -) - func (t *TopicFilter) Valid() error { - if len(*t) < minTopicCount { + if len(*t) < db.MinTopicCount { return errors.New("topic must have at least one segment") } - if len(*t) > maxTopicCount { + if len(*t) > db.MaxTopicCount { return errors.New("topic cannot have more than 4 segments") } for i, segment := range *t { @@ -374,7 +369,7 @@ func combineEventTypes(filters []EventFilter) []int { } func combineTopics(filters []EventFilter) ([][][]byte, error) { - encodedTopicsList := make([][][]byte, maxTopicCount) + encodedTopicsList := make([][][]byte, db.MaxTopicCount) for _, filter := range filters { if len(filter.Topics) == 0 { @@ -522,7 +517,7 @@ func eventInfoForEvent( } // base64-xdr encode the topic - topic := make([]string, 0, maxTopicCount) + topic := make([]string, 0, db.MaxTopicCount) for _, segment := range v0.Topics { seg, err := xdr.MarshalBase64(segment) if err != nil { From 4f7b675fc5df3e0df44f24e3042337c10d3dedaf Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Mon, 19 Aug 2024 20:29:28 -0700 Subject: [PATCH 61/63] Address review comments for event Types and fix unit tests --- cmd/soroban-rpc/internal/db/event.go | 2 +- cmd/soroban-rpc/internal/methods/get_events.go | 8 +++++--- cmd/soroban-rpc/internal/methods/get_events_test.go | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event.go b/cmd/soroban-rpc/internal/db/event.go index bc67f532..81f28c3d 100644 --- a/cmd/soroban-rpc/internal/db/event.go +++ b/cmd/soroban-rpc/internal/db/event.go @@ -302,7 +302,7 @@ func (eventHandler *eventHandler) GetEvents( WithField("duration", time.Since(start)). Debugf("Fetched and decoded all the events with filters - contractIDs: %v ", contractIDs) - return nil + return rows.Err() } type eventTableMigration struct { diff --git a/cmd/soroban-rpc/internal/methods/get_events.go b/cmd/soroban-rpc/internal/methods/get_events.go index b151d8fd..28696ea6 100644 --- a/cmd/soroban-rpc/internal/methods/get_events.go +++ b/cmd/soroban-rpc/internal/methods/get_events.go @@ -23,6 +23,7 @@ const ( maxContractIDsLimit = 5 maxTopicsLimit = 5 maxFiltersLimit = 5 + maxEventTypes = 3 ) type eventTypeSet map[string]interface{} @@ -354,14 +355,15 @@ func combineContractIDs(filters []EventFilter) ([][]byte, error) { } func combineEventTypes(filters []EventFilter) []int { - eventTypes := make(map[int]bool) + eventTypes := set.NewSet[int](maxEventTypes) + for _, filter := range filters { for _, eventType := range filter.EventType.Keys() { eventTypeXDR := getEventTypeXDRFromEventType()[eventType] - eventTypes[int(eventTypeXDR)] = true + eventTypes.Add(int(eventTypeXDR)) } } - uniqueEventTypes := make([]int, 0, len(eventTypes)) + uniqueEventTypes := make([]int, 0, maxEventTypes) for eventType := range eventTypes { uniqueEventTypes = append(uniqueEventTypes, eventType) } diff --git a/cmd/soroban-rpc/internal/methods/get_events_test.go b/cmd/soroban-rpc/internal/methods/get_events_test.go index b52295b0..bfe2b858 100644 --- a/cmd/soroban-rpc/internal/methods/get_events_test.go +++ b/cmd/soroban-rpc/internal/methods/get_events_test.go @@ -418,7 +418,7 @@ func TestGetEventsRequestValid(t *testing.T) { StartLedger: 1, Filters: []EventFilter{}, Pagination: &PaginationOptions{Cursor: &db.Cursor{}}, - }).Valid(1000), "startLedger/endLedger and cursor cannot both be set") + }).Valid(1000), "ledger ranges and cursor cannot both be set") assert.NoError(t, (&GetEventsRequest{ StartLedger: 1, From 1fbd38121f1e709af538bb9a66f428e7e64f8c97 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 20 Aug 2024 12:21:40 -0700 Subject: [PATCH 62/63] cleanup --- cmd/soroban-rpc/internal/db/event_test.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/soroban-rpc/internal/db/event_test.go b/cmd/soroban-rpc/internal/db/event_test.go index 60ad439a..568f05dd 100644 --- a/cmd/soroban-rpc/internal/db/event_test.go +++ b/cmd/soroban-rpc/internal/db/event_test.go @@ -166,5 +166,12 @@ func TestInsertEvents(t *testing.T) { err = eventW.InsertEvents(ledgerCloseMeta) assert.NoError(t, err) - // TODO: Call getEvents and validate events data. + eventReader := NewEventReader(log, db, passphrase) + start := Cursor{Ledger: 1} + end := Cursor{Ledger: 100} + cursorRange := CursorRange{Start: start, End: end} + + err = eventReader.GetEvents(ctx, cursorRange, nil, nil, nil, nil) + require.NoError(t, err) + } From 187e9d638f40089f739b4617845d5de5d6676e76 Mon Sep 17 00:00:00 2001 From: Prit Sheth Date: Tue, 20 Aug 2024 12:51:40 -0700 Subject: [PATCH 63/63] Fix linter errors for one last time --- cmd/soroban-rpc/internal/db/event_test.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/event_test.go b/cmd/soroban-rpc/internal/db/event_test.go index 568f05dd..4464c1f2 100644 --- a/cmd/soroban-rpc/internal/db/event_test.go +++ b/cmd/soroban-rpc/internal/db/event_test.go @@ -6,7 +6,6 @@ import ( "time" "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stellar/go/keypair" @@ -43,7 +42,11 @@ func contractEvent(contractID xdr.Hash, topic []xdr.ScVal, body xdr.ScVal) xdr.C } } -func ledgerCloseMetaWithEvents(sequence uint32, closeTimestamp int64, txMeta ...xdr.TransactionMeta) xdr.LedgerCloseMeta { +func ledgerCloseMetaWithEvents( + sequence uint32, + closeTimestamp int64, + txMeta ...xdr.TransactionMeta, +) xdr.LedgerCloseMeta { var txProcessing []xdr.TransactionResultMeta var phases []xdr.TransactionPhase @@ -145,7 +148,7 @@ func TestInsertEvents(t *testing.T) { counter := xdr.ScSymbol("COUNTER") var txMeta []xdr.TransactionMeta - for i := 0; i < 10; i++ { + for range []int{0, 1, 2, 3, 4, 5, 6, 7, 8, 9} { txMeta = append(txMeta, transactionMetaWithEvents( contractEvent( contractID, @@ -164,7 +167,7 @@ func TestInsertEvents(t *testing.T) { eventW := write.EventWriter() err = eventW.InsertEvents(ledgerCloseMeta) - assert.NoError(t, err) + require.NoError(t, err) eventReader := NewEventReader(log, db, passphrase) start := Cursor{Ledger: 1} @@ -173,5 +176,4 @@ func TestInsertEvents(t *testing.T) { err = eventReader.GetEvents(ctx, cursorRange, nil, nil, nil, nil) require.NoError(t, err) - }