From 62370cf68d514773f1608e992bd65454ea920d55 Mon Sep 17 00:00:00 2001 From: Shawn Reuland Date: Thu, 21 Jul 2022 18:14:45 -0700 Subject: [PATCH] #4433: used CheckpointManager for some of checkpoint math --- exp/lighthorizon/actions/accounts.go | 8 ++++---- exp/lighthorizon/actions/main.go | 3 ++- exp/lighthorizon/services/main.go | 28 ++++++++++++++------------ exp/lighthorizon/services/main_test.go | 28 +++++++++++++++----------- 4 files changed, 37 insertions(+), 30 deletions(-) diff --git a/exp/lighthorizon/actions/accounts.go b/exp/lighthorizon/actions/accounts.go index f1d17c1103..7d045ceb74 100644 --- a/exp/lighthorizon/actions/accounts.go +++ b/exp/lighthorizon/actions/accounts.go @@ -65,7 +65,7 @@ func NewTXByAccountHandler(lightHorizon services.LightHorizon) func(http.Respons txns, err := lightHorizon.Transactions.GetTransactionsByAccount(ctx, paginate.Cursor, paginate.Limit, accountId) if err != nil { log.Error(err) - sendErrorResponse(w, http.StatusInternalServerError, "") + sendErrorResponse(w, http.StatusInternalServerError, err.Error()) return } @@ -74,7 +74,7 @@ func NewTXByAccountHandler(lightHorizon services.LightHorizon) func(http.Respons response, err = adapters.PopulateTransaction(r, &txn) if err != nil { log.Error(err) - sendErrorResponse(w, http.StatusInternalServerError, "") + sendErrorResponse(w, http.StatusInternalServerError, err.Error()) return } @@ -107,7 +107,7 @@ func NewOpsByAccountHandler(lightHorizon services.LightHorizon) func(http.Respon ops, err := lightHorizon.Operations.GetOperationsByAccount(ctx, paginate.Cursor, paginate.Limit, accountId) if err != nil { log.Error(err) - sendErrorResponse(w, http.StatusInternalServerError, "") + sendErrorResponse(w, http.StatusInternalServerError, err.Error()) return } @@ -116,7 +116,7 @@ func NewOpsByAccountHandler(lightHorizon services.LightHorizon) func(http.Respon response, err = adapters.PopulateOperation(r, &op) if err != nil { log.Error(err) - sendErrorResponse(w, http.StatusInternalServerError, "") + sendErrorResponse(w, http.StatusInternalServerError, err.Error()) return } diff --git a/exp/lighthorizon/actions/main.go b/exp/lighthorizon/actions/main.go index 88a9dc11b4..c091dad1ab 100644 --- a/exp/lighthorizon/actions/main.go +++ b/exp/lighthorizon/actions/main.go @@ -3,6 +3,7 @@ package actions import ( "embed" "encoding/json" + "fmt" "net/http" "net/url" "strconv" @@ -65,7 +66,7 @@ func sendPageResponse(w http.ResponseWriter, page hal.Page) { func sendErrorResponse(w http.ResponseWriter, errorCode int, errorMsg string) { if errorMsg != "" { - http.Error(w, errorMsg, errorCode) + http.Error(w, fmt.Sprintf("Error: %s", errorMsg), errorCode) } else { http.Error(w, string(serverError), errorCode) } diff --git a/exp/lighthorizon/services/main.go b/exp/lighthorizon/services/main.go index 1d04096dcd..bc29e8d218 100644 --- a/exp/lighthorizon/services/main.go +++ b/exp/lighthorizon/services/main.go @@ -7,6 +7,7 @@ import ( "github.com/stellar/go/exp/lighthorizon/archive" "github.com/stellar/go/exp/lighthorizon/common" "github.com/stellar/go/exp/lighthorizon/index" + "github.com/stellar/go/historyarchive" "github.com/stellar/go/toid" "github.com/stellar/go/xdr" @@ -18,6 +19,10 @@ const ( allIndexes = "all/all" ) +var ( + checkpointManager = historyarchive.NewCheckpointManager(0) +) + type LightHorizon struct { Operations OperationsService Transactions TransactionsService @@ -111,7 +116,7 @@ func searchTxByAccount(ctx context.Context, cursor int64, accountId string, conf for { ledger, ledgerErr := config.Archive.GetLedger(ctx, uint32(nextLedger)) if ledgerErr != nil { - return errors.Wrapf(ledgerErr, "ledger export state is out of sync, missing ledger %v from checkpoint %v", nextLedger, nextLedger/64) + return errors.Wrapf(ledgerErr, "ledger export state is out of sync, missing ledger %v from checkpoint %v", nextLedger, getCheckpointCounter(uint32(nextLedger))) } reader, readerErr := config.Archive.NewLedgerTransactionReaderFromLedgerCloseMeta(config.Passphrase, ledger) @@ -157,21 +162,17 @@ func searchTxByAccount(ctx context.Context, cursor int64, accountId string, conf // this deals in ledgers but adapts to the index model, which is currently keyed by checkpoint for now func getAccountNextLedgerCursor(accountId string, cursor int64, store index.Store, indexName string) (uint64, error) { - nextLedger := toid.Parse(cursor).LedgerSequence + 1 + nextLedger := uint32(toid.Parse(cursor).LedgerSequence + 1) // done for performance reasons, skip reading the index for any requested ledger cursors // only need to read the index when next cursor falls on checkpoint boundary - if !isCheckpoint(nextLedger) { + if !checkpointManager.IsCheckpoint(nextLedger) { return uint64(nextLedger), nil } - nextCheckpoint := uint32(nextLedger / 64) - queryStartingCheckpoint := nextCheckpoint - if queryStartingCheckpoint > 0 { - // the 'NextActive' index query takes a starting checkpoint, from which the index is scanned AFTER that checkpoint, non-inclusive - // so, we need to calculate one prior to our starting query - queryStartingCheckpoint = queryStartingCheckpoint - 1 - } + // the 'NextActive' index query takes a starting checkpoint, from which the index is scanned AFTER that checkpoint, non-inclusive + // use the the currrent checkpoint as the starting point since it represents up to the cursor's ledger + queryStartingCheckpoint := getCheckpointCounter(nextLedger) indexNextCheckpoint, err := store.NextActive(accountId, indexName, queryStartingCheckpoint) if err != nil { @@ -179,9 +180,10 @@ func getAccountNextLedgerCursor(accountId string, cursor int64, store index.Stor } // return the first ledger of the next checkpoint that had account activity after cursor - return uint64(indexNextCheckpoint * 64), nil + return uint64(indexNextCheckpoint * checkpointManager.GetCheckpointFrequency()), nil } -func isCheckpoint(ledger int32) bool { - return ledger%64 == 0 +func getCheckpointCounter(ledger uint32) uint32 { + return checkpointManager.GetCheckpoint(uint32(ledger)) / + checkpointManager.GetCheckpointFrequency() } diff --git a/exp/lighthorizon/services/main_test.go b/exp/lighthorizon/services/main_test.go index ae402bfcea..0d31a36078 100644 --- a/exp/lighthorizon/services/main_test.go +++ b/exp/lighthorizon/services/main_test.go @@ -13,9 +13,10 @@ import ( ) func TestItGetsTransactionsByAccount(tt *testing.T) { - // l=1586111, t=1, o=1 - // cursor = 6812294872829953, checkpoint=24782 - cursor := int64(6812294872829953) + // l=1586045, t=1, o=1 + // cursor = 6812011404988417, checkpoint=24781 + + cursor := int64(6812011404988417) ctx := context.Background() passphrase := "White New England clam chowder" archive, store := mockArchiveAndIndex(ctx, passphrase) @@ -36,9 +37,10 @@ func TestItGetsTransactionsByAccount(tt *testing.T) { } func TestItGetsTransactionsByAccountAndPageLimit(tt *testing.T) { - // l=1586111, t=1, o=1 - // cursor = 6812294872829953, checkpoint=24782 - cursor := int64(6812294872829953) + // l=1586045, t=1, o=1 + // cursor = 6812011404988417, checkpoint=24781 + + cursor := int64(6812011404988417) ctx := context.Background() passphrase := "White New England clam chowder" archive, store := mockArchiveAndIndex(ctx, passphrase) @@ -61,9 +63,10 @@ func TestItGetsTransactionsByAccountAndPageLimit(tt *testing.T) { } func TestItGetsOperationsByAccount(tt *testing.T) { - // l=1586111, t=1, o=1 - // cursor = 6812294872829953, checkpoint=24782 - cursor := int64(6812294872829953) + // l=1586045, t=1, o=1 + // cursor = 6812011404988417, checkpoint=24781 + + cursor := int64(6812011404988417) ctx := context.Background() passphrase := "White New England clam chowder" archive, store := mockArchiveAndIndex(ctx, passphrase) @@ -84,9 +87,10 @@ func TestItGetsOperationsByAccount(tt *testing.T) { } func TestItGetsOperationsByAccountAndPageLimit(tt *testing.T) { - // l=1586111, t=1, o=1 - // cursor = 6812294872829953, checkpoint=24782 - cursor := int64(6812294872829953) + // l=1586045, t=1, o=1 + // cursor = 6812011404988417, checkpoint=24781 + + cursor := int64(6812011404988417) ctx := context.Background() passphrase := "White New England clam chowder" archive, store := mockArchiveAndIndex(ctx, passphrase)