From e88d772669f191a12e8d087a36f90188c1ac6669 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Tue, 27 Sep 2022 15:49:36 -0700 Subject: [PATCH 1/5] Add ability to preload cache in parallel after launching webserver. --- .../build/k8s/lighthorizon_web.yml | 6 ++-- exp/lighthorizon/build/web/Dockerfile | 5 +++- exp/lighthorizon/main.go | 28 +++++++++++++++++++ exp/lighthorizon/services/main.go | 7 ++++- exp/lighthorizon/tools/cache.go | 16 +++++------ exp/lighthorizon/tools/index.go | 2 +- 6 files changed, 51 insertions(+), 13 deletions(-) diff --git a/exp/lighthorizon/build/k8s/lighthorizon_web.yml b/exp/lighthorizon/build/k8s/lighthorizon_web.yml index 75a5c6436d..76f7e0b93b 100644 --- a/exp/lighthorizon/build/k8s/lighthorizon_web.yml +++ b/exp/lighthorizon/build/k8s/lighthorizon_web.yml @@ -7,11 +7,13 @@ metadata: app: lighthorizon-pubnet-web name: lighthorizon-pubnet-web-env data: - TXMETA_SOURCE: "s3://horizon-ledgermeta-prodnet-test" - INDEXES_SOURCE: "s3://horizon-index-prodnet-test" + TXMETA_SOURCE: "s3://horizon-indices-pubnet" + INDEXES_SOURCE: "s3://horizon-ledgermeta-pubnet" NETWORK_PASSPHRASE: "Public Global Stellar Network ; September 2015" MAX_PARALLEL_DOWNLOADS: 16 CACHE_PATH: "/ledgercache" + CACHE_PRELOAD_START_LEDGER: 0 + CACHE_PRELOAD_COUNT: 11400 --- apiVersion: v1 kind: Secret diff --git a/exp/lighthorizon/build/web/Dockerfile b/exp/lighthorizon/build/web/Dockerfile index 15cdb175f6..e61143c00c 100644 --- a/exp/lighthorizon/build/web/Dockerfile +++ b/exp/lighthorizon/build/web/Dockerfile @@ -16,6 +16,9 @@ COPY --from=builder /go/bin/lighthorizon ./ ENTRYPOINT ./lighthorizon serve \ --network-passphrase "$NETWORK_PASSPHRASE" \ - --ledger-cache "$CACHE_PATH" \ --parallel-downloads "$MAX_PARALLEL_DOWNLOADS" \ + --ledger-cache "$CACHE_PATH" \ + --ledger-cache-preload "$CACHE_PRELOAD_COUNT" \ + --ledger-cache-preload-start "$CACHE_PRELOAD_START_LEDGER" \ + --log-level debug \ "$TXMETA_SOURCE" "$INDEXES_SOURCE" \ diff --git a/exp/lighthorizon/main.go b/exp/lighthorizon/main.go index 3427f4a13c..952f766cc2 100644 --- a/exp/lighthorizon/main.go +++ b/exp/lighthorizon/main.go @@ -1,6 +1,7 @@ package main import ( + "context" "net/http" "github.com/go-chi/chi" @@ -96,6 +97,27 @@ break down accounts by active ledgers.`, return } + latestLedger, err := ingester.GetLatestLedgerSequence(context.Background()) + if err != nil { + log.Fatalf("Failed to retrieve latest ledger from %s: %v", sourceUrl, err) + return + } + log.Infof("The latest ledger stored at %s is %d.", sourceUrl, latestLedger) + + cachePreloadCount, _ := cmd.Flags().GetUint32("ledger-cache-preload") + cachePreloadStart, _ := cmd.Flags().GetUint32("ledger-cache-preload-start") + if cacheDir != "" && cachePreloadCount > 0 { + startLedger := latestLedger - cachePreloadCount + if cachePreloadStart > 0 { + startLedger = cachePreloadStart + } + + go func() { + tools.BuildCache(sourceUrl, cacheDir, + startLedger, cachePreloadCount, false) + }() + } + Config := services.Config{ Ingester: ingester, Passphrase: networkPassphrase, @@ -118,6 +140,8 @@ break down accounts by active ledgers.`, Version: HorizonLiteVersion, LedgerSource: sourceUrl, IndexSource: indexStoreUrl, + + LatestLedger: latestLedger, })) log.Fatal(http.ListenAndServe(":8080", router)) @@ -131,6 +155,10 @@ break down accounts by active ledgers.`, "if left empty, uses a temporary directory") serve.Flags().Uint("ledger-cache-size", defaultCacheSize, "number of ledgers to store in the cache") + serve.Flags().Uint32("ledger-cache-preload", 0, + "should the cache come preloaded with the latest ledgers?") + serve.Flags().Uint32("ledger-cache-preload-start", 0, + "the preload should start at ledger ") serve.Flags().Uint("parallel-downloads", 1, "how many workers should download ledgers in parallel?") diff --git a/exp/lighthorizon/services/main.go b/exp/lighthorizon/services/main.go index 30935c725e..d391fc8baf 100644 --- a/exp/lighthorizon/services/main.go +++ b/exp/lighthorizon/services/main.go @@ -124,7 +124,7 @@ func searchAccountTransactions(ctx context.Context, Low: nextLedger, High: checkpointMgr.NextCheckpoint(nextLedger + 1), } - log.Infof("prepare range %d, %d", r.Low, r.High) + log.Infof("Preparing ledger range [%d, %d]", r.Low, r.High) if innerErr := config.Ingester.PrepareRange(ctx, r); innerErr != nil { log.Errorf("failed to prepare ledger range [%d, %d]: %v", r.Low, r.High, innerErr) @@ -133,6 +133,11 @@ func searchAccountTransactions(ctx context.Context, start := time.Now() ledger, innerErr := config.Ingester.GetLedger(ctx, nextLedger) + + // TODO: We should have helpful error messages when innerErr points to a + // 404 for that particular ledger, since that situation shouldn't happen + // under normal operations, but rather indicates a problem with the + // backing archive. if innerErr != nil { return errors.Wrapf(innerErr, "failed to retrieve ledger %d from archive", nextLedger) diff --git a/exp/lighthorizon/tools/cache.go b/exp/lighthorizon/tools/cache.go index 8f06081f85..0290fcb164 100644 --- a/exp/lighthorizon/tools/cache.go +++ b/exp/lighthorizon/tools/cache.go @@ -103,9 +103,9 @@ purge /tmp/example 1000 1005 # purge a ledger range`, return cmd.Usage() } - count, err := cmd.Flags().GetUint("count") + count, err := cmd.Flags().GetUint32("count") if err != nil || count <= 0 { - cmd.Println("--count should be a positive integer") + cmd.Println("--count should be a positive 32-bit integer") return cmd.Usage() } repair, _ := cmd.Flags().GetBool("repair") @@ -115,7 +115,7 @@ purge /tmp/example 1000 1005 # purge a ledger range`, build.Flags().Bool("repair", false, "attempt to purge the cache and retry ledgers that error") build.Flags().Uint32("start", 0, "first ledger to cache (required)") - build.Flags().Uint("count", defaultCacheCount, "number of ledgers to cache") + build.Flags().Uint32("count", defaultCacheCount, "number of ledgers to cache") cmd.AddCommand(build, purge, show) if parent == nil { @@ -126,7 +126,7 @@ purge /tmp/example 1000 1005 # purge a ledger range`, return parent } -func BuildCache(ledgerSource, cacheDir string, start uint32, count uint, repair bool) error { +func BuildCache(ledgerSource, cacheDir string, start uint32, count uint32, repair bool) error { fullStart := time.Now() L := log.DefaultLogger L.SetLevel(log.InfoLevel) @@ -136,7 +136,7 @@ func BuildCache(ledgerSource, cacheDir string, start uint32, count uint, repair store, err := storage.ConnectBackend(ledgerSource, storage.ConnectOptions{ Context: ctx, Wrap: func(store storage.Storage) (storage.Storage, error) { - return storage.MakeOnDiskCache(store, cacheDir, count) + return storage.MakeOnDiskCache(store, cacheDir, uint(count)) }, }) if err != nil { @@ -151,10 +151,10 @@ func BuildCache(ledgerSource, cacheDir string, start uint32, count uint, repair source := metaarchive.NewMetaArchive(store) log.Infof("Filling local cache of ledgers at %s...", cacheDir) log.Infof("Ledger range: [%d, %d] (%d ledgers)", - start, uint(start)+count-1, count) + start, start+count-1, count) successful := uint(0) - for i := uint(0); i < count; i++ { + for i := uint32(0); i < count; i++ { ledgerSeq := start + uint32(i) // do "best effort" caching, skipping if too slow @@ -186,7 +186,7 @@ func BuildCache(ledgerSource, cacheDir string, start uint32, count uint, repair Warnf("Downloading ledger %d took a while.", ledgerSeq) } - log = log.WithField("failures", 1+i-successful) + log = log.WithField("failures", 1+uint(i)-successful) if successful%97 == 0 { log.Infof("Cached %d/%d ledgers (%0.1f%%)", successful, count, 100*float64(successful)/float64(count)) diff --git a/exp/lighthorizon/tools/index.go b/exp/lighthorizon/tools/index.go index 0f2305035a..e37a7eb38a 100644 --- a/exp/lighthorizon/tools/index.go +++ b/exp/lighthorizon/tools/index.go @@ -124,7 +124,7 @@ view s3:///indices GAXLQGKIUAIIUHAX4GJO3J7HFGLBCNF6ZCZSTLJE7EKO5IUHGLQLMXZO view file:///tmp/indices --limit=0 GAXLQGKIUAIIUHAX4GJO3J7HFGLBCNF6ZCZSTLJE7EKO5IUHGLQLMXZO view gcs://indices --limit=10 GAXLQGKIUAIIUHAX4GJO3J7HFGLBCNF6ZCZSTLJE7EKO5IUHGLQLMXZO,GBUUWQDVEEXBJCUF5UL24YGXKJIP5EMM7KFWIAR33KQRJR34GN6HEDPV,GBYETUYNBK2ZO5MSYBJKSLDEA2ZHIXLCFL3MMWU6RHFVAUBKEWQORYKS`, RunE: func(cmd *cobra.Command, args []string) error { - if len(args) > 2 { + if len(args) < 1 || len(args) > 2 { return cmd.Usage() } From 9ed669bf12c8ce4c66e43427553c7c73928ab739 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Tue, 27 Sep 2022 15:59:03 -0700 Subject: [PATCH 2/5] Default to 1 day of ledgers @ 6s each --- exp/lighthorizon/build/k8s/lighthorizon_web.yml | 2 +- exp/lighthorizon/build/web/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/exp/lighthorizon/build/k8s/lighthorizon_web.yml b/exp/lighthorizon/build/k8s/lighthorizon_web.yml index 76f7e0b93b..b680e7fb2c 100644 --- a/exp/lighthorizon/build/k8s/lighthorizon_web.yml +++ b/exp/lighthorizon/build/k8s/lighthorizon_web.yml @@ -13,7 +13,7 @@ data: MAX_PARALLEL_DOWNLOADS: 16 CACHE_PATH: "/ledgercache" CACHE_PRELOAD_START_LEDGER: 0 - CACHE_PRELOAD_COUNT: 11400 + CACHE_PRELOAD_COUNT: 14400 --- apiVersion: v1 kind: Secret diff --git a/exp/lighthorizon/build/web/Dockerfile b/exp/lighthorizon/build/web/Dockerfile index e61143c00c..78203d5c55 100644 --- a/exp/lighthorizon/build/web/Dockerfile +++ b/exp/lighthorizon/build/web/Dockerfile @@ -21,4 +21,4 @@ ENTRYPOINT ./lighthorizon serve \ --ledger-cache-preload "$CACHE_PRELOAD_COUNT" \ --ledger-cache-preload-start "$CACHE_PRELOAD_START_LEDGER" \ --log-level debug \ - "$TXMETA_SOURCE" "$INDEXES_SOURCE" \ + "$TXMETA_SOURCE" "$INDEXES_SOURCE" From 6c29639986fbf5dea7c6f39c33751cef57c1e00d Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Tue, 11 Oct 2022 11:58:32 -0700 Subject: [PATCH 3/5] Add logging instead of silent failure --- exp/lighthorizon/main.go | 26 +++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/exp/lighthorizon/main.go b/exp/lighthorizon/main.go index 952f766cc2..726ce4fcbf 100644 --- a/exp/lighthorizon/main.go +++ b/exp/lighthorizon/main.go @@ -106,16 +106,24 @@ break down accounts by active ledgers.`, cachePreloadCount, _ := cmd.Flags().GetUint32("ledger-cache-preload") cachePreloadStart, _ := cmd.Flags().GetUint32("ledger-cache-preload-start") - if cacheDir != "" && cachePreloadCount > 0 { - startLedger := latestLedger - cachePreloadCount - if cachePreloadStart > 0 { - startLedger = cachePreloadStart + if cachePreloadCount > 0 { + if cacheDir == "" { + log.Warnf("--ledger-cache-preload=%d specified but no "+ + "--ledger-cache directory provided, ignoring...", + cachePreloadCount) + } else { + startLedger := latestLedger - cachePreloadCount + if cachePreloadStart > 0 { + startLedger = cachePreloadStart + } + + log.Infof("Preloading cache at %s with %d ledgers, starting at ledger %d.", + cacheDir, startLedger, cachePreloadCount) + go func() { + tools.BuildCache(sourceUrl, cacheDir, + startLedger, cachePreloadCount, false) + }() } - - go func() { - tools.BuildCache(sourceUrl, cacheDir, - startLedger, cachePreloadCount, false) - }() } Config := services.Config{ From ea199e341e9aa4fd382b611a84d60d70469e7985 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Tue, 11 Oct 2022 11:59:27 -0700 Subject: [PATCH 4/5] Fatal instead of warning --- exp/lighthorizon/main.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/exp/lighthorizon/main.go b/exp/lighthorizon/main.go index 726ce4fcbf..5a616ad88c 100644 --- a/exp/lighthorizon/main.go +++ b/exp/lighthorizon/main.go @@ -108,9 +108,10 @@ break down accounts by active ledgers.`, cachePreloadStart, _ := cmd.Flags().GetUint32("ledger-cache-preload-start") if cachePreloadCount > 0 { if cacheDir == "" { - log.Warnf("--ledger-cache-preload=%d specified but no "+ + log.Fatalf("--ledger-cache-preload=%d specified but no "+ "--ledger-cache directory provided, ignoring...", cachePreloadCount) + return } else { startLedger := latestLedger - cachePreloadCount if cachePreloadStart > 0 { From 267757e69bf1cd33761696eb87595a183272fce7 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Tue, 11 Oct 2022 14:32:54 -0700 Subject: [PATCH 5/5] Check that startLedger is valid --- exp/lighthorizon/main.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/exp/lighthorizon/main.go b/exp/lighthorizon/main.go index 5a616ad88c..f7c502d465 100644 --- a/exp/lighthorizon/main.go +++ b/exp/lighthorizon/main.go @@ -109,20 +109,25 @@ break down accounts by active ledgers.`, if cachePreloadCount > 0 { if cacheDir == "" { log.Fatalf("--ledger-cache-preload=%d specified but no "+ - "--ledger-cache directory provided, ignoring...", + "--ledger-cache directory provided.", cachePreloadCount) return } else { - startLedger := latestLedger - cachePreloadCount + startLedger := int(latestLedger) - int(cachePreloadCount) if cachePreloadStart > 0 { - startLedger = cachePreloadStart + startLedger = int(cachePreloadStart) + } + if startLedger <= 0 { + log.Warnf("Starting ledger invalid (%d), defaulting to 2.", + startLedger) + startLedger = 2 } log.Infof("Preloading cache at %s with %d ledgers, starting at ledger %d.", cacheDir, startLedger, cachePreloadCount) go func() { tools.BuildCache(sourceUrl, cacheDir, - startLedger, cachePreloadCount, false) + uint32(startLedger), cachePreloadCount, false) }() } }