From 816592a6abc750beeb6625a4fc1a8a2385c71442 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Mon, 7 Jun 2021 14:07:28 -0700 Subject: [PATCH 01/11] Stop generating a random subdirectory --- ingest/ledgerbackend/stellar_core_runner.go | 21 ++++----------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go index 1105ee743c..3b6b783654 100644 --- a/ingest/ledgerbackend/stellar_core_runner.go +++ b/ingest/ledgerbackend/stellar_core_runner.go @@ -68,24 +68,12 @@ type stellarCoreRunner struct { log *log.Entry } -func createRandomHexString(n int) string { - hex := []rune("abcdef1234567890") - b := make([]rune, n) - for i := range b { - b[i] = hex[rand.Intn(len(hex))] - } - return string(b) -} - func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode) (*stellarCoreRunner, error) { // Use the specified directory to store Captive Core's data: // https://github.com/stellar/go/issues/3437 - // - // However, first we ALWAYS append something to the base storage path, - // because we will delete the directory entirely when Horizon stops. We also - // add a random suffix in order to ensure that there aren't naming - // conflicts. - fullStoragePath := path.Join(config.StoragePath, "captive-core-"+createRandomHexString(8)) + // but be sure to re-use rather than replace it: + // https://github.com/stellar/go/issues/3631 + fullStoragePath := path.Join(config.StoragePath, "captive-core") info, err := os.Stat(fullStoragePath) if os.IsNotExist(err) { @@ -95,8 +83,7 @@ func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode) "failed to create storage directory (%s)", fullStoragePath)) } } else if !info.IsDir() { - return nil, errors.New(fmt.Sprintf( - "%s is not a directory", fullStoragePath)) + return nil, errors.New(fmt.Sprintf("%s is not a directory", fullStoragePath)) } else if err != nil { return nil, errors.Wrap(err, fmt.Sprintf( "error accessing storage directory: %s", fullStoragePath)) From d563ec766dc65e893d2e0a0705fa86ebdf5842fa Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Mon, 7 Jun 2021 14:08:43 -0700 Subject: [PATCH 02/11] Add relevant CHANGELOG entries --- ingest/CHANGELOG.md | 2 ++ services/horizon/CHANGELOG.md | 11 ++++++++--- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/ingest/CHANGELOG.md b/ingest/CHANGELOG.md index e10a940801..099a86373e 100644 --- a/ingest/CHANGELOG.md +++ b/ingest/CHANGELOG.md @@ -5,6 +5,8 @@ All notable changes to this project will be documented in this file. This projec ## Unreleased +### New Features +* Performance improvement: Captive Core now reuses bucket files whenever it finds existing ones in the corresponding `--captive-core-storage-path` (introduced in [v2.0](#v2.0.0) rather than generating a one-time temporary sub-directory ([](https://github.com/stellar/go/pull/XXXX)). **This requires Stellar-Core version 17.1 or later.** ## v2.0.0 diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 41dbaea09e..4e5ad55536 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -3,20 +3,25 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](http://semver.org/). + ## Unreleased +### New Features * Add new command `horizon db detect-gaps`, which detects ingestion gaps in the database. The command prints out the `db reingest` commands to run in order to fill the gaps found. +* Performance improvement: Captive Core now reuses bucket files whenever it finds existing ones in the corresponding `--captive-core-storage-path` (introduced in [v2.1.0](#v2.1.0) rather than generating a one-time temporary sub-directory ([](https://github.com/stellar/go/pull/XXXX)). **This feature requires Stellar-Core version 17.1 or later.** + + ## v2.4.1 **Upgrading to this version from <= v2.1.1 will trigger a state rebuild. During this process (which can take up to 20 minutes), Horizon will not ingest new ledgers.** -### Code Changes - +### Bug Fixes * Fix bug in `horizon db reingest range` command, which would throw a duplicate entry conflict error from the DB. ([3661](https://github.com/stellar/go/pull/3661)). * Fix bug in DB metrics preventing Horizon from starting when read-only replica middleware is enabled. ([3668](https://github.com/stellar/go/pull/3668)). * Fix bug in the value of `route` in the logs for rate-limited requests (previously it was set to `undefined`). ([3658](https://github.com/stellar/go/pull/3658)). + ## v2.4.0 **Upgrading to this version from <= v2.1.1 will trigger a state rebuild. During this process (which can take up to 20 minutes), Horizon will not ingest new ledgers.** @@ -82,7 +87,7 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). ### Breaking changes -* Add a flag `--captive-core-storage-path/CAPTIVE_CORE_STORAGE_PATH` that allows users to control the storage location for Captive Core bucket data ([3479](https://github.com/stellar/go/pull/3479)). +* Add a flag `--captive-core-storage-path`/`CAPTIVE_CORE_STORAGE_PATH` that allows users to control the storage location for Captive Core bucket data ([3479](https://github.com/stellar/go/pull/3479)). - Previously, Horizon created a directory in `/tmp` to store Captive Core bucket data. Now, if the captive core storage path flag is not set, Horizon will default to using the current working directory. * Add a flag `--captive-core-log-path`/`CAPTIVE_CORE_LOG_PATH` that allows users to control the location of the logs emitted by Captive Core ([3472](https://github.com/stellar/go/pull/3472)). If you have a `LOG_FILE_PATH` entry in your Captive Core toml file remove that entry and use the horizon flag instead. * `--stellar-core-db-url` / `STELLAR_CORE_DATABASE_URL` should only be configured if Horizon ingestion is enabled otherwise Horizon will not start ([3477](https://github.com/stellar/go/pull/3477)). From 932f16c0e9a7e08450429968ef2ff63c8e1e5b04 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Mon, 7 Jun 2021 14:43:46 -0700 Subject: [PATCH 03/11] Don't remove directory on shutdown --- ingest/ledgerbackend/stellar_core_runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go index 3b6b783654..54560c283f 100644 --- a/ingest/ledgerbackend/stellar_core_runner.go +++ b/ingest/ledgerbackend/stellar_core_runner.go @@ -384,5 +384,5 @@ func (r *stellarCoreRunner) close() error { r.pipe.Reader.Close() } - return os.RemoveAll(storagePath) + return nil } From 90f29a704916d31c0b4228bc3232efbf7dd87fd1 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Wed, 9 Jun 2021 09:16:46 -0700 Subject: [PATCH 04/11] Update test case accordingly --- ingest/ledgerbackend/stellar_core_runner_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ingest/ledgerbackend/stellar_core_runner_test.go b/ingest/ledgerbackend/stellar_core_runner_test.go index 326c029ff1..36179bb976 100644 --- a/ingest/ledgerbackend/stellar_core_runner_test.go +++ b/ingest/ledgerbackend/stellar_core_runner_test.go @@ -29,7 +29,7 @@ func TestCloseBeforeStart(t *testing.T) { assert.NoError(t, runner.close()) + // Directory no longer cleaned up on shutdown (perf. bump in v2.5.0) _, err = os.Stat(tempDir) - assert.Error(t, err) - assert.True(t, os.IsNotExist(err)) + assert.NoError(t, err) } From 02c36b3c11b7cc3155e36c19dc0007a648bd228f Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Wed, 9 Jun 2021 10:42:54 -0700 Subject: [PATCH 05/11] Update Core versions to 17.1 --- .circleci/config.yml | 2 +- services/horizon/docker/docker-compose.integration-tests.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 44e0c2d1c1..7805ba0e4a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -147,7 +147,7 @@ commands: parameters: core-version: type: string - default: "17.0.0-568.ea5878e.xenial~buildtests" + default: "17.1.0-600.a5e7028.xenial" steps: - run: name: Install Stellar Core <<#parameters.core-version>> (version <>)<> diff --git a/services/horizon/docker/docker-compose.integration-tests.yml b/services/horizon/docker/docker-compose.integration-tests.yml index 0618bc90b4..a07b54991a 100644 --- a/services/horizon/docker/docker-compose.integration-tests.yml +++ b/services/horizon/docker/docker-compose.integration-tests.yml @@ -10,7 +10,7 @@ services: - "5641:5641" command: ["-p", "5641"] core: - image: ${CORE_IMAGE:-stellar/stellar-core:v17.0.0} + image: ${CORE_IMAGE:-stellar/stellar-core:17.1.0-591.151c5a6.focal} depends_on: - core-postgres restart: on-failure From 9c148ac8f6c45c70b4604ed85c7235376d526846 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Wed, 9 Jun 2021 10:44:06 -0700 Subject: [PATCH 06/11] Add random subfolder when running reingestion test --- .../horizon/internal/integration/db_test.go | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index b208bd40ea..3fe60d3bc9 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -2,6 +2,7 @@ package integration import ( "fmt" + "math/rand" "path/filepath" "strconv" "testing" @@ -18,6 +19,15 @@ import ( "github.com/stellar/go/txnbuild" ) +func createRandomHexString(n int) string { + hex := []rune("abcdef1234567890") + b := make([]rune, n) + for i := range b { + b[i] = hex[rand.Intn(len(hex))] + } + return string(b) +} + func initializeDBIntegrationTest(t *testing.T) (itest *integration.Test, reachedLedger int32) { itest = integration.NewTest(t, protocol15Config) master := itest.Master() @@ -87,6 +97,12 @@ func TestReingestDB(t *testing.T) { filepath.Dir(horizonConfig.CaptiveCoreConfigPath), "captive-core-reingest-range-integration-tests.cfg", ) + + randomSubfolder := filepath.Join( + horizonConfig.CaptiveCoreStoragePath, + "instance-"+createRandomHexString(8), + ) + horizoncmd.RootCmd.SetArgs([]string{ "--stellar-core-url", horizonConfig.StellarCoreURL, @@ -100,6 +116,8 @@ func TestReingestDB(t *testing.T) { horizonConfig.CaptiveCoreBinaryPath, "--captive-core-config-path", horizonConfig.CaptiveCoreConfigPath, + "--captive-core-storage-path", + randomSubfolder, "--enable-captive-core-ingestion=" + strconv.FormatBool(horizonConfig.EnableCaptiveCoreIngestion), "--network-passphrase", horizonConfig.NetworkPassphrase, From 46182a168543ad8a358da02f4f9f8a4b961ae19d Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Wed, 9 Jun 2021 12:12:56 -0700 Subject: [PATCH 07/11] Stop using deprecated flag for `stellar-core catchup` command --- ingest/ledgerbackend/stellar_core_runner.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go index 54560c283f..5f45395686 100644 --- a/ingest/ledgerbackend/stellar_core_runner.go +++ b/ingest/ledgerbackend/stellar_core_runner.go @@ -253,7 +253,7 @@ func (r *stellarCoreRunner) catchup(from, to uint32) error { cmd := r.createCmd( "catchup", rangeArg, "--metadata-output-stream", r.getPipeName(), - "--replay-in-memory", + "--in-memory", ) var err error From f9aaba985a033bdf5b657cabe89ef139ea0f3770 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Wed, 9 Jun 2021 12:13:30 -0700 Subject: [PATCH 08/11] Stop running Horizon server when doing reingestion test --- .../horizon/internal/integration/db_test.go | 5 +++++ .../internal/test/integration/integration.go | 17 ++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index 3fe60d3bc9..4c20fe8407 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -93,6 +93,11 @@ func TestReingestDB(t *testing.T) { toLedger = latestCheckpoint } + // We just want to test reingestion, so there's no reason for a background + // Horizon to run. Keeping it running will actually cause the Captive Core + // subprocesses to conflict. + itest.StopHorizon() + horizonConfig.CaptiveCoreConfigPath = filepath.Join( filepath.Dir(horizonConfig.CaptiveCoreConfigPath), "captive-core-reingest-range-integration-tests.cfg", diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go index b50f470067..8ce35386f4 100644 --- a/services/horizon/internal/test/integration/integration.go +++ b/services/horizon/internal/test/integration/integration.go @@ -135,11 +135,7 @@ func NewTest(t *testing.T, config Config) *Test { } func (i *Test) RestartHorizon() { - i.app.Close() - - // wait for horizon to shut down completely - <-i.appStopped - + i.StopHorizon() i.startHorizon( i.horizonConfig.CaptiveCoreBinaryPath, i.horizonConfig.CaptiveCoreConfigPath, @@ -332,6 +328,17 @@ func (i *Test) Horizon() *horizon.App { return i.app } +// StopHorizon shuts down the running Horizon process +func (i *Test) StopHorizon() { + i.app.CloseDB() + i.app.Close() + + // Wait for Horizon to shut down completely. + <-i.appStopped + + i.app = nil +} + // AdminPort returns Horizon admin port. func (i *Test) AdminPort() int { return adminPort From 376ce616e58265c70e096eb1a933a234cf9e0752 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Wed, 9 Jun 2021 12:22:41 -0700 Subject: [PATCH 09/11] Update CHANGELOG entry --- ingest/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ingest/CHANGELOG.md b/ingest/CHANGELOG.md index 099a86373e..27fa2b6e77 100644 --- a/ingest/CHANGELOG.md +++ b/ingest/CHANGELOG.md @@ -6,7 +6,8 @@ All notable changes to this project will be documented in this file. This projec ## Unreleased ### New Features -* Performance improvement: Captive Core now reuses bucket files whenever it finds existing ones in the corresponding `--captive-core-storage-path` (introduced in [v2.0](#v2.0.0) rather than generating a one-time temporary sub-directory ([](https://github.com/stellar/go/pull/XXXX)). **This requires Stellar-Core version 17.1 or later.** +* **Performance improvement**: the Captive Core backend now reuses bucket files whenever it finds existing ones in the corresponding `--captive-core-storage-path` (introduced in [v2.0](#v2.0.0)) rather than generating a one-time temporary sub-directory ([#3670](https://github.com/stellar/go/pull/3670)). Note that taking advantage of this feature requires [Stellar-Core v17.1.0](https://github.com/stellar/stellar-core/releases/tag/v17.1.0) or later. + ## v2.0.0 From a57d4113e05b947f3bf1557fc0c20d0351cf5143 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Wed, 9 Jun 2021 12:32:22 -0700 Subject: [PATCH 10/11] The random directory is not needed for the test --- .../horizon/internal/integration/db_test.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index 4c20fe8407..f690cd3f00 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -2,7 +2,6 @@ package integration import ( "fmt" - "math/rand" "path/filepath" "strconv" "testing" @@ -19,15 +18,6 @@ import ( "github.com/stellar/go/txnbuild" ) -func createRandomHexString(n int) string { - hex := []rune("abcdef1234567890") - b := make([]rune, n) - for i := range b { - b[i] = hex[rand.Intn(len(hex))] - } - return string(b) -} - func initializeDBIntegrationTest(t *testing.T) (itest *integration.Test, reachedLedger int32) { itest = integration.NewTest(t, protocol15Config) master := itest.Master() @@ -103,11 +93,6 @@ func TestReingestDB(t *testing.T) { "captive-core-reingest-range-integration-tests.cfg", ) - randomSubfolder := filepath.Join( - horizonConfig.CaptiveCoreStoragePath, - "instance-"+createRandomHexString(8), - ) - horizoncmd.RootCmd.SetArgs([]string{ "--stellar-core-url", horizonConfig.StellarCoreURL, @@ -121,8 +106,6 @@ func TestReingestDB(t *testing.T) { horizonConfig.CaptiveCoreBinaryPath, "--captive-core-config-path", horizonConfig.CaptiveCoreConfigPath, - "--captive-core-storage-path", - randomSubfolder, "--enable-captive-core-ingestion=" + strconv.FormatBool(horizonConfig.EnableCaptiveCoreIngestion), "--network-passphrase", horizonConfig.NetworkPassphrase, From 6628e51c5f0d666a19a3a32c8097a01347b30a18 Mon Sep 17 00:00:00 2001 From: George Kudrayvtsev Date: Thu, 10 Jun 2021 12:43:08 -0700 Subject: [PATCH 11/11] Fix typo introduced when merging in master --- .circleci/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 4513e39662..1ea616bfac 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -152,7 +152,7 @@ commands: - run: name: Install Stellar Core <<#parameters.core-version>> (version <>)<> command: | - sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add - + sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add - sudo bash -c 'echo "deb https://apt.stellar.org xenial unstable" > /etc/apt/sources.list.d/SDF-unstable.list' sudo apt-get update && sudo apt-get install -y stellar-core<<#parameters.core-version>>=<><> echo "using stellar core version $(stellar-core version)"