services/horizon: Propagate --captive-core-reuse-storage-dir to sub…

…systems. (#3750)

* Rename to storage*Path* instead of *Dir* for consistency with
  `--captive-core-storage-path`

ingest/ledgerbackend/captive_core_backend.go

@@ -123,10 +123,10 @@ type CaptiveCoreConfig struct {
 	// stored. We always append /captive-core to this directory, since we clean
 	// it up entirely on shutdown.
 	StoragePath string
-	// ReuseStorageDir determines if the storage dir in StoragePath should
+	// ReuseStoragePath determines if the storage dir in StoragePath should
 	// be reused between Stellar-Core executions. Defaults to false because of
 	// Stellar-Core 17.1.0 issue.
-	ReuseStorageDir bool
+	ReuseStoragePath bool
 }
 
 // NewCaptive returns a new CaptiveStellarCore instance.

ingest/ledgerbackend/stellar_core_runner.go

@@ -65,9 +65,9 @@ type stellarCoreRunner struct {
 	processExited    bool
 	processExitError error
 
-	storagePath     string
-	reuseStorageDir bool
-	nonce           string
+	storagePath      string
+	reuseStoragePath bool
+	nonce            string
 
 	log *log.Entry
 }
@@ -83,7 +83,7 @@ func createRandomHexString(n int) string {
 
 func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode) (*stellarCoreRunner, error) {
 	var fullStoragePath string
-	if runtime.GOOS == "windows" || mode == stellarCoreRunnerModeOffline || !config.ReuseStorageDir {
+	if runtime.GOOS == "windows" || mode == stellarCoreRunnerModeOffline || !config.ReuseStoragePath {
 		// On Windows, first we ALWAYS append something to the base storage path,
 		// because we will delete the directory entirely when Horizon stops. We also
 		// add a random suffix in order to ensure that there aren't naming
@@ -119,12 +119,12 @@ func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode)
 	ctx, cancel := context.WithCancel(config.Context)
 
 	runner := &stellarCoreRunner{
-		executablePath:  config.BinaryPath,
-		ctx:             ctx,
-		cancel:          cancel,
-		storagePath:     fullStoragePath,
-		reuseStorageDir: config.ReuseStorageDir,
-		mode:            mode,
+		executablePath:   config.BinaryPath,
+		ctx:              ctx,
+		cancel:           cancel,
+		storagePath:      fullStoragePath,
+		reuseStoragePath: config.ReuseStoragePath,
+		mode:             mode,
 		nonce: fmt.Sprintf(
 			"captive-stellar-core-%x",
 			rand.New(rand.NewSource(time.Now().UnixNano())).Uint64(),
@@ -483,7 +483,7 @@ func (r *stellarCoreRunner) close() error {
 	if runtime.GOOS == "windows" ||
 		(r.processExitError != nil && r.processExitError != context.Canceled) ||
 		r.mode == stellarCoreRunnerModeOffline ||
-		!r.reuseStorageDir {
+		!r.reuseStoragePath {
 		// It's impossible to send SIGINT on Windows so buckets can become
 		// corrupted. If we can't reuse it, then remove it.
 		// We also remove the storage path if there was an error terminating the

ingest/ledgerbackend/stellar_core_runner_test.go

@@ -47,7 +47,7 @@ func TestCloseBeforeStartOnline(t *testing.T) {
 		Log:                log.New(),
 		Context:            context.Background(),
 		Toml:               captiveCoreToml,
-		ReuseStorageDir:    true,
+		ReuseStoragePath:   true,
 	}, stellarCoreRunnerModeOnline)
 	assert.NoError(t, err)
 
@@ -74,7 +74,7 @@ func TestCloseBeforeStartOnlineReuseFlagFalse(t *testing.T) {
 		Log:                log.New(),
 		Context:            context.Background(),
 		Toml:               captiveCoreToml,
-		ReuseStorageDir:    false,
+		ReuseStoragePath:   false,
 	}, stellarCoreRunnerModeOnline)
 	assert.NoError(t, err)
 

services/horizon/CHANGELOG.md

@@ -6,10 +6,16 @@ file. This project adheres to [Semantic Versioning](http://semver.org/).
 
 ## Unreleased
 
-* Require `COUNT` param when running `horizon db migrate down COUNT` to prevent accidentally running all downwards migrations. Add `horizon db migrate status` command. ([#3737](https://github.com/stellar/go/pull/3737))
+### New features
+* Add `horizon db migrate status` command ([#3737](https://github.com/stellar/go/pull/3737)).
+* Add a feature flag `--captive-core-reuse-storage-path`/`CAPTIVE_CORE_REUSE_STORAGE_PATH` that will reuse Captive Core's storage path for bucket files when applicable for better performance ([3750](https://github.com/stellar/go/pull/3750)).
+
+### Improvements
 * Fix a bug in `fee_account_muxed` and `fee_account_muxed_id` fields (the fields were incorrectly populated with the source account details). ([3735](https://github.com/stellar/go/pull/3735))
+* Require `COUNT` param when running `horizon db migrate down COUNT` to prevent accidentally running all downwards migrations ([#3737](https://github.com/stellar/go/pull/3737)).
 * Validate ledger range when calling `horizon db reingest range` so that we respond with an error when attempting to ingest ledgers which are not available in the history archives. ([3738](https://github.com/stellar/go/pull/3738))
 
+
 ## v2.5.2
 
 **Upgrading to this version from <= v2.1.1 will trigger a state rebuild. During this process (which can take up to 20 minutes), Horizon will not ingest new ledgers.**

services/horizon/internal/config.go

@@ -19,14 +19,14 @@ type Config struct {
 	Port               uint
 	AdminPort          uint
 
-	EnableCaptiveCoreIngestion bool
-	CaptiveCoreBinaryPath      string
-	RemoteCaptiveCoreURL       string
-	CaptiveCoreConfigPath      string
-	CaptiveCoreTomlParams      ledgerbackend.CaptiveCoreTomlParams
-	CaptiveCoreToml            *ledgerbackend.CaptiveCoreToml
-	CaptiveCoreStoragePath     string
-	CaptiveCoreReuseStorageDir bool
+	EnableCaptiveCoreIngestion  bool
+	CaptiveCoreBinaryPath       string
+	RemoteCaptiveCoreURL        string
+	CaptiveCoreConfigPath       string
+	CaptiveCoreTomlParams       ledgerbackend.CaptiveCoreTomlParams
+	CaptiveCoreToml             *ledgerbackend.CaptiveCoreToml
+	CaptiveCoreStoragePath      string
+	CaptiveCoreReuseStoragePath bool
 
 	StellarCoreDatabaseURL string
 	StellarCoreURL         string

services/horizon/internal/flags.go

@@ -192,12 +192,12 @@ func Flags() (*Config, support.ConfigOptions) {
 			ConfigKey: &config.CaptiveCoreStoragePath,
 		},
 		&support.ConfigOption{
-			Name:        "captive-core-reuse-storage-dir",
+			Name:        "captive-core-reuse-storage-path",
 			OptType:     types.Bool,
 			Required:    false,
 			FlagDefault: false,
-			Usage:       "determines if storage-dir should be reused, disabled by default because of Stellar-Core 17.1.0 issue",
-			ConfigKey:   &config.CaptiveCoreReuseStorageDir,
+			Usage:       "determines if storage-path should be reused, disabled by default because of Stellar-Core 17.1.0 issue",
+			ConfigKey:   &config.CaptiveCoreReuseStoragePath,
 		},
 		&support.ConfigOption{
 			Name:           "captive-core-peer-port",

services/horizon/internal/ingest/main.go

@@ -66,16 +66,16 @@ const (
 var log = logpkg.DefaultLogger.WithField("service", "ingest")
 
 type Config struct {
-	CoreSession                db.SessionInterface
-	StellarCoreURL             string
-	StellarCoreCursor          string
-	EnableCaptiveCore          bool
-	CaptiveCoreBinaryPath      string
-	CaptiveCoreStoragePath     string
-	CaptiveCoreReuseStorageDir bool
-	CaptiveCoreToml            *ledgerbackend.CaptiveCoreToml
-	RemoteCaptiveCoreURL       string
-	NetworkPassphrase          string
+	CoreSession                 db.SessionInterface
+	StellarCoreURL              string
+	StellarCoreCursor           string
+	EnableCaptiveCore           bool
+	CaptiveCoreBinaryPath       string
+	CaptiveCoreStoragePath      string
+	CaptiveCoreReuseStoragePath bool
+	CaptiveCoreToml             *ledgerbackend.CaptiveCoreToml
+	RemoteCaptiveCoreURL        string
+	NetworkPassphrase           string
 
 	HistorySession           db.SessionInterface
 	HistoryArchiveURL        string
@@ -209,6 +209,7 @@ func NewSystem(config Config) (System, error) {
 				ledgerbackend.CaptiveCoreConfig{
 					BinaryPath:          config.CaptiveCoreBinaryPath,
 					StoragePath:         config.CaptiveCoreStoragePath,
+					ReuseStoragePath:    config.CaptiveCoreReuseStoragePath,
 					Toml:                config.CaptiveCoreToml,
 					NetworkPassphrase:   config.NetworkPassphrase,
 					HistoryArchiveURLs:  []string{config.HistoryArchiveURL},

services/horizon/internal/init.go

@@ -87,17 +87,17 @@ func initIngester(app *App) {
 		// TODO:
 		// Use the first archive for now. We don't have a mechanism to
 		// use multiple archives at the same time currently.
-		HistoryArchiveURL:          app.config.HistoryArchiveURLs[0],
-		CheckpointFrequency:        app.config.CheckpointFrequency,
-		StellarCoreURL:             app.config.StellarCoreURL,
-		StellarCoreCursor:          app.config.CursorName,
-		CaptiveCoreBinaryPath:      app.config.CaptiveCoreBinaryPath,
-		CaptiveCoreStoragePath:     app.config.CaptiveCoreStoragePath,
-		CaptiveCoreReuseStorageDir: app.config.CaptiveCoreReuseStorageDir,
-		CaptiveCoreToml:            app.config.CaptiveCoreToml,
-		RemoteCaptiveCoreURL:       app.config.RemoteCaptiveCoreURL,
-		EnableCaptiveCore:          app.config.EnableCaptiveCoreIngestion,
-		DisableStateVerification:   app.config.IngestDisableStateVerification,
+		HistoryArchiveURL:           app.config.HistoryArchiveURLs[0],
+		CheckpointFrequency:         app.config.CheckpointFrequency,
+		StellarCoreURL:              app.config.StellarCoreURL,
+		StellarCoreCursor:           app.config.CursorName,
+		CaptiveCoreBinaryPath:       app.config.CaptiveCoreBinaryPath,
+		CaptiveCoreStoragePath:      app.config.CaptiveCoreStoragePath,
+		CaptiveCoreReuseStoragePath: app.config.CaptiveCoreReuseStoragePath,
+		CaptiveCoreToml:             app.config.CaptiveCoreToml,
+		RemoteCaptiveCoreURL:        app.config.RemoteCaptiveCoreURL,
+		EnableCaptiveCore:           app.config.EnableCaptiveCoreIngestion,
+		DisableStateVerification:    app.config.IngestDisableStateVerification,
 	})
 
 	if err != nil {