services/horizon: Add flag to customize storage directory for Captive…

… Core data (#3479)

This introduces the --captive-core-storage-path parameter, which allows users to
control where Horizon stores Captive Core's data. This includes the
auto-generated configuration, the log, and most-importantly Core's bucket data.

It defaults to the current directory rather than /tmp in order to raise
visibility of the large amount of data Core stores in that directory.
Internally, /captive-core-[random hex] is appended to it since we wipe the
directory on shutdown.

ingest/ledgerbackend/captive_core_backend.go

@@ -135,6 +135,11 @@ type CaptiveCoreConfig struct {
 	// the CaptiveStellarCore instance will not be able to stream ledgers from Stellar Core or spawn new
 	// instances of Stellar Core. If Context is omitted CaptiveStellarCore will default to using context.Background.
 	Context context.Context
+	// StoragePath is the (optional) base path passed along to Core's
+	// BUCKET_DIR_PATH which specifies where various bucket data should be
+	// stored. We always append /captive-core to this directory, since we clean
+	// it up entirely on shutdown.
+	StoragePath string
 }
 
 // NewCaptive returns a new CaptiveStellarCore instance.

ingest/ledgerbackend/stellar_core_runner.go

@@ -9,6 +9,7 @@ import (
 	"math/rand"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"regexp"
 	"strings"
@@ -68,17 +69,44 @@ type stellarCoreRunner struct {
 	processExited    bool
 	processExitError error
 
-	tempDir string
-	nonce   string
+	storagePath string
+	nonce       string
 
 	log *log.Entry
 }
 
+func createRandomHexString(n int) string {
+	hex := []rune("abcdef1234567890")
+	b := make([]rune, n)
+	for i := range b {
+		b[i] = hex[rand.Intn(len(hex))]
+	}
+	return string(b)
+}
+
 func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode) (*stellarCoreRunner, error) {
-	// Create temp dir
-	tempDir, err := ioutil.TempDir("", "captive-stellar-core")
-	if err != nil {
-		return nil, errors.Wrap(err, "error creating subprocess tmpdir")
+	// Use the specified directory to store Captive Core's data:
+	//    https://github.com/stellar/go/issues/3437
+	//
+	// However, first we ALWAYS append something to the base storage path,
+	// because we will delete the directory entirely when Horizon stops. We also
+	// add a random suffix in order to ensure that there aren't naming
+	// conflicts.
+	fullStoragePath := path.Join(config.StoragePath, "captive-core-"+createRandomHexString(8))
+
+	info, err := os.Stat(fullStoragePath)
+	if os.IsNotExist(err) {
+		innerErr := os.MkdirAll(fullStoragePath, os.FileMode(int(0755))) // rwx|rx|rx
+		if innerErr != nil {
+			return nil, errors.Wrap(innerErr, fmt.Sprintf(
+				"failed to create storage directory (%s)", fullStoragePath))
+		}
+	} else if !info.IsDir() {
+		return nil, errors.New(fmt.Sprintf(
+			"%s is not a directory", fullStoragePath))
+	} else if err != nil {
+		return nil, errors.Wrap(err, fmt.Sprintf(
+			"error accessing storage directory: %s", fullStoragePath))
 	}
 
 	ctx, cancel := context.WithCancel(config.Context)
@@ -94,7 +122,7 @@ func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode)
 		mode:              mode,
 		ctx:               ctx,
 		cancel:            cancel,
-		tempDir:           tempDir,
+		storagePath:       fullStoragePath,
 		nonce: fmt.Sprintf(
 			"captive-stellar-core-%x",
 			rand.New(rand.NewSource(time.Now().UnixNano())).Uint64(),
@@ -113,12 +141,14 @@ func (r *stellarCoreRunner) generateConfig() (string, error) {
 	if r.mode == stellarCoreRunnerModeOnline && r.configAppendPath == "" {
 		return "", errors.New("stellar-core append config file path cannot be empty in online mode")
 	}
+
 	lines := []string{
 		"# Generated file -- do not edit",
 		"NODE_IS_VALIDATOR=false",
 		"DISABLE_XDR_FSYNC=true",
 		fmt.Sprintf(`NETWORK_PASSPHRASE="%s"`, r.networkPassphrase),
-		fmt.Sprintf(`BUCKET_DIR_PATH="%s"`, filepath.Join(r.tempDir, "buckets")),
+		// Note: We don't pass BUCKET_DIR_PATH here because it's created
+		// *relative* to the storage path (i.e. where the config lives).
 		fmt.Sprintf(`HTTP_PORT=%d`, r.httpPort),
 		fmt.Sprintf(`LOG_FILE_PATH="%s"`, r.logPath),
 	}
@@ -163,7 +193,20 @@ func (r *stellarCoreRunner) generateConfig() (string, error) {
 }
 
 func (r *stellarCoreRunner) getConfFileName() string {
-	return filepath.Join(r.tempDir, "stellar-core.conf")
+	joinedPath := filepath.Join(r.storagePath, "stellar-core.conf")
+
+	// Given that `storagePath` can be anything, we need the full, absolute path
+	// here so that everything Core needs is created under the storagePath
+	// subdirectory.
+	//
+	// If the path *can't* be absolutely resolved (bizarre), we can still try
+	// recovering by using the path the user specified directly.
+	path, err := filepath.Abs(joinedPath)
+	if err != nil {
+		r.log.Warnf("Failed to resolve %s as an absolute path: %s", joinedPath, err)
+		return joinedPath
+	}
+	return path
 }
 
 func (r *stellarCoreRunner) getLogLineWriter() io.Writer {
@@ -226,7 +269,7 @@ func (r *stellarCoreRunner) writeConf() error {
 func (r *stellarCoreRunner) createCmd(params ...string) *exec.Cmd {
 	allParams := append([]string{"--conf", r.getConfFileName()}, params...)
 	cmd := exec.CommandContext(r.ctx, r.executablePath, allParams...)
-	cmd.Dir = r.tempDir
+	cmd.Dir = r.storagePath
 	cmd.Stdout = r.getLogLineWriter()
 	cmd.Stderr = r.getLogLineWriter()
 	return cmd
@@ -374,12 +417,12 @@ func (r *stellarCoreRunner) getProcessExitError() (bool, error) {
 func (r *stellarCoreRunner) close() error {
 	r.lock.Lock()
 	started := r.started
-	tempDir := r.tempDir
+	storagePath := r.storagePath
 
-	r.tempDir = ""
+	r.storagePath = ""
 
 	// check if we have already closed
-	if tempDir == "" {
+	if storagePath == "" {
 		r.lock.Unlock()
 		return nil
 	}
@@ -403,5 +446,5 @@ func (r *stellarCoreRunner) close() error {
 		r.pipe.Reader.Close()
 	}
 
-	return os.RemoveAll(tempDir)
+	return os.RemoveAll(storagePath)
 }

ingest/ledgerbackend/stellar_core_runner_test.go

@@ -44,15 +44,13 @@ func TestGenerateConfig(t *testing.T) {
 				HistoryArchiveURLs: []string{"http://localhost:1170"},
 				Log:                log.New(),
 				ConfigAppendPath:   testCase.appendPath,
+				StoragePath:        "./test-temp-dir",
 				PeerPort:           12345,
 				Context:            context.Background(),
 				NetworkPassphrase:  "Public Global Stellar Network ; September 2015",
 			}, testCase.mode)
 			assert.NoError(t, err)
 
-			tempDir := stellarCoreRunner.tempDir
-			stellarCoreRunner.tempDir = "/test-temp-dir"
-
 			config, err := stellarCoreRunner.generateConfig()
 			assert.NoError(t, err)
 
@@ -61,7 +59,6 @@ func TestGenerateConfig(t *testing.T) {
 
 			assert.Equal(t, config, string(expectedByte))
 
-			stellarCoreRunner.tempDir = tempDir
 			assert.NoError(t, stellarCoreRunner.close())
 		})
 	}
@@ -75,7 +72,7 @@ func TestCloseBeforeStart(t *testing.T) {
 	}, stellarCoreRunnerModeOffline)
 	assert.NoError(t, err)
 
-	tempDir := runner.tempDir
+	tempDir := runner.storagePath
 	info, err := os.Stat(tempDir)
 	assert.NoError(t, err)
 	assert.True(t, info.IsDir())

ingest/ledgerbackend/testdata/expected-offline-core.cfg

@@ -2,7 +2,6 @@
 NODE_IS_VALIDATOR=false
 DISABLE_XDR_FSYNC=true
 NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"
-BUCKET_DIR_PATH="/test-temp-dir/buckets"
 HTTP_PORT=6789
 LOG_FILE_PATH=""
 PEER_PORT=12345

ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg

@@ -2,7 +2,6 @@
 NODE_IS_VALIDATOR=false
 DISABLE_XDR_FSYNC=true
 NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"
-BUCKET_DIR_PATH="/test-temp-dir/buckets"
 HTTP_PORT=6789
 LOG_FILE_PATH=""
 PEER_PORT=12345

ingest/ledgerbackend/testdata/expected-online-core.cfg

@@ -2,7 +2,6 @@
 NODE_IS_VALIDATOR=false
 DISABLE_XDR_FSYNC=true
 NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015"
-BUCKET_DIR_PATH="/test-temp-dir/buckets"
 HTTP_PORT=6789
 LOG_FILE_PATH=""
 PEER_PORT=12345

services/horizon/internal/config.go

@@ -23,6 +23,7 @@ type Config struct {
 	CaptiveCoreHTTPPort         uint
 	CaptiveCorePeerPort         uint
 	CaptiveCoreLogPath          string
+	CaptiveCoreStoragePath      string
 
 	StellarCoreDatabaseURL string
 	StellarCoreURL         string

services/horizon/internal/flags.go

@@ -133,6 +133,25 @@ func Flags() (*Config, support.ConfigOptions) {
 			Usage:       "HTTP port for Captive Core to listen on (0 disables the HTTP server)",
 			ConfigKey:   &config.CaptiveCoreHTTPPort,
 		},
+		&support.ConfigOption{
+			Name:        "captive-core-storage-path",
+			OptType:     types.String,
+			FlagDefault: "",
+			CustomSetValue: func(opt *support.ConfigOption) {
+				existingValue := viper.GetString(opt.Name)
+				if existingValue == "" || existingValue == "." {
+					cwd, err := os.Getwd()
+					if err != nil {
+						stdLog.Fatalf("Unable to determine the current directory: %s", err)
+					}
+					existingValue = cwd
+				}
+				*opt.ConfigKey.(*string) = existingValue
+			},
+			Required:  false,
+			Usage:     "Storage location for Captive Core bucket data",
+			ConfigKey: &config.CaptiveCoreStoragePath,
+		},
 		&support.ConfigOption{
 			Name:        "captive-core-peer-port",
 			OptType:     types.Uint,

services/horizon/internal/ingest/main.go

@@ -70,6 +70,7 @@ type Config struct {
 	StellarCoreCursor           string
 	EnableCaptiveCore           bool
 	CaptiveCoreBinaryPath       string
+	CaptiveCoreStoragePath      string
 	CaptiveCoreConfigAppendPath string
 	CaptiveCoreHTTPPort         uint
 	CaptiveCorePeerPort         uint
@@ -197,6 +198,7 @@ func NewSystem(config Config) (System, error) {
 				ledgerbackend.CaptiveCoreConfig{
 					LogPath:             config.CaptiveCoreLogPath,
 					BinaryPath:          config.CaptiveCoreBinaryPath,
+					StoragePath:         config.CaptiveCoreStoragePath,
 					ConfigAppendPath:    config.CaptiveCoreConfigAppendPath,
 					HTTPPort:            config.CaptiveCoreHTTPPort,
 					PeerPort:            config.CaptiveCorePeerPort,

services/horizon/internal/init.go

@@ -70,6 +70,7 @@ func initIngester(app *App) {
 		StellarCoreURL:              app.config.StellarCoreURL,
 		StellarCoreCursor:           app.config.CursorName,
 		CaptiveCoreBinaryPath:       app.config.CaptiveCoreBinaryPath,
+		CaptiveCoreStoragePath:      app.config.CaptiveCoreStoragePath,
 		CaptiveCoreConfigAppendPath: app.config.CaptiveCoreConfigAppendPath,
 		CaptiveCoreHTTPPort:         app.config.CaptiveCoreHTTPPort,
 		CaptiveCorePeerPort:         app.config.CaptiveCorePeerPort,