services/horizon: Refactor Captive Core configuration flags (#3262)

* Refactor Captive Core configuration flags

* Rename `--stellar-core-config-path` to `captive-core-append-config-path`.
  The base config is now automatically generated in captive-core online mode.

* Add `--captive-core-http-port` to indicate what port (if any) captive core
  shoulf listen on. If `--stellar-core-url` is unset and the local Captive core is enabled,
  `--stellar-core-url` will be implicitly use `http://localhost:<stellar_captive_core_http_port>`

exp/services/captivecore/main.go

@@ -20,8 +20,9 @@ import (
 
 func main() {
 	var port int
-	var networkPassphrase, binaryPath, configPath, dbURL string
+	var networkPassphrase, binaryPath, configAppendPath, dbURL string
 	var historyArchiveURLs []string
+	var stellarCoreHTTPPort uint
 	var logLevel logrus.Level
 	logger := supportlog.New()
 
@@ -51,12 +52,12 @@ func main() {
 			ConfigKey:   &binaryPath,
 		},
 		&config.ConfigOption{
-			Name:        "stellar-core-config-path",
+			Name:        "captive-core-config-append-path",
 			OptType:     types.String,
 			FlagDefault: "",
 			Required:    false,
-			Usage:       "path to stellar core config file",
-			ConfigKey:   &configPath,
+			Usage:       "path to additional configuration for the Stellar Core configuration file used by captive core. It must, at least, include enough details to define a quorum set",
+			ConfigKey:   &configAppendPath,
 		},
 		&config.ConfigOption{
 			Name:        "history-archive-urls",
@@ -94,6 +95,14 @@ func main() {
 			Required:  false,
 			Usage:     "horizon postgres database to connect with",
 		},
+		&config.ConfigOption{
+			Name:        "stellar-captive-core-http-port",
+			ConfigKey:   &stellarCoreHTTPPort,
+			OptType:     types.Uint,
+			FlagDefault: uint(11626),
+			Required:    false,
+			Usage:       "HTTP port for captive core to listen on (0 disables the HTTP server)",
+		},
 	}
 	cmd := &cobra.Command{
 		Use:   "captivecore",
@@ -104,10 +113,11 @@ func main() {
 			logger.SetLevel(logLevel)
 
 			captiveConfig := ledgerbackend.CaptiveCoreConfig{
-				StellarCoreBinaryPath: binaryPath,
-				StellarCoreConfigPath: configPath,
-				NetworkPassphrase:     networkPassphrase,
-				HistoryArchiveURLs:    historyArchiveURLs,
+				BinaryPath:         binaryPath,
+				ConfigAppendPath:   configAppendPath,
+				NetworkPassphrase:  networkPassphrase,
+				HistoryArchiveURLs: historyArchiveURLs,
+				HTTPPort:           stellarCoreHTTPPort,
 			}
 
 			var dbConn *db.Session

exp/tools/captive-core-start-tester/main.go

@@ -25,10 +25,10 @@ func main() {
 func check(ledger uint32) bool {
 	c, err := ledgerbackend.NewCaptive(
 		ledgerbackend.CaptiveCoreConfig{
-			StellarCoreBinaryPath: "stellar-core",
-			StellarCoreConfigPath: "stellar-core-standalone2.cfg",
-			NetworkPassphrase:     "Standalone Network ; February 2017",
-			HistoryArchiveURLs:    []string{"http://localhost:1570"},
+			BinaryPath:         "stellar-core",
+			ConfigAppendPath:   "stellar-core-standalone2.cfg",
+			NetworkPassphrase:  "Standalone Network ; February 2017",
+			HistoryArchiveURLs: []string{"http://localhost:1570"},
 		},
 	)
 	if err != nil {

ingest/doc_test.go

@@ -104,10 +104,10 @@ func Example_changes() {
 	// Requires Stellar-Core 13.2.0+
 	backend, err := ledgerbackend.NewCaptive(
 		ledgerbackend.CaptiveCoreConfig{
-			StellarCoreBinaryPath: "/bin/stellar-core",
-			StellarCoreConfigPath: "/opt/stellar-core.cfg",
-			NetworkPassphrase:     networkPassphrase,
-			HistoryArchiveURLs:    []string{archiveURL},
+			BinaryPath:         "/bin/stellar-core",
+			ConfigAppendPath:   "/opt/stellar-core.cfg",
+			NetworkPassphrase:  networkPassphrase,
+			HistoryArchiveURLs: []string{archiveURL},
 		},
 	)
 	if err != nil {

ingest/ledgerbackend/captive_core_backend.go

@@ -34,7 +34,7 @@ func roundDownToFirstReplayAfterCheckpointStart(ledger uint32) uint32 {
 //     keep ledger state in RAM. It requires around 3GB of RAM as of August 2020.
 //   * When a UnboundedRange is prepared it runs Stellar-Core catchup mode to
 //     sync with the first ledger and then runs it in a normal mode. This
-//     requires the configPath to be provided because a database connection is
+//     requires the configAppendPath to be provided because a database connection is
 //     required and quorum set needs to be selected.
 //
 // The database requirement for UnboundedRange will soon be removed when some
@@ -62,12 +62,9 @@ func roundDownToFirstReplayAfterCheckpointStart(ledger uint32) uint32 {
 //
 // Requires Stellar-Core v13.2.0+.
 type CaptiveStellarCore struct {
-	executablePath    string
-	configPath        string
-	networkPassphrase string
-	historyURLs       []string
-	archive           historyarchive.ArchiveInterface
-	ledgerHashStore   TrustedLedgerHashStore
+	configAppendPath string
+	archive          historyarchive.ArchiveInterface
+	ledgerHashStore  TrustedLedgerHashStore
 
 	// Quick note on how shutdown works:
 	// If Stellar-Core exits, the exit signal is "catched" by bufferedLedgerMetaReader
@@ -79,7 +76,7 @@ type CaptiveStellarCore struct {
 	stellarCoreRunner stellarCoreRunnerInterface
 
 	// For testing
-	stellarCoreRunnerFactory func(configPath string) (stellarCoreRunnerInterface, error)
+	stellarCoreRunnerFactory func(mode stellarCoreRunnerMode, captiveCoreConfigAppendPath string) (stellarCoreRunnerInterface, error)
 
 	// Defines if the blocking mode (off by default) is on or off. In blocking mode,
 	// calling GetLedger blocks until the requested ledger is available. This is useful
@@ -105,21 +102,23 @@ type CaptiveStellarCore struct {
 
 // CaptiveCoreConfig contains all the parameters required to create a CaptiveStellarCore instance
 type CaptiveCoreConfig struct {
-	// StellarCoreBinaryPath is the file path to the Stellar Core binary
-	StellarCoreBinaryPath string
-	// StellarCoreConfigPath is the file path to the Stellar Core configuration file used by captive core
-	StellarCoreConfigPath string
+	// BinaryPath is the file path to the Stellar Core binary
+	BinaryPath string
+	// ConfigAppendPath is the file path to additional configuration for the Stellar Core configuration file used by captive core
+	ConfigAppendPath string
 	// NetworkPassphrase is the Stellar network passphrase used by captive core when connecting to the Stellar network
 	NetworkPassphrase string
 	// HistoryArchiveURLs are a list of history archive urls
 	HistoryArchiveURLs []string
 	// LedgerHashStore is an optional store used to obtain hashes for ledger sequences from a trusted source
 	LedgerHashStore TrustedLedgerHashStore
+	// HTTPPort is the TCP port to listen for requests (defaults to 0, which disables the HTTP server)
+	HTTPPort uint
 }
 
 // NewCaptive returns a new CaptiveStellarCore.
 //
-// All parameters are required, except configPath which is not required when
+// All parameters are required, except configAppendPath which is not required when
 // working with BoundedRanges only.
 func NewCaptive(config CaptiveCoreConfig) (*CaptiveStellarCore, error) {
 	archive, err := historyarchive.Connect(
@@ -134,19 +133,18 @@ func NewCaptive(config CaptiveCoreConfig) (*CaptiveStellarCore, error) {
 
 	c := &CaptiveStellarCore{
 		archive:                  archive,
-		executablePath:           config.StellarCoreBinaryPath,
-		configPath:               config.StellarCoreConfigPath,
-		historyURLs:              config.HistoryArchiveURLs,
-		networkPassphrase:        config.NetworkPassphrase,
+		configAppendPath:         config.ConfigAppendPath,
 		waitIntervalPrepareRange: time.Second,
 		ledgerHashStore:          config.LedgerHashStore,
 	}
-	c.stellarCoreRunnerFactory = func(configPath2 string) (stellarCoreRunnerInterface, error) {
+	c.stellarCoreRunnerFactory = func(mode stellarCoreRunnerMode, appendConfigPath string) (stellarCoreRunnerInterface, error) {
 		runner, innerErr := newStellarCoreRunner(
-			config.StellarCoreBinaryPath,
-			configPath2,
+			config.BinaryPath,
+			appendConfigPath,
 			config.NetworkPassphrase,
+			config.HTTPPort,
 			config.HistoryArchiveURLs,
+			mode,
 		)
 		if innerErr != nil {
 			return runner, innerErr
@@ -192,8 +190,8 @@ func (c *CaptiveStellarCore) openOfflineReplaySubprocess(from, to uint32) error
 	}
 
 	if c.stellarCoreRunner == nil {
-		// configPath is empty in an offline mode because it's generated
-		c.stellarCoreRunner, err = c.stellarCoreRunnerFactory("")
+		// configAppendPath is empty in an offline mode because it's generated
+		c.stellarCoreRunner, err = c.stellarCoreRunnerFactory(stellarCoreRunnerModeOffline, "")
 		if err != nil {
 			return errors.Wrap(err, "error creating stellar-core runner")
 		}
@@ -245,10 +243,7 @@ func (c *CaptiveStellarCore) openOnlineReplaySubprocess(from uint32) error {
 	}
 
 	if c.stellarCoreRunner == nil {
-		if c.configPath == "" {
-			return errors.New("stellar-core config file path cannot be empty in an online mode")
-		}
-		c.stellarCoreRunner, err = c.stellarCoreRunnerFactory(c.configPath)
+		c.stellarCoreRunner, err = c.stellarCoreRunnerFactory(stellarCoreRunnerModeOnline, c.configAppendPath)
 		if err != nil {
 			return errors.Wrap(err, "error creating stellar-core runner")
 		}