From e142246a90da04c0cd07c52fe14cd2c25f641199 Mon Sep 17 00:00:00 2001 From: tamirms Date: Wed, 21 Apr 2021 14:57:17 +0200 Subject: [PATCH] Parse captive core toml to implement better merging --- exp/services/captivecore/main.go | 33 +- exp/tools/captive-core-start-tester/main.go | 1 - ingest/doc_test.go | 1 - ingest/ledgerbackend/captive_core_backend.go | 13 +- .../captive_core_backend_test.go | 6 +- ingest/ledgerbackend/stellar_core_runner.go | 99 ++--- .../ledgerbackend/stellar_core_runner_test.go | 32 +- .../testdata/expected-offline-core.cfg | 29 +- .../expected-offline-with-appendix-core.cfg | 34 +- .../testdata/expected-online-core.cfg | 30 +- ingest/ledgerbackend/toml.go | 345 ++++++++++++++++++ ingest/tutorial/example_common.go | 4 +- services/horizon/cmd/db.go | 5 +- services/horizon/internal/config.go | 15 +- services/horizon/internal/flags.go | 64 ++-- services/horizon/internal/ingest/main.go | 30 +- services/horizon/internal/init.go | 23 +- .../horizon/internal/integration/db_test.go | 4 +- .../internal/test/integration/integration.go | 2 +- support/config/config_option.go | 24 ++ support/config/config_option_test.go | 6 + 21 files changed, 567 insertions(+), 233 deletions(-) create mode 100644 ingest/ledgerbackend/toml.go diff --git a/exp/services/captivecore/main.go b/exp/services/captivecore/main.go index f5ec78447d..8f1cdaf7aa 100644 --- a/exp/services/captivecore/main.go +++ b/exp/services/captivecore/main.go @@ -13,6 +13,7 @@ import ( "github.com/stellar/go/ingest/ledgerbackend" "github.com/stellar/go/network" "github.com/stellar/go/support/config" + support "github.com/stellar/go/support/config" "github.com/stellar/go/support/db" supporthttp "github.com/stellar/go/support/http" supportlog "github.com/stellar/go/support/log" @@ -20,9 +21,9 @@ import ( func main() { var port int - var networkPassphrase, binaryPath, configAppendPath, dbURL string + var networkPassphrase, binaryPath, dbURL string + var captiveCoreTomlParams ledgerbackend.CaptiveCoreTomlParams var historyArchiveURLs []string - var stellarCoreHTTPPort uint var checkpointFrequency uint32 var logLevel logrus.Level logger := supportlog.New() @@ -56,9 +57,9 @@ func main() { Name: "captive-core-config-append-path", OptType: types.String, FlagDefault: "", - Required: false, + Required: true, Usage: "path to additional configuration for the Stellar Core configuration file used by captive core. It must, at least, include enough details to define a quorum set", - ConfigKey: &configAppendPath, + ConfigKey: &captiveCoreTomlParams.ConfigPath, }, &config.ConfigOption{ Name: "history-archive-urls", @@ -97,12 +98,13 @@ func main() { Usage: "horizon postgres database to connect with", }, &config.ConfigOption{ - Name: "stellar-captive-core-http-port", - ConfigKey: &stellarCoreHTTPPort, - OptType: types.Uint, - FlagDefault: uint(11626), - Required: false, - Usage: "HTTP port for captive core to listen on (0 disables the HTTP server)", + Name: "stellar-captive-core-http-port", + ConfigKey: &captiveCoreTomlParams.HTTPPort, + OptType: types.Uint, + CustomSetValue: support.SetOptionalUint, + Required: false, + FlagDefault: uint(0), + Usage: "HTTP port for Captive Core to listen on (0 disables the HTTP server)", }, &config.ConfigOption{ Name: "checkpoint-frequency", @@ -121,19 +123,24 @@ func main() { configOpts.SetValues() logger.SetLevel(logLevel) + captiveCoreTomlParams.HistoryArchiveURLs = historyArchiveURLs + captiveCoreTomlParams.NetworkPassphrase = networkPassphrase + captiveCoreToml, err := ledgerbackend.NewCaptiveCoreToml(captiveCoreTomlParams) + if err != nil { + logger.WithError(err).Fatal("Invalid captive core toml") + } + captiveConfig := ledgerbackend.CaptiveCoreConfig{ BinaryPath: binaryPath, - ConfigAppendPath: configAppendPath, NetworkPassphrase: networkPassphrase, HistoryArchiveURLs: historyArchiveURLs, CheckpointFrequency: checkpointFrequency, - HTTPPort: stellarCoreHTTPPort, Log: logger.WithField("subservice", "stellar-core"), + Toml: captiveCoreToml, } var dbConn *db.Session if len(dbURL) > 0 { - var err error dbConn, err = db.Open("postgres", dbURL) if err != nil { logger.WithError(err).Fatal("Could not create db connection instance") diff --git a/exp/tools/captive-core-start-tester/main.go b/exp/tools/captive-core-start-tester/main.go index 18984818ed..89f351f557 100644 --- a/exp/tools/captive-core-start-tester/main.go +++ b/exp/tools/captive-core-start-tester/main.go @@ -26,7 +26,6 @@ func check(ledger uint32) bool { c, err := ledgerbackend.NewCaptive( ledgerbackend.CaptiveCoreConfig{ BinaryPath: "stellar-core", - ConfigAppendPath: "stellar-core-standalone2.cfg", NetworkPassphrase: "Standalone Network ; February 2017", HistoryArchiveURLs: []string{"http://localhost:1570"}, }, diff --git a/ingest/doc_test.go b/ingest/doc_test.go index efad710140..47e1de6855 100644 --- a/ingest/doc_test.go +++ b/ingest/doc_test.go @@ -71,7 +71,6 @@ func Example_changes() { backend, err := ledgerbackend.NewCaptive( ledgerbackend.CaptiveCoreConfig{ BinaryPath: "/bin/stellar-core", - ConfigAppendPath: "/opt/stellar-core.cfg", NetworkPassphrase: networkPassphrase, HistoryArchiveURLs: []string{archiveURL}, }, diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go index a4f00c191b..bab868d513 100644 --- a/ingest/ledgerbackend/captive_core_backend.go +++ b/ingest/ledgerbackend/captive_core_backend.go @@ -106,13 +106,11 @@ type CaptiveStellarCore struct { type CaptiveCoreConfig struct { // BinaryPath is the file path to the Stellar Core binary BinaryPath string - // ConfigAppendPath is the file path to additional configuration for the Stellar Core configuration file used - // by captive core. This field is only required when ingesting in online mode (e.g. UnboundedRange). - ConfigAppendPath string // NetworkPassphrase is the Stellar network passphrase used by captive core when connecting to the Stellar network NetworkPassphrase string // HistoryArchiveURLs are a list of history archive urls HistoryArchiveURLs []string + Toml *CaptiveCoreToml // Optional fields @@ -121,18 +119,9 @@ type CaptiveCoreConfig struct { CheckpointFrequency uint32 // LedgerHashStore is an optional store used to obtain hashes for ledger sequences from a trusted source LedgerHashStore TrustedLedgerHashStore - // HTTPPort is the TCP port to listen for requests (defaults to 0, which disables the HTTP server) - HTTPPort uint - // PeerPort is the TCP port to bind to for connecting to the Stellar network - // (defaults to 11625). It may be useful for example when there's >1 Stellar-Core - // instance running on a machine. - PeerPort uint // Log is an (optional) custom logger which will capture any output from the Stellar Core process. // If Log is omitted then all output will be printed to stdout. Log *log.Entry - // LogPath is the (optional) path in which to store Core logs, passed along - // to Stellar Core's LOG_FILE_PATH - LogPath string // Context is the (optional) context which controls the lifetime of a CaptiveStellarCore instance. Once the context is done // the CaptiveStellarCore instance will not be able to stream ledgers from Stellar Core or spawn new // instances of Stellar Core. If Context is omitted CaptiveStellarCore will default to using context.Background. diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go index 3870199f61..38c4cdcf1a 100644 --- a/ingest/ledgerbackend/captive_core_backend_test.go +++ b/ingest/ledgerbackend/captive_core_backend_test.go @@ -132,14 +132,12 @@ type testLedgerHeader struct { func TestCaptiveNew(t *testing.T) { executablePath := "/etc/stellar-core" - configPath := "/etc/stellar-core.cfg" networkPassphrase := network.PublicNetworkPassphrase historyURLs := []string{"http://history.stellar.org/prd/core-live/core_live_001"} captiveStellarCore, err := NewCaptive( CaptiveCoreConfig{ BinaryPath: executablePath, - ConfigAppendPath: configPath, NetworkPassphrase: networkPassphrase, HistoryArchiveURLs: historyURLs, }, @@ -631,11 +629,15 @@ func TestCaptiveStellarCore_PrepareRangeAfterClose(t *testing.T) { networkPassphrase := network.PublicNetworkPassphrase historyURLs := []string{"http://localhost"} + captiveCoreToml, err := NewCaptiveCoreToml(CaptiveCoreTomlParams{}) + assert.NoError(t, err) + captiveStellarCore, err := NewCaptive( CaptiveCoreConfig{ BinaryPath: executablePath, NetworkPassphrase: networkPassphrase, HistoryArchiveURLs: historyURLs, + Toml: captiveCoreToml, }, ) assert.NoError(t, err) diff --git a/ingest/ledgerbackend/stellar_core_runner.go b/ingest/ledgerbackend/stellar_core_runner.go index bbfd78d41a..c53b44a2cf 100644 --- a/ingest/ledgerbackend/stellar_core_runner.go +++ b/ingest/ledgerbackend/stellar_core_runner.go @@ -49,14 +49,7 @@ type pipe struct { } type stellarCoreRunner struct { - logPath string - executablePath string - configAppendPath string - networkPassphrase string - historyURLs []string - httpPort uint - peerPort uint - mode stellarCoreRunnerMode + executablePath string started bool wg sync.WaitGroup @@ -112,17 +105,10 @@ func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode) ctx, cancel := context.WithCancel(config.Context) runner := &stellarCoreRunner{ - logPath: config.LogPath, - executablePath: config.BinaryPath, - configAppendPath: config.ConfigAppendPath, - networkPassphrase: config.NetworkPassphrase, - historyURLs: config.HistoryArchiveURLs, - httpPort: config.HTTPPort, - peerPort: config.PeerPort, - mode: mode, - ctx: ctx, - cancel: cancel, - storagePath: fullStoragePath, + executablePath: config.BinaryPath, + ctx: ctx, + cancel: cancel, + storagePath: fullStoragePath, nonce: fmt.Sprintf( "captive-stellar-core-%x", rand.New(rand.NewSource(time.Now().UnixNano())).Uint64(), @@ -130,66 +116,38 @@ func newStellarCoreRunner(config CaptiveCoreConfig, mode stellarCoreRunnerMode) log: config.Log, } - if err := runner.writeConf(); err != nil { + if conf, err := writeConf(config.Toml, mode, runner.getConfFileName()); err != nil { return nil, errors.Wrap(err, "error writing configuration") + } else { + runner.log.Debugf("captive core config file contents:\n%s", conf) } return runner, nil } -func (r *stellarCoreRunner) generateConfig() (string, error) { - if r.mode == stellarCoreRunnerModeOnline && r.configAppendPath == "" { - return "", errors.New("stellar-core append config file path cannot be empty in online mode") - } - - lines := []string{ - "# Generated file -- do not edit", - "NODE_IS_VALIDATOR=false", - "DISABLE_XDR_FSYNC=true", - fmt.Sprintf(`NETWORK_PASSPHRASE="%s"`, r.networkPassphrase), - // Note: We don't pass BUCKET_DIR_PATH here because it's created - // *relative* to the storage path (i.e. where the config lives). - fmt.Sprintf(`HTTP_PORT=%d`, r.httpPort), - fmt.Sprintf(`LOG_FILE_PATH="%s"`, r.logPath), - } - - if r.peerPort > 0 { - lines = append(lines, fmt.Sprintf(`PEER_PORT=%d`, r.peerPort)) +func writeConf(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode, location string) (string, error) { + text, err := generateConfig(captiveCoreToml, mode) + if err != nil { + return "", err } - if r.mode == stellarCoreRunnerModeOffline { - // In offline mode, there is no need to connect to peers - lines = append(lines, "RUN_STANDALONE=true") - // We don't need consensus when catching up - lines = append(lines, "UNSAFE_QUORUM=true") - } + return string(text), ioutil.WriteFile(location, text, 0644) +} - if r.mode == stellarCoreRunnerModeOffline && r.configAppendPath == "" { - // Add a fictional quorum -- necessary to convince core to start up; - // but not used at all for our purposes. Pubkey here is just random. - lines = append(lines, - "[QUORUM_SET]", - "THRESHOLD_PERCENT=100", - `VALIDATORS=["GCZBOIAY4HLKAJVNJORXZOZRAY2BJDBZHKPBHZCRAIUR5IHC2UHBGCQR"]`) +func generateConfig(captiveCoreToml *CaptiveCoreToml, mode stellarCoreRunnerMode) ([]byte, error) { + if mode == stellarCoreRunnerModeOffline { + captiveCoreToml = captiveCoreToml.CatchupToml() } - result := strings.ReplaceAll(strings.Join(lines, "\n"), `\`, `\\`) + "\n\n" - if r.configAppendPath != "" { - appendConfigContents, err := ioutil.ReadFile(r.configAppendPath) - if err != nil { - return "", errors.Wrap(err, "reading quorum config file") - } - result += string(appendConfigContents) + "\n\n" + if !captiveCoreToml.QuorumSetIsConfigured() { + return nil, errors.New("stellar-core append config file does not define any quorum set") } - lines = []string{} - for i, val := range r.historyURLs { - lines = append(lines, fmt.Sprintf("[HISTORY.h%d]", i)) - lines = append(lines, fmt.Sprintf(`get="curl -sf %s/{0} -o {1}"`, val)) + text, err := captiveCoreToml.Marshall() + if err != nil { + return nil, errors.Wrap(err, "could not marshal captive core config") } - result += strings.Join(lines, "\n") - - return result, nil + return text, nil } func (r *stellarCoreRunner) getConfFileName() string { @@ -255,17 +213,6 @@ func (r *stellarCoreRunner) getLogLineWriter() io.Writer { return wr } -// Makes the temp directory and writes the config file to it; called by the -// platform-specific captiveStellarCore.Start() methods. -func (r *stellarCoreRunner) writeConf() error { - conf, err := r.generateConfig() - if err != nil { - return err - } - r.log.Debugf("captive core config file contents:\n%s", conf) - return ioutil.WriteFile(r.getConfFileName(), []byte(conf), 0644) -} - func (r *stellarCoreRunner) createCmd(params ...string) *exec.Cmd { allParams := append([]string{"--conf", r.getConfFileName()}, params...) cmd := exec.CommandContext(r.ctx, r.executablePath, allParams...) diff --git a/ingest/ledgerbackend/stellar_core_runner_test.go b/ingest/ledgerbackend/stellar_core_runner_test.go index 24d99e17b8..b81e7025f5 100644 --- a/ingest/ledgerbackend/stellar_core_runner_test.go +++ b/ingest/ledgerbackend/stellar_core_runner_test.go @@ -12,6 +12,12 @@ import ( "github.com/stellar/go/support/log" ) +func newUint(v uint) *uint { + p := new(uint) + *p = v + return p +} + func TestGenerateConfig(t *testing.T) { for _, testCase := range []struct { name string @@ -39,36 +45,36 @@ func TestGenerateConfig(t *testing.T) { }, } { t.Run(testCase.name, func(t *testing.T) { - stellarCoreRunner, err := newStellarCoreRunner(CaptiveCoreConfig{ - HTTPPort: 6789, - HistoryArchiveURLs: []string{"http://localhost:1170"}, - Log: log.New(), - ConfigAppendPath: testCase.appendPath, - StoragePath: "./test-temp-dir", - PeerPort: 12345, - Context: context.Background(), + captiveCoreToml, err := NewCaptiveCoreToml(CaptiveCoreTomlParams{ + ConfigPath: testCase.appendPath, + Strict: true, NetworkPassphrase: "Public Global Stellar Network ; September 2015", - }, testCase.mode) + HistoryArchiveURLs: []string{"http://localhost:1170"}, + HTTPPort: newUint(6789), + PeerPort: newUint(12345), + }) assert.NoError(t, err) - config, err := stellarCoreRunner.generateConfig() + configBytes, err := generateConfig(captiveCoreToml, testCase.mode) assert.NoError(t, err) expectedByte, err := ioutil.ReadFile(testCase.expectedPath) assert.NoError(t, err) - assert.Equal(t, config, string(expectedByte)) - - assert.NoError(t, stellarCoreRunner.close()) + assert.Equal(t, string(configBytes), string(expectedByte)) }) } } func TestCloseBeforeStart(t *testing.T) { + captiveCoreToml, err := NewCaptiveCoreToml(CaptiveCoreTomlParams{}) + assert.NoError(t, err) + runner, err := newStellarCoreRunner(CaptiveCoreConfig{ HistoryArchiveURLs: []string{"http://localhost"}, Log: log.New(), Context: context.Background(), + Toml: captiveCoreToml, }, stellarCoreRunnerModeOffline) assert.NoError(t, err) diff --git a/ingest/ledgerbackend/testdata/expected-offline-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-core.cfg index 40daca9422..e11c15fd41 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-core.cfg @@ -1,15 +1,18 @@ # Generated file -- do not edit -NODE_IS_VALIDATOR=false -DISABLE_XDR_FSYNC=true -NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" -HTTP_PORT=6789 -LOG_FILE_PATH="" -PEER_PORT=12345 -RUN_STANDALONE=true -UNSAFE_QUORUM=true -[QUORUM_SET] -THRESHOLD_PERCENT=100 -VALIDATORS=["GCZBOIAY4HLKAJVNJORXZOZRAY2BJDBZHKPBHZCRAIUR5IHC2UHBGCQR"] +LOG_FILE_PATH = "" +HTTP_PORT = 0 +NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015" +PEER_PORT = 12345 +FAILURE_SAFETY = 0 +UNSAFE_QUORUM = true +RUN_STANDALONE = true +DISABLE_XDR_FSYNC = true + +[HISTORY] + [HISTORY.h0] + get = "curl -sf http://localhost:1170/{0} -o {1}" -[HISTORY.h0] -get="curl -sf http://localhost:1170/{0} -o {1}" \ No newline at end of file +[QUORUM_SET] + [QUORUM_SET.generated] + THRESHOLD_PERCENT = 100 + VALIDATORS = ["GCZBOIAY4HLKAJVNJORXZOZRAY2BJDBZHKPBHZCRAIUR5IHC2UHBGCQR"] diff --git a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg index 4e6011cdf0..9804a7867b 100644 --- a/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-offline-with-appendix-core.cfg @@ -1,23 +1,23 @@ # Generated file -- do not edit -NODE_IS_VALIDATOR=false -DISABLE_XDR_FSYNC=true -NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" -HTTP_PORT=6789 -LOG_FILE_PATH="" -PEER_PORT=12345 -RUN_STANDALONE=true -UNSAFE_QUORUM=true +LOG_FILE_PATH = "" +HTTP_PORT = 0 +NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015" +PEER_PORT = 12345 +FAILURE_SAFETY = 0 +UNSAFE_QUORUM = true +RUN_STANDALONE = true +DISABLE_XDR_FSYNC = true [[HOME_DOMAINS]] -HOME_DOMAIN="testnet.stellar.org" -QUALITY="MEDIUM" + HOME_DOMAIN = "testnet.stellar.org" + QUALITY = "MEDIUM" [[VALIDATORS]] -NAME="sdf_testnet_1" -HOME_DOMAIN="testnet.stellar.org" -PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" -ADDRESS="core-testnet1.stellar.org" + NAME = "sdf_testnet_1" + HOME_DOMAIN = "testnet.stellar.org" + PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" + ADDRESS = "core-testnet1.stellar.org" - -[HISTORY.h0] -get="curl -sf http://localhost:1170/{0} -o {1}" \ No newline at end of file +[HISTORY] + [HISTORY.h0] + get = "curl -sf http://localhost:1170/{0} -o {1}" diff --git a/ingest/ledgerbackend/testdata/expected-online-core.cfg b/ingest/ledgerbackend/testdata/expected-online-core.cfg index 464554156e..a64f773298 100644 --- a/ingest/ledgerbackend/testdata/expected-online-core.cfg +++ b/ingest/ledgerbackend/testdata/expected-online-core.cfg @@ -1,21 +1,21 @@ # Generated file -- do not edit -NODE_IS_VALIDATOR=false -DISABLE_XDR_FSYNC=true -NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" -HTTP_PORT=6789 -LOG_FILE_PATH="" -PEER_PORT=12345 +LOG_FILE_PATH = "" +HTTP_PORT = 6789 +NETWORK_PASSPHRASE = "Public Global Stellar Network ; September 2015" +PEER_PORT = 12345 +FAILURE_SAFETY = -1 +DISABLE_XDR_FSYNC = true [[HOME_DOMAINS]] -HOME_DOMAIN="testnet.stellar.org" -QUALITY="MEDIUM" + HOME_DOMAIN = "testnet.stellar.org" + QUALITY = "MEDIUM" [[VALIDATORS]] -NAME="sdf_testnet_1" -HOME_DOMAIN="testnet.stellar.org" -PUBLIC_KEY="GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" -ADDRESS="core-testnet1.stellar.org" + NAME = "sdf_testnet_1" + HOME_DOMAIN = "testnet.stellar.org" + PUBLIC_KEY = "GDKXE2OZMJIPOSLNA6N6F2BVCI3O777I2OOC4BV7VOYUEHYX7RTRYA7Y" + ADDRESS = "core-testnet1.stellar.org" - -[HISTORY.h0] -get="curl -sf http://localhost:1170/{0} -o {1}" \ No newline at end of file +[HISTORY] + [HISTORY.h0] + get = "curl -sf http://localhost:1170/{0} -o {1}" diff --git a/ingest/ledgerbackend/toml.go b/ingest/ledgerbackend/toml.go new file mode 100644 index 0000000000..fb861d1ae2 --- /dev/null +++ b/ingest/ledgerbackend/toml.go @@ -0,0 +1,345 @@ +package ledgerbackend + +import ( + "fmt" + "io/ioutil" + "strings" + + "github.com/stellar/go/support/errors" + "github.com/stellar/go/xdr" + + "github.com/BurntSushi/toml" +) + +const ( + defaultHTTPPort = 11626 + defaultFailureSafety = -1 + + // if LOG_FILE_PATH is omitted stellar core actually defaults to "stellar-core.log" + // however, we are overriding this default for captive core + defaultLogFilePath = "" // by default we disable logging to a file + + // if DISABLE_XDR_FSYNC is omitted stellar core actually defaults to false + // however, we are overriding this default for captive core + defaultDisableXDRFsync = true +) + +var validQuality = map[string]bool{ + "CRITICAL": true, + "HIGH": true, + "MEDIUM": true, + "LOW": true, +} + +type Validator struct { + Name string `toml:"NAME"` + Quality string `toml:"QUALITY,omitempty"` + HomeDomain string `toml:"HOME_DOMAIN"` + PublicKey string `toml:"PUBLIC_KEY"` + Address string `toml:"ADDRESS,omitempty"` + History string `toml:"HISTORY,omitempty"` +} + +type HomeDomain struct { + HomeDomain string `toml:"HOME_DOMAIN"` + Quality string `toml:"QUALITY"` +} + +type History struct { + Get string `toml:"get"` + // should we allow put and mkdir for captive core? + Put string `toml:"put,omitempty"` + Mkdir string `toml:"mkdir,omitempty"` +} + +type QuorumSet struct { + ThresholdPercent int `toml:"THRESHOLD_PERCENT"` + Validators []string `toml:"VALIDATORS"` +} + +type captiveCoreTomlValues struct { + // we cannot omitempty because the empty string is a valid configuration for LOG_FILE_PATH + // and the default is stellar-core.log + LogFilePath string `toml:"LOG_FILE_PATH"` + BucketDirPath string `toml:"BUCKET_DIR_PATH,omitempty"` + // we cannot omitzero because 0 is a valid configuration for HTTP_PORT + // and the default is 11626 + HTTPPort uint `toml:"HTTP_PORT"` + PublicHTTPPort bool `toml:"PUBLIC_HTTP_PORT,omitempty"` + NodeNames []string `toml:"NODE_NAMES,omitempty"` + NetworkPassphrase string `toml:"NETWORK_PASSPHRASE,omitempty"` + PeerPort uint `toml:"PEER_PORT,omitzero"` + // we cannot omitzero because 0 is a valid configuration for FAILURE_SAFETY + // and the default is -1 + FailureSafety int `toml:"FAILURE_SAFETY"` + UnsafeQuorum bool `toml:"UNSAFE_QUORUM,omitempty"` + RunStandalone bool `toml:"RUN_STANDALONE,omitempty"` + ArtificiallyAccelerateTimeForTesting bool `toml:"ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING,omitempty"` + DisableXDRFsync bool `toml:"DISABLE_XDR_FSYNC,omitempty"` + HomeDomains []HomeDomain `toml:"HOME_DOMAINS,omitempty"` + Validators []Validator `toml:"VALIDATORS,omitempty"` + HistoryEntries map[string]History `toml:"HISTORY,omitempty"` + QuorumSetEntries map[string]QuorumSet `toml:"QUORUM_SET,omitempty"` +} + +func (c captiveCoreTomlValues) Marshall() ([]byte, error) { + var sb strings.Builder + sb.WriteString("# Generated file -- do not edit\n") + if err := toml.NewEncoder(&sb).Encode(c); err != nil { + return nil, errors.Wrap(err, "could not encode toml file") + } + return []byte(sb.String()), nil +} + +func (c *captiveCoreTomlValues) QuorumSetIsConfigured() bool { + return len(c.QuorumSetEntries) > 0 || len(c.Validators) > 0 +} + +func (c *captiveCoreTomlValues) historyIsConfigured() bool { + if len(c.HistoryEntries) > 0 { + return true + } + for _, v := range c.Validators { + if v.History != "" { + return true + } + } + return false +} + +type CaptiveCoreToml struct { + captiveCoreTomlValues + metadata toml.MetaData +} + +func (c *CaptiveCoreToml) Unmarshal(text []byte) error { + metadata, err := toml.Decode(string(text), &c.captiveCoreTomlValues) + if err != nil { + return err + } + c.metadata = metadata + return nil +} + +type CaptiveCoreTomlParams struct { + ConfigPath string + Strict bool + // NetworkPassphrase is the Stellar network passphrase used by captive core when connecting to the Stellar network + NetworkPassphrase string + // HistoryArchiveURLs are a list of history archive urls + HistoryArchiveURLs []string + // HTTPPort is the TCP port to listen for requests (defaults to 0, which disables the HTTP server) + HTTPPort *uint + // PeerPort is the TCP port to bind to for connecting to the Stellar network + // (defaults to 11625). It may be useful for example when there's >1 Stellar-Core + // instance running on a machine. + PeerPort *uint + // LogPath is the (optional) path in which to store Core logs, passed along + // to Stellar Core's LOG_FILE_PATH + LogPath *string +} + +func NewCaptiveCoreToml(params CaptiveCoreTomlParams) (*CaptiveCoreToml, error) { + var captiveCoreToml CaptiveCoreToml + if params.ConfigPath != "" { + text, err := ioutil.ReadFile(params.ConfigPath) + if err != nil { + return nil, errors.Wrap(err, "could not load toml path") + } + if err = captiveCoreToml.Unmarshal(text); err != nil { + return nil, errors.Wrap(err, "could not unmarshall captive core toml") + } + if err = captiveCoreToml.validate(params); err != nil { + return nil, errors.Wrap(err, "invalid captive core toml") + } + } + captiveCoreToml.setDefaults(params) + + return &captiveCoreToml, nil +} + +func (c *CaptiveCoreToml) CatchupToml() *CaptiveCoreToml { + offline := *c + if !c.metadata.IsDefined("RUN_STANDALONE") { + offline.RunStandalone = true + } + if !c.metadata.IsDefined("UNSAFE_QUORUM") { + offline.UnsafeQuorum = true + } + if !c.QuorumSetIsConfigured() { + // Add a fictional quorum -- necessary to convince core to start up; + // but not used at all for our purposes. Pubkey here is just random. + offline.QuorumSetEntries = map[string]QuorumSet{ + "generated": QuorumSet{ + ThresholdPercent: 100, + Validators: []string{"GCZBOIAY4HLKAJVNJORXZOZRAY2BJDBZHKPBHZCRAIUR5IHC2UHBGCQR"}, + }, + } + } + offline.PublicHTTPPort = false + offline.HTTPPort = 0 + offline.FailureSafety = 0 + return &offline +} + +func (c *CaptiveCoreToml) setDefaults(params CaptiveCoreTomlParams) { + if !c.metadata.IsDefined("NETWORK_PASSPHRASE") { + c.NetworkPassphrase = params.NetworkPassphrase + } + + if def := c.metadata.IsDefined("HTTP_PORT"); !def && params.HTTPPort != nil { + c.HTTPPort = *params.HTTPPort + } else if !def && params.HTTPPort == nil { + c.HTTPPort = defaultHTTPPort + } + + if def := c.metadata.IsDefined("PEER_PORT"); !def && params.PeerPort != nil { + c.PeerPort = *params.PeerPort + } + + if def := c.metadata.IsDefined("LOG_FILE_PATH"); !def && params.LogPath != nil { + c.LogFilePath = *params.LogPath + } else if !def && params.LogPath == nil { + c.LogFilePath = defaultLogFilePath + } + + if !c.metadata.IsDefined("FAILURE_SAFETY") { + c.FailureSafety = defaultFailureSafety + } + if !c.metadata.IsDefined("DISABLE_XDR_FSYNC") { + c.DisableXDRFsync = defaultDisableXDRFsync + } + + if !c.historyIsConfigured() { + c.HistoryEntries = map[string]History{} + for i, val := range params.HistoryArchiveURLs { + c.HistoryEntries[fmt.Sprintf("h%d", i)] = History{ + Get: fmt.Sprintf("curl -sf %s/{0} -o {1}", val), + } + } + } +} + +func (c *CaptiveCoreToml) validate(params CaptiveCoreTomlParams) error { + if params.Strict { + if unexpected := c.metadata.Undecoded(); len(unexpected) > 0 { + var parts []string + for _, key := range unexpected { + parts = append(parts, key.String()) + } + return fmt.Errorf("invalid keys in captive core configuration: %s", strings.Join(parts, ", ")) + } + } + + if def := c.metadata.IsDefined("NETWORK_PASSPHRASE"); def && c.NetworkPassphrase != params.NetworkPassphrase { + return fmt.Errorf( + "NETWORK_PASSPHRASE in captive core config file: %s does not match Horizon network-passphrase flag: %s", + c.NetworkPassphrase, + params.NetworkPassphrase, + ) + } + + if def := c.metadata.IsDefined("HTTP_PORT"); def && params.HTTPPort != nil && c.HTTPPort != *params.HTTPPort { + return fmt.Errorf( + "HTTP_PORT in captive core config file: %d does not match Horizon captive-core-http-port flag: %d", + c.HTTPPort, + *params.HTTPPort, + ) + } + + if def := c.metadata.IsDefined("PEER_PORT"); def && params.PeerPort != nil && c.PeerPort != *params.PeerPort { + return fmt.Errorf( + "PEER_PORT in captive core config file: %d does not match Horizon captive-core-peer-port flag: %d", + c.PeerPort, + *params.PeerPort, + ) + } + + if def := c.metadata.IsDefined("LOG_FILE_PATH"); def && params.LogPath != nil && c.LogFilePath != *params.LogPath { + return fmt.Errorf( + "LOG_FILE_PATH in captive core config file: %s does not match Horizon captive-core-log-path flag: %s", + c.LogFilePath, + *params.LogPath, + ) + } + + homeDomainSet := map[string]HomeDomain{} + for _, hd := range c.HomeDomains { + if _, ok := homeDomainSet[hd.HomeDomain]; ok { + return fmt.Errorf( + "found duplicate home domain in captive core configuration: %s", + hd.HomeDomain, + ) + } + if hd.HomeDomain == "" { + return fmt.Errorf( + "found invalid home domain entry which is missing a HOME_DOMAIN value: %s", + hd.HomeDomain, + ) + } + if hd.Quality == "" { + return fmt.Errorf( + "found invalid home domain entry which is missing a QUALITY value: %s", + hd.HomeDomain, + ) + } + if !validQuality[hd.Quality] { + return fmt.Errorf( + "found invalid home domain entry which has an invalid QUALITY value: %s", + hd.HomeDomain, + ) + } + homeDomainSet[hd.HomeDomain] = hd + } + + names := map[string]bool{} + for _, v := range c.Validators { + if names[v.Name] { + return fmt.Errorf( + "found duplicate validator in captive core configuration: %s", + v.Name, + ) + } + if v.Name == "" { + return fmt.Errorf( + "found invalid validator entry which is missing a NAME value: %s", + v.Name, + ) + } + if v.HomeDomain == "" { + return fmt.Errorf( + "found invalid validator entry which is missing a HOME_DOMAIN value: %s", + v.Name, + ) + } + if v.PublicKey == "" { + return fmt.Errorf( + "found invalid validator entry which is missing a PUBLIC_KEY value: %s", + v.Name, + ) + } + if _, err := xdr.AddressToAccountId(v.PublicKey); err != nil { + return fmt.Errorf( + "found invalid validator entry which has an invalid PUBLIC_KEY : %s", + v.Name, + ) + } + if v.Quality == "" { + if _, ok := homeDomainSet[v.HomeDomain]; !ok { + return fmt.Errorf( + "found invalid validator entry which is missing a QUALITY value: %s", + v.Name, + ) + } + } else if !validQuality[v.Quality] { + return fmt.Errorf( + "found invalid validator entry which has an invalid QUALITY value: %s", + v.Name, + ) + } + + names[v.Name] = true + } + + return nil +} diff --git a/ingest/tutorial/example_common.go b/ingest/tutorial/example_common.go index b10829352b..35c13b5c55 100644 --- a/ingest/tutorial/example_common.go +++ b/ingest/tutorial/example_common.go @@ -9,8 +9,8 @@ import ( var ( config = ledgerbackend.CaptiveCoreConfig{ // Change these based on your environment: - BinaryPath: "/usr/local/bin/stellar-core", - ConfigAppendPath: "stellar-core-stub.toml", + BinaryPath: "/usr/local/bin/stellar-core", + // TODO configure TOML NetworkPassphrase: "Test SDF Network ; September 2015", HistoryArchiveURLs: []string{ "https://history.stellar.org/prd/core-testnet/core_testnet_001", diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 640a94e742..a360e806a3 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -249,7 +249,10 @@ func RunDBReingestRange(from, to uint32, reingestForce bool, parallelWorkers uin EnableCaptiveCore: config.EnableCaptiveCoreIngestion, CaptiveCoreBinaryPath: config.CaptiveCoreBinaryPath, RemoteCaptiveCoreURL: config.RemoteCaptiveCoreURL, - CaptiveCoreConfigAppendPath: config.CaptiveCoreConfigAppendPath, + CaptiveCoreToml: config.CaptiveCoreToml, + CaptiveCoreStoragePath: config.CaptiveCoreStoragePath, + StellarCoreCursor: config.CursorName, + StellarCoreURL: config.StellarCoreURL, } if !ingestConfig.EnableCaptiveCore { diff --git a/services/horizon/internal/config.go b/services/horizon/internal/config.go index 2bca813e91..8849fd6ac7 100644 --- a/services/horizon/internal/config.go +++ b/services/horizon/internal/config.go @@ -1,6 +1,7 @@ package horizon import ( + "github.com/stellar/go/ingest/ledgerbackend" "net/url" "time" @@ -16,14 +17,12 @@ type Config struct { Port uint AdminPort uint - EnableCaptiveCoreIngestion bool - CaptiveCoreBinaryPath string - CaptiveCoreConfigAppendPath string - RemoteCaptiveCoreURL string - CaptiveCoreHTTPPort uint - CaptiveCorePeerPort uint - CaptiveCoreLogPath string - CaptiveCoreStoragePath string + EnableCaptiveCoreIngestion bool + CaptiveCoreBinaryPath string + RemoteCaptiveCoreURL string + CaptiveCoreTomlParams ledgerbackend.CaptiveCoreTomlParams + CaptiveCoreToml *ledgerbackend.CaptiveCoreToml + CaptiveCoreStoragePath string StellarCoreDatabaseURL string StellarCoreURL string diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 491be01803..ef5833db4f 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -2,6 +2,7 @@ package horizon import ( "fmt" + "github.com/stellar/go/ingest/ledgerbackend" "go/types" stdLog "log" "os" @@ -23,7 +24,7 @@ const ( DatabaseURLFlagName = "db-url" // StellarCoreDBURLFlagName is the command line flag for configuring the postgres Stellar Core URL StellarCoreDBURLFlagName = "stellar-core-db-url" - // StellarCoreDBURLFlagName is the command line flag for configuring the URL fore Stellar Core HTTP endpoint + // StellarCoreURLFlagName is the command line flag for configuring the URL fore Stellar Core HTTP endpoint StellarCoreURLFlagName = "stellar-core-url" // StellarCoreBinaryPathName is the command line flag for configuring the path to the stellar core binary StellarCoreBinaryPathName = "stellar-core-binary-path" @@ -115,7 +116,7 @@ func Flags() (*Config, support.ConfigOptions) { FlagDefault: "", Required: false, Usage: "path to additional configuration for the Stellar Core configuration file used by captive core. It must, at least, include enough details to define a quorum set", - ConfigKey: &config.CaptiveCoreConfigAppendPath, + ConfigKey: &config.CaptiveCoreTomlParams.ConfigPath, }, &support.ConfigOption{ Name: "enable-captive-core-ingestion", @@ -126,17 +127,17 @@ func Flags() (*Config, support.ConfigOptions) { ConfigKey: &config.EnableCaptiveCoreIngestion, }, &support.ConfigOption{ - Name: "captive-core-http-port", - OptType: types.Uint, - FlagDefault: uint(11626), - Required: false, - Usage: "HTTP port for Captive Core to listen on (0 disables the HTTP server)", - ConfigKey: &config.CaptiveCoreHTTPPort, + Name: "captive-core-http-port", + OptType: types.Uint, + CustomSetValue: support.SetOptionalUint, + Required: false, + FlagDefault: uint(0), + Usage: "HTTP port for Captive Core to listen on (0 disables the HTTP server)", + ConfigKey: &config.CaptiveCoreTomlParams.HTTPPort, }, &support.ConfigOption{ - Name: "captive-core-storage-path", - OptType: types.String, - FlagDefault: "", + Name: "captive-core-storage-path", + OptType: types.String, CustomSetValue: func(opt *support.ConfigOption) { existingValue := viper.GetString(opt.Name) if existingValue == "" || existingValue == "." { @@ -153,12 +154,13 @@ func Flags() (*Config, support.ConfigOptions) { ConfigKey: &config.CaptiveCoreStoragePath, }, &support.ConfigOption{ - Name: "captive-core-peer-port", - OptType: types.Uint, - FlagDefault: uint(0), - Required: false, - Usage: "port for Captive Core to bind to for connecting to the Stellar swarm (0 uses Stellar Core's default)", - ConfigKey: &config.CaptiveCorePeerPort, + Name: "captive-core-peer-port", + OptType: types.Uint, + FlagDefault: uint(0), + CustomSetValue: support.SetOptionalUint, + Required: false, + Usage: "port for Captive Core to bind to for connecting to the Stellar swarm (0 uses Stellar Core's default)", + ConfigKey: &config.CaptiveCoreTomlParams.PeerPort, }, &support.ConfigOption{ Name: StellarCoreDBURLFlagName, @@ -285,10 +287,12 @@ func Flags() (*Config, support.ConfigOptions) { Usage: "name of the file where logs will be saved (leave empty to send logs to stdout)", }, &support.ConfigOption{ - Name: "captive-core-log-path", - ConfigKey: &config.CaptiveCoreLogPath, - OptType: types.String, - Usage: "name of the path for Core logs (leave empty to log w/ Horizon only)", + Name: "captive-core-log-path", + ConfigKey: &config.CaptiveCoreTomlParams.LogPath, + OptType: types.String, + CustomSetValue: support.SetOptionalString, + Required: false, + Usage: "name of the path for Core logs (leave empty to log w/ Horizon only)", }, &support.ConfigOption{ Name: "max-path-length", @@ -480,20 +484,30 @@ func ApplyFlags(config *Config, flags support.ConfigOptions) { StellarCoreBinaryPathName, captiveCoreMigrationHint) } - if binaryPath == "" || config.CaptiveCoreConfigAppendPath == "" { + if config.RemoteCaptiveCoreURL == "" && (binaryPath == "" || config.CaptiveCoreTomlParams.ConfigPath == "") { stdLog.Fatalf("Invalid config: captive core requires that both --%s and --%s are set. %s", StellarCoreBinaryPathName, CaptiveCoreConfigAppendPathName, captiveCoreMigrationHint) } + if config.RemoteCaptiveCoreURL == "" { + var err error + config.CaptiveCoreTomlParams.HistoryArchiveURLs = config.HistoryArchiveURLs + config.CaptiveCoreTomlParams.NetworkPassphrase = config.NetworkPassphrase + config.CaptiveCoreToml, err = ledgerbackend.NewCaptiveCoreToml(config.CaptiveCoreTomlParams) + if err != nil { + stdLog.Fatalf("Invalid captive core toml file %v", err) + } + } + // If we don't supply an explicit core URL and we are running a local // captive core process with the http port enabled, point to it. - if config.StellarCoreURL == "" && config.RemoteCaptiveCoreURL == "" && config.CaptiveCoreHTTPPort != 0 { - config.StellarCoreURL = fmt.Sprintf("http://localhost:%d", config.CaptiveCoreHTTPPort) + if config.StellarCoreURL == "" && config.RemoteCaptiveCoreURL == "" && config.CaptiveCoreToml.HTTPPort != 0 { + config.StellarCoreURL = fmt.Sprintf("http://localhost:%d", config.CaptiveCoreToml.HTTPPort) viper.Set(StellarCoreURLFlagName, config.StellarCoreURL) } } } else { - if config.EnableCaptiveCoreIngestion && (config.CaptiveCoreBinaryPath != "" || config.CaptiveCoreConfigAppendPath != "") { + if config.EnableCaptiveCoreIngestion && (config.CaptiveCoreBinaryPath != "" || config.CaptiveCoreTomlParams.ConfigPath != "") { stdLog.Fatalf("Invalid config: one or more captive core params passed (--%s or --%s) but --ingest not set. "+captiveCoreMigrationHint, StellarCoreBinaryPathName, CaptiveCoreConfigAppendPathName) } diff --git a/services/horizon/internal/ingest/main.go b/services/horizon/internal/ingest/main.go index 79feb9a44e..ba505756bd 100644 --- a/services/horizon/internal/ingest/main.go +++ b/services/horizon/internal/ingest/main.go @@ -66,18 +66,15 @@ const ( var log = logpkg.DefaultLogger.WithField("service", "ingest") type Config struct { - CoreSession *db.Session - StellarCoreURL string - StellarCoreCursor string - EnableCaptiveCore bool - CaptiveCoreBinaryPath string - CaptiveCoreStoragePath string - CaptiveCoreConfigAppendPath string - CaptiveCoreHTTPPort uint - CaptiveCorePeerPort uint - CaptiveCoreLogPath string - RemoteCaptiveCoreURL string - NetworkPassphrase string + CoreSession *db.Session + StellarCoreURL string + StellarCoreCursor string + EnableCaptiveCore bool + CaptiveCoreBinaryPath string + CaptiveCoreStoragePath string + CaptiveCoreToml *ledgerbackend.CaptiveCoreToml + RemoteCaptiveCoreURL string + NetworkPassphrase string HistorySession *db.Session HistoryArchiveURL string @@ -197,12 +194,9 @@ func NewSystem(config Config) (System, error) { logger := log.WithField("subservice", "stellar-core") ledgerBackend, err = ledgerbackend.NewCaptive( ledgerbackend.CaptiveCoreConfig{ - LogPath: config.CaptiveCoreLogPath, BinaryPath: config.CaptiveCoreBinaryPath, StoragePath: config.CaptiveCoreStoragePath, - ConfigAppendPath: config.CaptiveCoreConfigAppendPath, - HTTPPort: config.CaptiveCoreHTTPPort, - PeerPort: config.CaptiveCorePeerPort, + Toml: config.CaptiveCoreToml, NetworkPassphrase: config.NetworkPassphrase, HistoryArchiveURLs: []string{config.HistoryArchiveURL}, CheckpointFrequency: config.CheckpointFrequency, @@ -314,7 +308,7 @@ func (s *system) initMetrics() { Help: "1 if sync, 0 if not synced, -1 if unable to connect or HTTP server disabled.", }, func() float64 { - if !s.config.EnableCaptiveCore || s.config.CaptiveCoreHTTPPort == 0 { + if !s.config.EnableCaptiveCore || (s.config.CaptiveCoreToml.HTTPPort == 0) { return -1 } @@ -322,7 +316,7 @@ func (s *system) initMetrics() { HTTP: &http.Client{ Timeout: 2 * time.Second, }, - URL: fmt.Sprintf("http://localhost:%d", s.config.CaptiveCoreHTTPPort), + URL: fmt.Sprintf("http://localhost:%d", s.config.CaptiveCoreToml.HTTPPort), } info, err := client.Info(s.ctx) diff --git a/services/horizon/internal/init.go b/services/horizon/internal/init.go index db6a87573b..1e61809e73 100644 --- a/services/horizon/internal/init.go +++ b/services/horizon/internal/init.go @@ -65,19 +65,16 @@ func initIngester(app *App) { // TODO: // Use the first archive for now. We don't have a mechanism to // use multiple archives at the same time currently. - HistoryArchiveURL: app.config.HistoryArchiveURLs[0], - CheckpointFrequency: app.config.CheckpointFrequency, - StellarCoreURL: app.config.StellarCoreURL, - StellarCoreCursor: app.config.CursorName, - CaptiveCoreBinaryPath: app.config.CaptiveCoreBinaryPath, - CaptiveCoreStoragePath: app.config.CaptiveCoreStoragePath, - CaptiveCoreConfigAppendPath: app.config.CaptiveCoreConfigAppendPath, - CaptiveCoreHTTPPort: app.config.CaptiveCoreHTTPPort, - CaptiveCorePeerPort: app.config.CaptiveCorePeerPort, - CaptiveCoreLogPath: app.config.CaptiveCoreLogPath, - RemoteCaptiveCoreURL: app.config.RemoteCaptiveCoreURL, - EnableCaptiveCore: app.config.EnableCaptiveCoreIngestion, - DisableStateVerification: app.config.IngestDisableStateVerification, + HistoryArchiveURL: app.config.HistoryArchiveURLs[0], + CheckpointFrequency: app.config.CheckpointFrequency, + StellarCoreURL: app.config.StellarCoreURL, + StellarCoreCursor: app.config.CursorName, + CaptiveCoreBinaryPath: app.config.CaptiveCoreBinaryPath, + CaptiveCoreStoragePath: app.config.CaptiveCoreStoragePath, + CaptiveCoreToml: app.config.CaptiveCoreToml, + RemoteCaptiveCoreURL: app.config.RemoteCaptiveCoreURL, + EnableCaptiveCore: app.config.EnableCaptiveCoreIngestion, + DisableStateVerification: app.config.IngestDisableStateVerification, }) if err != nil { diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index f7163e0413..891f6637d4 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -81,8 +81,8 @@ func TestReingestDB(t *testing.T) { toLedger = latestCheckpoint } - horizonConfig.CaptiveCoreConfigAppendPath = filepath.Join( - filepath.Dir(horizonConfig.CaptiveCoreConfigAppendPath), + horizonConfig.CaptiveCoreTomlParams.ConfigPath = filepath.Join( + filepath.Dir(horizonConfig.CaptiveCoreTomlParams.ConfigPath), "captive-core-reingest-range-integration-tests.cfg", ) diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go index 3e4f02a8c5..ecb7627c5d 100644 --- a/services/horizon/internal/test/integration/integration.go +++ b/services/horizon/internal/test/integration/integration.go @@ -142,7 +142,7 @@ func (i *Test) RestartHorizon() { i.startHorizon( i.horizonConfig.CaptiveCoreBinaryPath, - i.horizonConfig.CaptiveCoreConfigAppendPath, + i.horizonConfig.CaptiveCoreTomlParams.ConfigPath, i.horizonConfig.DatabaseURL, false, ) diff --git a/support/config/config_option.go b/support/config/config_option.go index c152c5059d..7a8300966d 100644 --- a/support/config/config_option.go +++ b/support/config/config_option.go @@ -161,3 +161,27 @@ func SetURL(co *ConfigOption) { *(co.ConfigKey.(**url.URL)) = urlType } } + +// SetOptionalUint converts a command line uint to a *uint where the nil +// value indicates the flag was not explicitly set +func SetOptionalUint(co *ConfigOption) { + key := co.ConfigKey.(**uint) + if co.flag.Changed { + *key = new(uint) + **key = uint(viper.GetInt(co.Name)) + } else { + *key = nil + } +} + +// SetOptionalString converts a command line uint to a *string where the nil +// value indicates the flag was not explicitly set +func SetOptionalString(co *ConfigOption) { + key := co.ConfigKey.(**string) + if co.flag.Changed { + *key = new(string) + **key = viper.GetString(co.Name) + } else { + *key = nil + } +} diff --git a/support/config/config_option_test.go b/support/config/config_option_test.go index a7853b0dbd..35eabe358c 100644 --- a/support/config/config_option_test.go +++ b/support/config/config_option_test.go @@ -52,6 +52,9 @@ func TestConfigOption_getSimpleValue_defaults(t *testing.T) { assert.Equal(t, true, opts.Bool) assert.Equal(t, uint(2), opts.Uint) assert.Equal(t, uint32(3), opts.Uint32) + for _, opt := range configOpts { + assert.False(t, opt.flag.Changed) + } } // Test that when args are given, their values are used. @@ -86,6 +89,9 @@ func TestConfigOption_getSimpleValue_setFlag(t *testing.T) { assert.Equal(t, true, opts.Bool) assert.Equal(t, uint(20), opts.Uint) assert.Equal(t, uint32(30), opts.Uint32) + for _, opt := range configOpts { + assert.True(t, opt.flag.Changed) + } } // Test that when args are not given but env vars are, their values are used.