diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 37a166533f..59ffc8701c 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -321,7 +321,7 @@ var dbReingestRangeCmd = &cobra.Command{ } } - err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}) + err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}) if err != nil { return err } @@ -369,7 +369,7 @@ var dbFillGapsCmd = &cobra.Command{ withRange = true } - err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}) + err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}) if err != nil { return err } diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go index db9eccea30..42a62449d8 100644 --- a/services/horizon/cmd/ingest.go +++ b/services/horizon/cmd/ingest.go @@ -94,7 +94,7 @@ var ingestVerifyRangeCmd = &cobra.Command{ co.SetValue() } - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { return err } @@ -203,7 +203,7 @@ var ingestStressTestCmd = &cobra.Command{ co.SetValue() } - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { return err } @@ -267,7 +267,7 @@ var ingestTriggerStateRebuildCmd = &cobra.Command{ Short: "updates a database to trigger state rebuild, state will be rebuilt by a running Horizon instance, DO NOT RUN production DB, some endpoints will be unavailable until state is rebuilt", RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { return err } @@ -291,7 +291,7 @@ var ingestInitGenesisStateCmd = &cobra.Command{ Short: "ingests genesis state (ledger 1)", RunE: func(cmd *cobra.Command, args []string) error { ctx := context.Background() - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { return err } @@ -363,7 +363,7 @@ var ingestBuildStateCmd = &cobra.Command{ co.SetValue() } - if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreConfig: false, AlwaysIngest: true}); err != nil { + if err := horizon.ApplyFlags(config, flags, horizon.ApplyOptions{RequireCaptiveCoreFullConfig: false, AlwaysIngest: true}); err != nil { return err } diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 25fda452f4..aed3483375 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -642,7 +642,7 @@ func Flags() (*Config, support.ConfigOptions) { // NewAppFromFlags constructs a new Horizon App from the given command line flags func NewAppFromFlags(config *Config, flags support.ConfigOptions) (*App, error) { - err := ApplyFlags(config, flags, ApplyOptions{RequireCaptiveCoreConfig: true, AlwaysIngest: false}) + err := ApplyFlags(config, flags, ApplyOptions{RequireCaptiveCoreFullConfig: true, AlwaysIngest: false}) if err != nil { return nil, err } @@ -663,8 +663,8 @@ func NewAppFromFlags(config *Config, flags support.ConfigOptions) (*App, error) } type ApplyOptions struct { - AlwaysIngest bool - RequireCaptiveCoreConfig bool + AlwaysIngest bool + RequireCaptiveCoreFullConfig bool } type networkConfig struct { @@ -703,97 +703,30 @@ func getCaptiveCoreBinaryPath() (string, error) { return result, nil } -// loadDefaultCaptiveCoreToml loads the default Captive Core TOML based on the default config data. -func loadDefaultCaptiveCoreToml(config *Config, defaultConfigData []byte) error { - - config.CaptiveCoreTomlParams.CoreBinaryPath = config.CaptiveCoreBinaryPath - config.CaptiveCoreTomlParams.HistoryArchiveURLs = config.HistoryArchiveURLs - config.CaptiveCoreTomlParams.NetworkPassphrase = config.NetworkPassphrase - - var err error - config.CaptiveCoreToml, err = ledgerbackend.NewCaptiveCoreTomlFromData(defaultConfigData, config.CaptiveCoreTomlParams) - if err != nil { - return errors.Wrap(err, "invalid captive core toml") - } - return nil -} - -// loadCaptiveCoreTomlFromFile loads the Captive Core TOML file from the specified path provided config. -func loadCaptiveCoreTomlFromFile(config *Config) error { - var err error - - config.CaptiveCoreTomlParams.CoreBinaryPath = config.CaptiveCoreBinaryPath - config.CaptiveCoreTomlParams.HistoryArchiveURLs = config.HistoryArchiveURLs - config.CaptiveCoreTomlParams.NetworkPassphrase = config.NetworkPassphrase - - config.CaptiveCoreToml, err = ledgerbackend.NewCaptiveCoreTomlFromFile(config.CaptiveCoreConfigPath, config.CaptiveCoreTomlParams) - if err != nil { - return errors.Wrap(err, "invalid captive core toml file") - } - return nil -} - -// createCaptiveCoreConfigFromNetwork generates the default Captive Core configuration. -// validates the configuration settings, sets default values, and loads the Captive Core TOML file. -func createCaptiveCoreConfigFromNetwork(config *Config) error { +// getCaptiveCoreConfigFromNetworkParameter returns the default Captive Core configuration based on the network. +func getCaptiveCoreConfigFromNetworkParameter(config *Config) (networkConfig, error) { + var defaultNetworkConfig networkConfig if config.NetworkPassphrase != "" { - return fmt.Errorf("invalid config: %s parameter not allowed with the %s parameter", NetworkPassphraseFlagName, NetworkFlagName) + return defaultNetworkConfig, fmt.Errorf("invalid config: %s parameter not allowed with the %s parameter", + NetworkPassphraseFlagName, NetworkFlagName) } if len(config.HistoryArchiveURLs) > 0 { - return fmt.Errorf("invalid config: %s parameter not allowed with the %s parameter", HistoryArchiveURLsFlagName, NetworkFlagName) + return defaultNetworkConfig, fmt.Errorf("invalid config: %s parameter not allowed with the %s parameter", + HistoryArchiveURLsFlagName, NetworkFlagName) } - var defaultNetworkConfig networkConfig switch config.Network { case StellarPubnet: defaultNetworkConfig = PubnetConf case StellarTestnet: defaultNetworkConfig = TestnetConf default: - return fmt.Errorf("no default configuration found for network %s", config.Network) - } - config.NetworkPassphrase = defaultNetworkConfig.NetworkPassphrase - config.HistoryArchiveURLs = defaultNetworkConfig.HistoryArchiveURLs - - if config.CaptiveCoreConfigPath == "" { - return loadDefaultCaptiveCoreToml(config, defaultNetworkConfig.defaultConfig) - } - - return loadCaptiveCoreTomlFromFile(config) -} - -// createCaptiveCoreConfigFromParameters generates the Captive Core configuration. -// validates the configuration settings, sets necessary values, and loads the Captive Core TOML file. -func createCaptiveCoreConfigFromParameters(config *Config, options ApplyOptions) error { - - if config.NetworkPassphrase == "" { - return fmt.Errorf("%s must be set", NetworkPassphraseFlagName) - } - - if len(config.HistoryArchiveURLs) == 0 { - return fmt.Errorf("%s must be set", HistoryArchiveURLsFlagName) - } - - if config.CaptiveCoreConfigPath != "" { - return loadCaptiveCoreTomlFromFile(config) - } else if options.RequireCaptiveCoreConfig { - return fmt.Errorf( - "invalid config: captive core requires that --%s is set", CaptiveCoreConfigPathName) - } else { - config.CaptiveCoreTomlParams.CoreBinaryPath = config.CaptiveCoreBinaryPath - config.CaptiveCoreTomlParams.HistoryArchiveURLs = config.HistoryArchiveURLs - config.CaptiveCoreTomlParams.NetworkPassphrase = config.NetworkPassphrase - - var err error - config.CaptiveCoreToml, err = ledgerbackend.NewCaptiveCoreToml(config.CaptiveCoreTomlParams) - if err != nil { - return errors.Wrap(err, "invalid captive core toml file") - } + return defaultNetworkConfig, fmt.Errorf("no default configuration found for network %s", config.Network) } - return nil + return defaultNetworkConfig, nil } // setCaptiveCoreConfiguration prepares configuration for the Captive Core @@ -809,16 +742,52 @@ func setCaptiveCoreConfiguration(config *Config, options ApplyOptions) error { } } + var defaultNetworkConfig networkConfig if config.Network != "" { - err := createCaptiveCoreConfigFromNetwork(config) + var err error + defaultNetworkConfig, err = getCaptiveCoreConfigFromNetworkParameter(config) if err != nil { return err } + config.NetworkPassphrase = defaultNetworkConfig.NetworkPassphrase + config.HistoryArchiveURLs = defaultNetworkConfig.HistoryArchiveURLs } else { - err := createCaptiveCoreConfigFromParameters(config, options) + if config.NetworkPassphrase == "" { + return fmt.Errorf("%s must be set", NetworkPassphraseFlagName) + } + + if len(config.HistoryArchiveURLs) == 0 { + return fmt.Errorf("%s must be set", HistoryArchiveURLsFlagName) + } + } + + config.CaptiveCoreTomlParams.CoreBinaryPath = config.CaptiveCoreBinaryPath + config.CaptiveCoreTomlParams.HistoryArchiveURLs = config.HistoryArchiveURLs + config.CaptiveCoreTomlParams.NetworkPassphrase = config.NetworkPassphrase + + var err error + if config.CaptiveCoreConfigPath != "" { + config.CaptiveCoreToml, err = ledgerbackend.NewCaptiveCoreTomlFromFile(config.CaptiveCoreConfigPath, + config.CaptiveCoreTomlParams) if err != nil { - return err + return errors.Wrap(err, "invalid captive core toml file") } + } else if !options.RequireCaptiveCoreFullConfig { + // Creates a minimal captive-core config (without quorum information), just enough to run captive core. + // This is used by certain database commands, such as `reingest and fill-gaps, to reingest historical data. + config.CaptiveCoreToml, err = ledgerbackend.NewCaptiveCoreToml(config.CaptiveCoreTomlParams) + if err != nil { + return errors.Wrap(err, "invalid captive core toml file") + } + } else if len(defaultNetworkConfig.defaultConfig) != 0 { + config.CaptiveCoreToml, err = ledgerbackend.NewCaptiveCoreTomlFromData(defaultNetworkConfig.defaultConfig, + config.CaptiveCoreTomlParams) + if err != nil { + return errors.Wrap(err, "invalid captive core toml file") + } + } else { + return fmt.Errorf("invalid config: captive core requires that --%s is set or you can set the --%s "+ + "parameter to use the default captive core config", CaptiveCoreConfigPathName, NetworkFlagName) } // If we don't supply an explicit core URL and running captive core process with the http port enabled, diff --git a/services/horizon/internal/flags_test.go b/services/horizon/internal/flags_test.go index 42c16cea76..c9b6c94bd1 100644 --- a/services/horizon/internal/flags_test.go +++ b/services/horizon/internal/flags_test.go @@ -19,63 +19,75 @@ func Test_createCaptiveCoreDefaultConfig(t *testing.T) { errStr string }{ { - name: "testnet default config", - config: Config{Network: StellarTestnet}, + name: "testnet default config", + config: Config{Network: StellarTestnet, + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", + }, networkPassphrase: TestnetConf.NetworkPassphrase, historyArchiveURLs: TestnetConf.HistoryArchiveURLs, }, { - name: "pubnet default config", - config: Config{Network: StellarPubnet}, + name: "pubnet default config", + config: Config{Network: StellarPubnet, + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", + }, networkPassphrase: PubnetConf.NetworkPassphrase, historyArchiveURLs: PubnetConf.HistoryArchiveURLs, }, { name: "testnet validation; history archive urls supplied", config: Config{Network: StellarTestnet, - HistoryArchiveURLs: []string{"network history archive urls supplied"}, + HistoryArchiveURLs: []string{"network history archive urls supplied"}, + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", }, errStr: fmt.Sprintf(errorMsgDefaultConfig, HistoryArchiveURLsFlagName), }, { name: "pubnet validation; history archive urls supplied", config: Config{Network: StellarPubnet, - HistoryArchiveURLs: []string{"network history archive urls supplied"}, + HistoryArchiveURLs: []string{"network history archive urls supplied"}, + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", }, errStr: fmt.Sprintf(errorMsgDefaultConfig, HistoryArchiveURLsFlagName), }, { name: "testnet validation; network passphrase supplied", config: Config{Network: StellarTestnet, - NetworkPassphrase: "network passphrase supplied", - HistoryArchiveURLs: []string{}, + NetworkPassphrase: "network passphrase supplied", + HistoryArchiveURLs: []string{}, + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", }, errStr: fmt.Sprintf(errorMsgDefaultConfig, NetworkPassphraseFlagName), }, { name: "pubnet validation; network passphrase supplied", config: Config{Network: StellarPubnet, - NetworkPassphrase: "pubnet network passphrase supplied", - HistoryArchiveURLs: []string{}, + NetworkPassphrase: "pubnet network passphrase supplied", + HistoryArchiveURLs: []string{}, + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", }, errStr: fmt.Sprintf(errorMsgDefaultConfig, NetworkPassphraseFlagName), }, { name: "unknown network specified", config: Config{Network: "unknown", - NetworkPassphrase: "", - HistoryArchiveURLs: []string{}, + NetworkPassphrase: "", + HistoryArchiveURLs: []string{}, + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", }, errStr: "no default configuration found for network unknown", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - e := createCaptiveCoreConfigFromNetwork(&tt.config) + e := setCaptiveCoreConfiguration(&tt.config, + ApplyOptions{RequireCaptiveCoreFullConfig: true}) if tt.errStr == "" { assert.NoError(t, e) assert.Equal(t, tt.networkPassphrase, tt.config.NetworkPassphrase) assert.Equal(t, tt.historyArchiveURLs, tt.config.HistoryArchiveURLs) + assert.Equal(t, tt.networkPassphrase, tt.config.CaptiveCoreTomlParams.NetworkPassphrase) + assert.Equal(t, tt.historyArchiveURLs, tt.config.CaptiveCoreTomlParams.HistoryArchiveURLs) } else { assert.Equal(t, tt.errStr, e.Error()) } @@ -101,6 +113,7 @@ func Test_createCaptiveCoreConfig(t *testing.T) { NetworkPassphrase: PubnetConf.NetworkPassphrase, HistoryArchiveURLs: PubnetConf.HistoryArchiveURLs, CaptiveCoreConfigPath: "configs/captive-core-pubnet.cfg", + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", }, networkPassphrase: PubnetConf.NetworkPassphrase, historyArchiveURLs: PubnetConf.HistoryArchiveURLs, @@ -109,7 +122,8 @@ func Test_createCaptiveCoreConfig(t *testing.T) { name: "no network specified; passphrase not supplied", requireCaptiveCoreConfig: true, config: Config{ - HistoryArchiveURLs: []string{"HistoryArchiveURLs"}, + HistoryArchiveURLs: []string{"HistoryArchiveURLs"}, + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", }, errStr: fmt.Sprintf(errorMsgConfig, NetworkPassphraseFlagName), }, @@ -117,7 +131,8 @@ func Test_createCaptiveCoreConfig(t *testing.T) { name: "no network specified; history archive urls not supplied", requireCaptiveCoreConfig: true, config: Config{ - NetworkPassphrase: "NetworkPassphrase", + NetworkPassphrase: "NetworkPassphrase", + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", }, errStr: fmt.Sprintf(errorMsgConfig, HistoryArchiveURLsFlagName), }, @@ -125,11 +140,12 @@ func Test_createCaptiveCoreConfig(t *testing.T) { name: "no network specified; captive-core-config-path not supplied", requireCaptiveCoreConfig: true, config: Config{ - NetworkPassphrase: PubnetConf.NetworkPassphrase, - HistoryArchiveURLs: PubnetConf.HistoryArchiveURLs, + NetworkPassphrase: PubnetConf.NetworkPassphrase, + HistoryArchiveURLs: PubnetConf.HistoryArchiveURLs, + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", }, - errStr: fmt.Sprintf("invalid config: captive core requires that --%s is set", - CaptiveCoreConfigPathName), + errStr: fmt.Sprintf("invalid config: captive core requires that --%s is set or "+ + "you can set the --%s parameter to use the default captive core config", CaptiveCoreConfigPathName, NetworkFlagName), }, { name: "no network specified; captive-core-config-path invalid file", @@ -138,6 +154,7 @@ func Test_createCaptiveCoreConfig(t *testing.T) { NetworkPassphrase: PubnetConf.NetworkPassphrase, HistoryArchiveURLs: PubnetConf.HistoryArchiveURLs, CaptiveCoreConfigPath: "xyz.cfg", + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", }, errStr: "invalid captive core toml file: could not load toml path:" + " open xyz.cfg: no such file or directory", @@ -149,6 +166,7 @@ func Test_createCaptiveCoreConfig(t *testing.T) { NetworkPassphrase: PubnetConf.NetworkPassphrase, HistoryArchiveURLs: PubnetConf.HistoryArchiveURLs, CaptiveCoreConfigPath: "configs/captive-core-testnet.cfg", + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", }, errStr: fmt.Sprintf("invalid captive core toml file: invalid captive core toml: "+ "NETWORK_PASSPHRASE in captive core config file: %s does not match Horizon "+ @@ -158,8 +176,9 @@ func Test_createCaptiveCoreConfig(t *testing.T) { name: "no network specified; captive-core-config not required", requireCaptiveCoreConfig: false, config: Config{ - NetworkPassphrase: PubnetConf.NetworkPassphrase, - HistoryArchiveURLs: PubnetConf.HistoryArchiveURLs, + NetworkPassphrase: PubnetConf.NetworkPassphrase, + HistoryArchiveURLs: PubnetConf.HistoryArchiveURLs, + CaptiveCoreBinaryPath: "/path/to/captive-core/binary", }, networkPassphrase: PubnetConf.NetworkPassphrase, historyArchiveURLs: PubnetConf.HistoryArchiveURLs, @@ -167,10 +186,12 @@ func Test_createCaptiveCoreConfig(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - e := createCaptiveCoreConfigFromParameters(&tt.config, - ApplyOptions{RequireCaptiveCoreConfig: tt.requireCaptiveCoreConfig}) + e := setCaptiveCoreConfiguration(&tt.config, + ApplyOptions{RequireCaptiveCoreFullConfig: tt.requireCaptiveCoreConfig}) if tt.errStr == "" { assert.NoError(t, e) + assert.Equal(t, tt.networkPassphrase, tt.config.NetworkPassphrase) + assert.Equal(t, tt.historyArchiveURLs, tt.config.HistoryArchiveURLs) assert.Equal(t, tt.networkPassphrase, tt.config.CaptiveCoreTomlParams.NetworkPassphrase) assert.Equal(t, tt.historyArchiveURLs, tt.config.CaptiveCoreTomlParams.HistoryArchiveURLs) } else {