diff --git a/services/horizon/main.go b/services/horizon/main.go index b6da1ce9b6..fa62e48fc0 100644 --- a/services/horizon/main.go +++ b/services/horizon/main.go @@ -2,154 +2,22 @@ package main import ( stdLog "log" - "net/url" "os" - "time" - "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" "github.com/stellar/go/network" horizon "github.com/stellar/go/services/horizon/internal" "github.com/stellar/go/services/horizon/internal/db2/schema" apkg "github.com/stellar/go/support/app" + support "github.com/stellar/go/support/config" "github.com/stellar/go/support/log" - "github.com/stellar/go/support/strutils" - "github.com/throttled/throttled" ) var app *horizon.App var config horizon.Config -var rootCmd *cobra.Command - -// flagType implements a generic interface for the different command line flags, -// allowing them to be configured in a uniform way. -type flagType func(name string, value interface{}, usage string) interface{} - -var ( - stringFlag flagType = func(name string, value interface{}, usage string) interface{} { - return rootCmd.PersistentFlags().String(name, value.(string), usage) - } - intFlag flagType = func(name string, value interface{}, usage string) interface{} { - return rootCmd.PersistentFlags().Int(name, value.(int), usage) - } - uintFlag flagType = func(name string, value interface{}, usage string) interface{} { - return rootCmd.PersistentFlags().Uint(name, value.(uint), usage) - } - boolFlag flagType = func(name string, value interface{}, usage string) interface{} { - return rootCmd.PersistentFlags().Bool(name, value.(bool), usage) - } -) - -// configOption is a complete description of the configuration of a command line option -type configOption struct { - name string // e.g. "db-url" - envVar string // e.g. "DATABASE_URL". Defaults to uppercase/underscore representation of name - flagDefault interface{} // A default if no option is provided. Set to "" if no default - required bool // Whether this option must be set for Horizon to run - usage string // Help text - customSetValue func(*configOption) // Optional function for custom validation/transformation - configKey interface{} // Pointer to the final key in the horizon.Config struct -} - -// init handles initialisation steps, including configuring and binding the env variable name. -func (co *configOption) init() { - // Bind the command line and environment variable name - // Unless overriden, default to a transform like tls-key -> TLS_KEY - if co.envVar == "" { - co.envVar = strutils.KebabToConstantCase(co.name) - } - viper.BindEnv(co.name, co.envVar) - // Initialise the persistent flags - co.setFlag() -} - -// require checks that a required string configuration option is not empty, raising a user error if it is. -func (co *configOption) require() { - if co.required == true && viper.GetString(co.name) == "" { - stdLog.Fatalf("Invalid config: %s is blank. Please specify --%s on the command line or set the %s environment variable.", co.name, co.name, co.envVar) - } -} - -// setValue sets a value in the global config, using a custom function, if one was provided. -func (co *configOption) setValue() { - // Use a custom setting function, if one is provided - if co.customSetValue != nil { - co.customSetValue(co) - // Otherwise, just set the provided arg directly - } else if co.configKey != nil { - co.setSimpleValue() - } -} - -// setSimpleValue sets the value of a configOption's configKey, based on the configOption's default type. -func (co *configOption) setSimpleValue() { - if co.configKey != nil { - switch co.flagDefault.(type) { - case string: - *(co.configKey.(*string)) = viper.GetString(co.name) - case int: - *(co.configKey.(*int)) = viper.GetInt(co.name) - case bool: - *(co.configKey.(*bool)) = viper.GetBool(co.name) - case uint: - *(co.configKey.(*uint)) = uint(viper.GetInt(co.name)) - } - } -} - -// setFlag sets the correct pFlag type, based on the configOption's default type. -func (co *configOption) setFlag() { - switch co.flagDefault.(type) { - case string: - stringFlag(co.name, co.flagDefault, co.usage) - case int: - intFlag(co.name, co.flagDefault, co.usage) - case bool: - boolFlag(co.name, co.flagDefault, co.usage) - case uint: - uintFlag(co.name, co.flagDefault, co.usage) - } -} - -// setDuration converts a command line int to a duration, and stores it in the final config. -func setDuration(co *configOption) { - *(co.configKey.(*time.Duration)) = time.Duration(viper.GetInt(co.name)) * time.Second -} - -// setURL converts a command line string to a URL, and stores it in the final config. -func setURL(co *configOption) { - urlString := viper.GetString(co.name) - if urlString != "" { - urlType, err := url.Parse(urlString) - if err != nil { - stdLog.Fatalf("Unable to parse URL: %s/%v", urlString, err) - } - *(co.configKey.(**url.URL)) = urlType - } -} - -// setLogLevel validates and sets the log level in the final config. -func setLogLevel(co *configOption) { - ll, err := logrus.ParseLevel(viper.GetString(co.name)) - if err != nil { - stdLog.Fatalf("Could not parse log-level: %v", viper.GetString(co.name)) - } - *(co.configKey.(*logrus.Level)) = ll -} -// setRateLimit converts a command line rate limit, and sets rate and burst limiting in the final config. -func setRateLimit(co *configOption) { - var rateLimit *throttled.RateQuota = nil - perHourRateLimit := viper.GetInt(co.name) - if perHourRateLimit != 0 { - rateLimit = &throttled.RateQuota{ - MaxRate: throttled.PerHour(perHourRateLimit), - MaxBurst: 100, - } - *(co.configKey.(**throttled.RateQuota)) = rateLimit - } -} +var rootCmd *cobra.Command // validateBothOrNeither ensures that both options are provided, if either is provided func validateBothOrNeither(option1, option2 string) { @@ -182,167 +50,167 @@ func checkMigrations() { // configOpts defines the complete flag configuration for horizon // Add a new entry here to connect a new field in the horizon.Config struct -var configOpts = []*configOption{ - &configOption{ - name: "db-url", - envVar: "DATABASE_URL", - configKey: &config.DatabaseURL, - flagDefault: "", - required: true, - usage: "horizon postgres database to connect with", +var configOpts = []*support.ConfigOption{ + &support.ConfigOption{ + Name: "db-url", + EnvVar: "DATABASE_URL", + ConfigKey: &config.DatabaseURL, + FlagDefault: "", + Required: true, + Usage: "horizon postgres database to connect with", }, - &configOption{ - name: "stellar-core-db-url", - envVar: "STELLAR_CORE_DATABASE_URL", - configKey: &config.StellarCoreDatabaseURL, - flagDefault: "", - required: true, - usage: "stellar-core postgres database to connect with", + &support.ConfigOption{ + Name: "stellar-core-db-url", + EnvVar: "STELLAR_CORE_DATABASE_URL", + ConfigKey: &config.StellarCoreDatabaseURL, + FlagDefault: "", + Required: true, + Usage: "stellar-core postgres database to connect with", }, - &configOption{ - name: "stellar-core-url", - configKey: &config.StellarCoreURL, - flagDefault: "", - required: true, - usage: "stellar-core to connect with (for http commands)", + &support.ConfigOption{ + Name: "stellar-core-url", + ConfigKey: &config.StellarCoreURL, + FlagDefault: "", + Required: true, + Usage: "stellar-core to connect with (for http commands)", }, - &configOption{ - name: "port", - configKey: &config.Port, - flagDefault: uint(8000), - usage: "tcp port to listen on for http requests", + &support.ConfigOption{ + Name: "port", + ConfigKey: &config.Port, + FlagDefault: uint(8000), + Usage: "tcp port to listen on for http requests", }, - &configOption{ - name: "max-db-connections", - configKey: &config.MaxDBConnections, - flagDefault: int(20), - usage: "max db connections (per DB), may need to be increased when responses are slow but DB CPU is normal", + &support.ConfigOption{ + Name: "max-db-connections", + ConfigKey: &config.MaxDBConnections, + FlagDefault: int(20), + Usage: "max db connections (per DB), may need to be increased when responses are slow but DB CPU is normal", }, - &configOption{ - name: "sse-update-frequency", - configKey: &config.SSEUpdateFrequency, - flagDefault: 5, - customSetValue: setDuration, - usage: "defines how often streams should check if there's a new ledger (in seconds), may need to increase in case of big number of streams", + &support.ConfigOption{ + Name: "sse-update-frequency", + ConfigKey: &config.SSEUpdateFrequency, + FlagDefault: 5, + CustomSetValue: support.SetDuration, + Usage: "defines how often streams should check if there's a new ledger (in seconds), may need to increase in case of big number of streams", }, - &configOption{ - name: "connection-timeout", - configKey: &config.ConnectionTimeout, - flagDefault: 55, - customSetValue: setDuration, - usage: "defines the timeout of connection after which 504 response will be sent or stream will be closed, if Horizon is behind a load balancer with idle connection timeout, this should be set to a few seconds less that idle timeout", + &support.ConfigOption{ + Name: "connection-timeout", + ConfigKey: &config.ConnectionTimeout, + FlagDefault: 55, + CustomSetValue: support.SetDuration, + Usage: "defines the timeout of connection after which 504 response will be sent or stream will be closed, if Horizon is behind a load balancer with idle connection timeout, this should be set to a few seconds less that idle timeout", }, - &configOption{ - name: "per-hour-rate-limit", - configKey: &config.RateLimit, - flagDefault: 3600, - customSetValue: setRateLimit, - usage: "max count of requests allowed in a one hour period, by remote ip address", + &support.ConfigOption{ + Name: "per-hour-rate-limit", + ConfigKey: &config.RateLimit, + FlagDefault: 3600, + CustomSetValue: support.SetRateLimit, + Usage: "max count of requests allowed in a one hour period, by remote ip address", }, - &configOption{ - name: "rate-limit-redis-key", - configKey: &config.RateLimitRedisKey, - flagDefault: "", - usage: "redis key for storing rate limit data, useful when deploying a cluster of Horizons, ignored when redis-url is empty", + &support.ConfigOption{ + Name: "rate-limit-redis-key", + ConfigKey: &config.RateLimitRedisKey, + FlagDefault: "", + Usage: "redis key for storing rate limit data, useful when deploying a cluster of Horizons, ignored when redis-url is empty", }, - &configOption{ - name: "redis-url", - configKey: &config.RedisURL, - flagDefault: "", - usage: "redis to connect with, for rate limiting", + &support.ConfigOption{ + Name: "redis-url", + ConfigKey: &config.RedisURL, + FlagDefault: "", + Usage: "redis to connect with, for rate limiting", }, - &configOption{ - name: "friendbot-url", - configKey: &config.FriendbotURL, - flagDefault: "", - customSetValue: setURL, - usage: "friendbot service to redirect to", + &support.ConfigOption{ + Name: "friendbot-url", + ConfigKey: &config.FriendbotURL, + FlagDefault: "", + CustomSetValue: support.SetURL, + Usage: "friendbot service to redirect to", }, - &configOption{ - name: "log-level", - configKey: &config.LogLevel, - flagDefault: "info", - customSetValue: setLogLevel, - usage: "minimum log severity (debug, info, warn, error) to log", + &support.ConfigOption{ + Name: "log-level", + ConfigKey: &config.LogLevel, + FlagDefault: "info", + CustomSetValue: support.SetLogLevel, + Usage: "minimum log severity (debug, info, warn, error) to log", }, - &configOption{ - name: "log-file", - configKey: &config.LogFile, - flagDefault: "", - usage: "name of the file where logs will be saved (leave empty to send logs to stdout)", + &support.ConfigOption{ + Name: "log-file", + ConfigKey: &config.LogFile, + FlagDefault: "", + Usage: "name of the file where logs will be saved (leave empty to send logs to stdout)", }, - &configOption{ - name: "max-path-length", - configKey: &config.MaxPathLength, - flagDefault: uint(4), - usage: "the maximum number of assets on the path in `/paths` endpoint", + &support.ConfigOption{ + Name: "max-path-length", + ConfigKey: &config.MaxPathLength, + FlagDefault: uint(4), + Usage: "the maximum number of assets on the path in `/paths` endpoint", }, - &configOption{ - name: "network-passphrase", - configKey: &config.NetworkPassphrase, - flagDefault: network.TestNetworkPassphrase, - required: true, - usage: "Override the network passphrase", + &support.ConfigOption{ + Name: "network-passphrase", + ConfigKey: &config.NetworkPassphrase, + FlagDefault: network.TestNetworkPassphrase, + Required: true, + Usage: "Override the network passphrase", }, - &configOption{ - name: "sentry-dsn", - configKey: &config.SentryDSN, - flagDefault: "", - usage: "Sentry URL to which panics and errors should be reported", + &support.ConfigOption{ + Name: "sentry-dsn", + ConfigKey: &config.SentryDSN, + FlagDefault: "", + Usage: "Sentry URL to which panics and errors should be reported", }, - &configOption{ - name: "loggly-token", - configKey: &config.LogglyToken, - flagDefault: "", - usage: "Loggly token, used to configure log forwarding to loggly", + &support.ConfigOption{ + Name: "loggly-token", + ConfigKey: &config.LogglyToken, + FlagDefault: "", + Usage: "Loggly token, used to configure log forwarding to loggly", }, - &configOption{ - name: "loggly-tag", - configKey: &config.LogglyTag, - flagDefault: "horizon", - usage: "Tag to be added to every loggly log event", + &support.ConfigOption{ + Name: "loggly-tag", + ConfigKey: &config.LogglyTag, + FlagDefault: "horizon", + Usage: "Tag to be added to every loggly log event", }, - &configOption{ - name: "tls-cert", - configKey: &config.TLSCert, - flagDefault: "", - usage: "TLS certificate file to use for securing connections to horizon", + &support.ConfigOption{ + Name: "tls-cert", + ConfigKey: &config.TLSCert, + FlagDefault: "", + Usage: "TLS certificate file to use for securing connections to horizon", }, - &configOption{ - name: "tls-key", - configKey: &config.TLSKey, - flagDefault: "", - usage: "TLS private key file to use for securing connections to horizon", + &support.ConfigOption{ + Name: "tls-key", + ConfigKey: &config.TLSKey, + FlagDefault: "", + Usage: "TLS private key file to use for securing connections to horizon", }, - &configOption{ - name: "ingest", - configKey: &config.Ingest, - flagDefault: false, - usage: "causes this horizon process to ingest data from stellar-core into horizon's db", + &support.ConfigOption{ + Name: "ingest", + ConfigKey: &config.Ingest, + FlagDefault: false, + Usage: "causes this horizon process to ingest data from stellar-core into horizon's db", }, - &configOption{ - name: "history-retention-count", - configKey: &config.HistoryRetentionCount, - flagDefault: uint(0), - usage: "the minimum number of ledgers to maintain within horizon's history tables. 0 signifies an unlimited number of ledgers will be retained", + &support.ConfigOption{ + Name: "history-retention-count", + ConfigKey: &config.HistoryRetentionCount, + FlagDefault: uint(0), + Usage: "the minimum number of ledgers to maintain within horizon's history tables. 0 signifies an unlimited number of ledgers will be retained", }, - &configOption{ - name: "history-stale-threshold", - configKey: &config.StaleThreshold, - flagDefault: uint(0), - usage: "the maximum number of ledgers the history db is allowed to be out of date from the connected stellar-core db before horizon considers history stale", + &support.ConfigOption{ + Name: "history-stale-threshold", + ConfigKey: &config.StaleThreshold, + FlagDefault: uint(0), + Usage: "the maximum number of ledgers the history db is allowed to be out of date from the connected stellar-core db before horizon considers history stale", }, - &configOption{ - name: "skip-cursor-update", - configKey: &config.SkipCursorUpdate, - flagDefault: false, - usage: "causes the ingester to skip reporting the last imported ledger state to stellar-core", + &support.ConfigOption{ + Name: "skip-cursor-update", + ConfigKey: &config.SkipCursorUpdate, + FlagDefault: false, + Usage: "causes the ingester to skip reporting the last imported ledger state to stellar-core", }, - &configOption{ - name: "enable-asset-stats", - configKey: &config.EnableAssetStats, - flagDefault: false, - usage: "enables asset stats during the ingestion and expose `/assets` endpoint, Enabling it has a negative impact on CPU", + &support.ConfigOption{ + Name: "enable-asset-stats", + ConfigKey: &config.EnableAssetStats, + FlagDefault: false, + Usage: "enables asset stats during the ingestion and expose `/assets` endpoint, Enabling it has a negative impact on CPU", }, } @@ -362,7 +230,7 @@ func init() { } for _, co := range configOpts { - co.init() + co.Init(rootCmd) } rootCmd.AddCommand(dbCmd) @@ -385,7 +253,7 @@ func initApp(cmd *cobra.Command, args []string) *horizon.App { func initConfig() { // Check all required args were provided - needed for migrations check for _, co := range configOpts { - co.require() + co.Require() } // Migrations should be checked as early as possible @@ -393,7 +261,7 @@ func initConfig() { // Initialise and validate the global configuration for _, co := range configOpts { - co.setValue() + co.SetValue() } // Validate options that should be provided together validateBothOrNeither("tls-cert", "tls-key") diff --git a/support/config/config_option.go b/support/config/config_option.go new file mode 100644 index 0000000000..c3b9b7a339 --- /dev/null +++ b/support/config/config_option.go @@ -0,0 +1,142 @@ +package config + +import ( + stdLog "log" + "net/url" + "time" + + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "github.com/stellar/go/support/strutils" + "github.com/throttled/throttled" +) + +// flagType implements a generic interface for the different command line flags, +// allowing them to be configured in a uniform way. +type flagType func(name string, value interface{}, usage string, rootCmd *cobra.Command) interface{} + +var ( + stringFlag flagType = func(name string, value interface{}, usage string, rootCmd *cobra.Command) interface{} { + return rootCmd.PersistentFlags().String(name, value.(string), usage) + } + intFlag flagType = func(name string, value interface{}, usage string, rootCmd *cobra.Command) interface{} { + return rootCmd.PersistentFlags().Int(name, value.(int), usage) + } + uintFlag flagType = func(name string, value interface{}, usage string, rootCmd *cobra.Command) interface{} { + return rootCmd.PersistentFlags().Uint(name, value.(uint), usage) + } + boolFlag flagType = func(name string, value interface{}, usage string, rootCmd *cobra.Command) interface{} { + return rootCmd.PersistentFlags().Bool(name, value.(bool), usage) + } +) + +// ConfigOption is a complete description of the configuration of a command line option +type ConfigOption struct { + Name string // e.g. "db-url" + EnvVar string // e.g. "DATABASE_URL". Defaults to uppercase/underscore representation of name + FlagDefault interface{} // A default if no option is provided. Set to "" if no default + Required bool // Whether this option must be set for Horizon to run + Usage string // Help text + CustomSetValue func(*ConfigOption) // Optional function for custom validation/transformation + ConfigKey interface{} // Pointer to the final key in the horizon.Config struct +} + +// Init handles initialisation steps, including configuring and binding the env variable name. +func (co *ConfigOption) Init(cmd *cobra.Command) { + // Bind the command line and environment variable name + // Unless overriden, default to a transform like tls-key -> TLS_KEY + if co.EnvVar == "" { + co.EnvVar = strutils.KebabToConstantCase(co.Name) + } + viper.BindEnv(co.Name, co.EnvVar) + // Initialise the persistent flags + co.setFlag(cmd) +} + +// Require checks that a required string configuration option is not empty, raising a user error if it is. +func (co *ConfigOption) Require() { + if co.Required == true && viper.GetString(co.Name) == "" { + stdLog.Fatalf("Invalid config: %s is blank. Please specify --%s on the command line or set the %s environment variable.", co.Name, co.Name, co.EnvVar) + } +} + +// SetValue sets a value in the global config, using a custom function, if one was provided. +func (co *ConfigOption) SetValue() { + // Use a custom setting function, if one is provided + if co.CustomSetValue != nil { + co.CustomSetValue(co) + // Otherwise, just set the provided arg directly + } else if co.ConfigKey != nil { + co.setSimpleValue() + } +} + +// setSimpleValue sets the value of a ConfigOption's configKey, based on the ConfigOption's default type. +func (co *ConfigOption) setSimpleValue() { + if co.ConfigKey != nil { + switch co.FlagDefault.(type) { + case string: + *(co.ConfigKey.(*string)) = viper.GetString(co.Name) + case int: + *(co.ConfigKey.(*int)) = viper.GetInt(co.Name) + case bool: + *(co.ConfigKey.(*bool)) = viper.GetBool(co.Name) + case uint: + *(co.ConfigKey.(*uint)) = uint(viper.GetInt(co.Name)) + } + } +} + +// setFlag sets the correct pFlag type, based on the ConfigOption's default type. +func (co *ConfigOption) setFlag(cmd *cobra.Command) { + switch co.FlagDefault.(type) { + case string: + stringFlag(co.Name, co.FlagDefault, co.Usage, cmd) + case int: + intFlag(co.Name, co.FlagDefault, co.Usage, cmd) + case bool: + boolFlag(co.Name, co.FlagDefault, co.Usage, cmd) + case uint: + uintFlag(co.Name, co.FlagDefault, co.Usage, cmd) + } +} + +// SetDuration converts a command line int to a duration, and stores it in the final config. +func SetDuration(co *ConfigOption) { + *(co.ConfigKey.(*time.Duration)) = time.Duration(viper.GetInt(co.Name)) * time.Second +} + +// SetURL converts a command line string to a URL, and stores it in the final config. +func SetURL(co *ConfigOption) { + urlString := viper.GetString(co.Name) + if urlString != "" { + urlType, err := url.Parse(urlString) + if err != nil { + stdLog.Fatalf("Unable to parse URL: %s/%v", urlString, err) + } + *(co.ConfigKey.(**url.URL)) = urlType + } +} + +// SetLogLevel validates and sets the log level in the final config. +func SetLogLevel(co *ConfigOption) { + ll, err := logrus.ParseLevel(viper.GetString(co.Name)) + if err != nil { + stdLog.Fatalf("Could not parse log-level: %v", viper.GetString(co.Name)) + } + *(co.ConfigKey.(*logrus.Level)) = ll +} + +// SetRateLimit converts a command line rate limit, and sets rate and burst limiting in the final config. +func SetRateLimit(co *ConfigOption) { + var rateLimit *throttled.RateQuota = nil + perHourRateLimit := viper.GetInt(co.Name) + if perHourRateLimit != 0 { + rateLimit = &throttled.RateQuota{ + MaxRate: throttled.PerHour(perHourRateLimit), + MaxBurst: 100, + } + *(co.ConfigKey.(**throttled.RateQuota)) = rateLimit + } +}