egnine/sync-manager: add config support (thrasher-corp#1326)

* allows sync manager customisation for values and logs * config-example add * who doesnt like more coverage? * ensures you can actually disable it via config el oh el * less ifs, better control * fix verbose * sync trades default false * fix summary being printed when not enabled * fixes config checker and output * nits * I can put this behind me now * Fixed logCaSiNg Co-authored-by: Adrian Gallagher <[email protected]> * combines if statements --------- Co-authored-by: Adrian Gallagher <[email protected]>
shazbert · Nov 9, 2023 · 612b9da · 612b9da
1 parent a362b53
commit 612b9da
Show file tree

Hide file tree

Showing 10 changed files with 281 additions and 108 deletions.
diff --git a/config/config.go b/config/config.go
@@ -696,6 +696,63 @@ func (c *Config) GetAvailablePairs(exchName string, assetType asset.Item) (curre
 	return pairs.Format(pairFormat), nil
 }
 
+// GetDefaultSyncManagerConfig returns a config with default values
+func GetDefaultSyncManagerConfig() SyncManagerConfig {
+	return SyncManagerConfig{
+		Enabled:                 true,
+		SynchronizeTicker:       true,
+		SynchronizeOrderbook:    true,
+		SynchronizeTrades:       false,
+		SynchronizeContinuously: true,
+		TimeoutREST:             DefaultSyncerTimeoutREST,
+		TimeoutWebsocket:        DefaultSyncerTimeoutWebsocket,
+		NumWorkers:              DefaultSyncerWorkers,
+		FiatDisplayCurrency:     currency.USD,
+		PairFormatDisplay: &currency.PairFormat{
+			Delimiter: "-",
+			Uppercase: true,
+		},
+		Verbose:                 false,
+		LogSyncUpdateEvents:     true,
+		LogSwitchProtocolEvents: true,
+		LogInitialSyncEvents:    true,
+	}
+}
+
+// CheckSyncManagerConfig checks config for valid values
+// sets defaults if values are invalid
+func (c *Config) CheckSyncManagerConfig() {
+	m.Lock()
+	defer m.Unlock()
+	if c.SyncManagerConfig == (SyncManagerConfig{}) {
+		c.SyncManagerConfig = GetDefaultSyncManagerConfig()
+		return
+	}
+	if c.SyncManagerConfig.TimeoutWebsocket <= 0 {
+		log.Warnf(log.ConfigMgr, "Invalid sync manager websocket timeout value %v, defaulting to %v\n", c.SyncManagerConfig.TimeoutWebsocket, DefaultSyncerTimeoutWebsocket)
+		c.SyncManagerConfig.TimeoutWebsocket = DefaultSyncerTimeoutWebsocket
+	}
+	if c.SyncManagerConfig.PairFormatDisplay == nil {
+		log.Warnf(log.ConfigMgr, "Invalid sync manager pair format value %v, using default format eg BTC-USD\n", c.SyncManagerConfig.PairFormatDisplay)
+		c.SyncManagerConfig.PairFormatDisplay = &currency.PairFormat{
+			Uppercase: true,
+			Delimiter: currency.DashDelimiter,
+		}
+	}
+	if c.SyncManagerConfig.TimeoutREST <= 0 {
+		log.Warnf(log.ConfigMgr, "Invalid sync manager REST timeout value %v, defaulting to %v\n", c.SyncManagerConfig.TimeoutREST, DefaultSyncerTimeoutREST)
+		c.SyncManagerConfig.TimeoutREST = DefaultSyncerTimeoutREST
+	}
+	if c.SyncManagerConfig.NumWorkers <= 0 {
+		log.Warnf(log.ConfigMgr, "Invalid sync manager worker count value %v, defaulting to %v\n", c.SyncManagerConfig.NumWorkers, DefaultSyncerWorkers)
+		c.SyncManagerConfig.NumWorkers = DefaultSyncerWorkers
+	}
+	if c.SyncManagerConfig.FiatDisplayCurrency.IsEmpty() {
+		log.Warnf(log.ConfigMgr, "Invalid sync manager fiat display currency value, defaulting to %v\n", currency.USD)
+		c.SyncManagerConfig.FiatDisplayCurrency = currency.USD
+	}
+}
+
 // GetEnabledPairs returns a list of currency pairs for a specific exchange
 func (c *Config) GetEnabledPairs(exchName string, assetType asset.Item) (currency.Pairs, error) {
 	exchCfg, err := c.GetExchangeConfig(exchName)
@@ -1730,6 +1787,7 @@ func (c *Config) CheckConfig() error {
 	c.CheckClientBankAccounts()
 	c.CheckBankAccountConfig()
 	c.CheckRemoteControlConfig()
+	c.CheckSyncManagerConfig()
 
 	err = c.CheckCurrencyConfigValues()
 	if err != nil {

diff --git a/config/config_test.go b/config/config_test.go
@@ -2299,3 +2299,46 @@ func TestExchangeConfigValidate(t *testing.T) {
 		t.Fatalf("received: '%v' but expected: '%v'", err, nil)
 	}
 }
+
+func TestGetDefaultSyncManagerConfig(t *testing.T) {
+	t.Parallel()
+	cfg := GetDefaultSyncManagerConfig()
+	if cfg == (SyncManagerConfig{}) {
+		t.Error("expected config")
+	}
+	if cfg.TimeoutREST != DefaultSyncerTimeoutREST {
+		t.Errorf("expected %v, received %v", DefaultSyncerTimeoutREST, cfg.TimeoutREST)
+	}
+}
+
+func TestCheckSyncManagerConfig(t *testing.T) {
+	t.Parallel()
+	c := Config{}
+	if c.SyncManagerConfig != (SyncManagerConfig{}) {
+		t.Error("expected empty config")
+	}
+	c.CheckSyncManagerConfig()
+	if c.SyncManagerConfig.TimeoutREST != DefaultSyncerTimeoutREST {
+		t.Error("expected default config")
+	}
+	c.SyncManagerConfig.TimeoutWebsocket = -1
+	c.SyncManagerConfig.PairFormatDisplay = nil
+	c.SyncManagerConfig.TimeoutREST = -1
+	c.SyncManagerConfig.NumWorkers = -1
+	c.CurrencyPairFormat = &currency.PairFormat{
+		Uppercase: true,
+	}
+	c.CheckSyncManagerConfig()
+	if c.SyncManagerConfig.TimeoutWebsocket != DefaultSyncerTimeoutWebsocket {
+		t.Errorf("received %v expected %v", c.SyncManagerConfig.TimeoutWebsocket, DefaultSyncerTimeoutWebsocket)
+	}
+	if c.SyncManagerConfig.PairFormatDisplay == nil {
+		t.Errorf("received %v expected %v", c.SyncManagerConfig.PairFormatDisplay, c.CurrencyPairFormat)
+	}
+	if c.SyncManagerConfig.TimeoutREST != DefaultSyncerTimeoutREST {
+		t.Errorf("received %v expected %v", c.SyncManagerConfig.TimeoutREST, DefaultSyncerTimeoutREST)
+	}
+	if c.SyncManagerConfig.NumWorkers != DefaultSyncerWorkers {
+		t.Errorf("received %v expected %v", c.SyncManagerConfig.NumWorkers, DefaultSyncerWorkers)
+	}
+}
diff --git a/config/config_types.go b/config/config_types.go
@@ -41,6 +41,12 @@ const (
 	defaultCurrencyStateManagerDelay     = time.Minute
 	defaultMaxJobsPerCycle               = 5
 	DefaultOrderbookPublishPeriod        = time.Second * 10
+	// DefaultSyncerWorkers limits the number of sync workers
+	DefaultSyncerWorkers = 15
+	// DefaultSyncerTimeoutREST the default time to switch from REST to websocket protocols without a response
+	DefaultSyncerTimeoutREST = time.Second * 15
+	// DefaultSyncerTimeoutWebsocket the default time to switch from websocket to REST protocols without a response
+	DefaultSyncerTimeoutWebsocket = time.Minute
 )
 
 // Constants here hold some messages
@@ -81,6 +87,7 @@ type Config struct {
 	GlobalHTTPTimeout    time.Duration             `json:"globalHTTPTimeout"`
 	Database             database.Config           `json:"database"`
 	Logging              log.Config                `json:"logging"`
+	SyncManagerConfig    SyncManagerConfig         `json:"syncManager"`
 	ConnectionMonitor    ConnectionMonitorConfig   `json:"connectionMonitor"`
 	OrderManager         OrderManager              `json:"orderManager"`
 	DataHistoryManager   DataHistoryManager        `json:"dataHistoryManager"`
@@ -130,6 +137,25 @@ type CurrencyStateManager struct {
 	Delay   time.Duration `json:"delay"`
 }
 
+// SyncManagerConfig stores the currency pair synchronization manager config
+type SyncManagerConfig struct {
+	Enabled                 bool                 `json:"enabled"`
+	SynchronizeTicker       bool                 `json:"synchronizeTicker"`
+	SynchronizeOrderbook    bool                 `json:"synchronizeOrderbook"`
+	SynchronizeTrades       bool                 `json:"synchronizeTrades"`
+	SynchronizeContinuously bool                 `json:"synchronizeContinuously"`
+	TimeoutREST             time.Duration        `json:"timeoutREST"`
+	TimeoutWebsocket        time.Duration        `json:"timeoutWebsocket"`
+	NumWorkers              int                  `json:"numWorkers"`
+	FiatDisplayCurrency     currency.Code        `json:"fiatDisplayCurrency"`
+	PairFormatDisplay       *currency.PairFormat `json:"pairFormatDisplay,omitempty"`
+	// log events
+	Verbose                 bool `json:"verbose"`
+	LogSyncUpdateEvents     bool `json:"logSyncUpdateEvents"`
+	LogSwitchProtocolEvents bool `json:"logSwitchProtocolEvents"`
+	LogInitialSyncEvents    bool `json:"logInitialSyncEvents"`
+}
+
 // ConnectionMonitorConfig defines the connection monitor variables to ensure
 // that there is internet connectivity
 type ConnectionMonitorConfig struct {

diff --git a/config_example.json b/config_example.json
@@ -37,6 +37,25 @@
    }
   }
  },
+  "syncManager": {
+    "enabled": true,
+    "synchronizeTicker": true,
+    "synchronizeOrderbook": true,
+    "synchronizeTrades": true,
+    "synchronizeContinuously": true,
+    "timeoutREST": 15000000000,
+    "timeoutWebsocket": 60000000000,
+    "numWorkers": 15,
+    "fiatDisplayCurrency": "USD",
+    "pairFormatDisplay": {
+      "uppercase": true,
+      "delimiter": "-"
+    },
+    "verbose": false,
+    "logSyncUpdateEvents": true,
+    "logSwitchProtocolEvents": true,
+    "logInitialSyncEvents": true
+  },
  "connectionMonitor": {
   "preferredDNSList": [
    "8.8.8.8",

diff --git a/engine/engine.go b/engine/engine.go
@@ -179,6 +179,12 @@ func validateSettings(b *Engine, s *Settings, flagSet FlagSet) {
 	flagSet.WithBool("currencystatemanager", &b.Settings.EnableCurrencyStateManager, b.Config.CurrencyStateManager.Enabled != nil && *b.Config.CurrencyStateManager.Enabled)
 	flagSet.WithBool("gctscriptmanager", &b.Settings.EnableGCTScriptManager, b.Config.GCTScript.Enabled)
 
+	flagSet.WithBool("tickersync", &b.Settings.EnableTickerSyncing, b.Config.SyncManagerConfig.SynchronizeTicker)
+	flagSet.WithBool("orderbooksync", &b.Settings.EnableOrderbookSyncing, b.Config.SyncManagerConfig.SynchronizeOrderbook)
+	flagSet.WithBool("tradesync", &b.Settings.EnableTradeSyncing, b.Config.SyncManagerConfig.SynchronizeTrades)
+	flagSet.WithBool("synccontinuously", &b.Settings.SyncContinuously, b.Config.SyncManagerConfig.SynchronizeContinuously)
+	flagSet.WithBool("syncmanager", &b.Settings.EnableExchangeSyncManager, b.Config.SyncManagerConfig.Enabled)
+
 	if b.Settings.EnablePortfolioManager &&
 		b.Settings.PortfolioManagerDelay <= 0 {
 		b.Settings.PortfolioManagerDelay = PortfolioSleepDelay
@@ -491,21 +497,27 @@ func (bot *Engine) Start() error {
 	}
 
 	if bot.Settings.EnableExchangeSyncManager {
-		exchangeSyncCfg := &SyncManagerConfig{
-			SynchronizeTicker:       bot.Settings.EnableTickerSyncing,
-			SynchronizeOrderbook:    bot.Settings.EnableOrderbookSyncing,
-			SynchronizeTrades:       bot.Settings.EnableTradeSyncing,
-			SynchronizeContinuously: bot.Settings.SyncContinuously,
-			TimeoutREST:             bot.Settings.SyncTimeoutREST,
-			TimeoutWebsocket:        bot.Settings.SyncTimeoutWebsocket,
-			NumWorkers:              bot.Settings.SyncWorkersCount,
-			Verbose:                 bot.Settings.Verbose,
-			FiatDisplayCurrency:     bot.Config.Currency.FiatDisplayCurrency,
-			PairFormatDisplay:       bot.Config.Currency.CurrencyPairFormat,
-		}
+		cfg := bot.Config.SyncManagerConfig
+		cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing
+		cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing
+		cfg.SynchronizeContinuously = bot.Settings.SyncContinuously
+		cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing
+		cfg.Verbose = bot.Settings.Verbose || cfg.Verbose
 
+		if cfg.TimeoutREST != bot.Settings.SyncTimeoutREST &&
+			bot.Settings.SyncTimeoutREST != config.DefaultSyncerTimeoutREST {
+			cfg.TimeoutREST = bot.Settings.SyncTimeoutREST
+		}
+		if cfg.TimeoutWebsocket != bot.Settings.SyncTimeoutWebsocket &&
+			bot.Settings.SyncTimeoutWebsocket != config.DefaultSyncerTimeoutWebsocket {
+			cfg.TimeoutWebsocket = bot.Settings.SyncTimeoutWebsocket
+		}
+		if cfg.NumWorkers != bot.Settings.SyncWorkersCount &&
+			bot.Settings.SyncWorkersCount != config.DefaultSyncerWorkers {
+			cfg.NumWorkers = bot.Settings.SyncWorkersCount
+		}
 		if s, err := setupSyncManager(
-			exchangeSyncCfg,
+			&cfg,
 			bot.ExchangeManager,
 			&bot.Config.RemoteControl,
 			bot.Settings.EnableWebsocketRoutine,

diff --git a/engine/helpers.go b/engine/helpers.go
@@ -185,19 +185,27 @@ func (bot *Engine) SetSubsystem(subSystemName string, enable bool) error {
 	case SyncManagerName:
 		if enable {
 			if bot.currencyPairSyncer == nil {
-				exchangeSyncCfg := &SyncManagerConfig{
-					SynchronizeTicker:       bot.Settings.EnableTickerSyncing,
-					SynchronizeOrderbook:    bot.Settings.EnableOrderbookSyncing,
-					SynchronizeTrades:       bot.Settings.EnableTradeSyncing,
-					SynchronizeContinuously: bot.Settings.SyncContinuously,
-					TimeoutREST:             bot.Settings.SyncTimeoutREST,
-					TimeoutWebsocket:        bot.Settings.SyncTimeoutWebsocket,
-					NumWorkers:              bot.Settings.SyncWorkersCount,
-					FiatDisplayCurrency:     bot.Config.Currency.FiatDisplayCurrency,
-					Verbose:                 bot.Settings.Verbose,
+				cfg := bot.Config.SyncManagerConfig
+				cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing
+				cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing
+				cfg.SynchronizeContinuously = bot.Settings.SyncContinuously
+				cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing
+				cfg.Verbose = bot.Settings.Verbose || cfg.Verbose
+
+				if cfg.TimeoutREST != bot.Settings.SyncTimeoutREST &&
+					bot.Settings.SyncTimeoutREST != config.DefaultSyncerTimeoutREST {
+					cfg.TimeoutREST = bot.Settings.SyncTimeoutREST
+				}
+				if cfg.TimeoutWebsocket != bot.Settings.SyncTimeoutWebsocket &&
+					bot.Settings.SyncTimeoutWebsocket != config.DefaultSyncerTimeoutWebsocket {
+					cfg.TimeoutWebsocket = bot.Settings.SyncTimeoutWebsocket
+				}
+				if cfg.NumWorkers != bot.Settings.SyncWorkersCount &&
+					bot.Settings.SyncWorkersCount != config.DefaultSyncerWorkers {
+					cfg.NumWorkers = bot.Settings.SyncWorkersCount
 				}
 				bot.currencyPairSyncer, err = setupSyncManager(
-					exchangeSyncCfg,
+					&cfg,
 					bot.ExchangeManager,
 					&bot.Config.RemoteControl,
 					bot.Settings.EnableWebsocketRoutine)