From 0bbbe6f17f82895d5b8794ff2144af149cfdaa8a Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 22 Aug 2023 16:17:11 +1000 Subject: [PATCH 01/13] allows sync manager customisation for values and logs --- config/config.go | 51 ++++++++++++++++ config/config_test.go | 11 ++++ config/config_types.go | 26 ++++++++ engine/engine.go | 41 +++++++++---- engine/helpers.go | 36 ++++++++---- engine/sync_manager.go | 111 +++++++++++++++++++---------------- engine/sync_manager_test.go | 22 +++---- engine/sync_manager_types.go | 16 +---- main.go | 6 +- 9 files changed, 218 insertions(+), 102 deletions(-) diff --git a/config/config.go b/config/config.go index 11593e2cfb5..7674abef211 100644 --- a/config/config.go +++ b/config/config.go @@ -696,6 +696,56 @@ func (c *Config) GetAvailablePairs(exchName string, assetType asset.Item) (curre return pairs.Format(pairFormat), nil } +// GetDefaultSyncManagerConfig returns a config with default values +func GetDefaultSyncManagerConfig() SyncManagerConfig { + return SyncManagerConfig{ + Enabled: true, + SynchronizeTicker: true, + SynchronizeOrderbook: true, + SynchronizeTrades: true, + SynchronizeContinuously: true, + TimeoutREST: DefaultSyncerTimeoutREST, + TimeoutWebsocket: DefaultSyncerTimeoutWebsocket, + NumWorkers: DefaultSyncerWorkers, + FiatDisplayCurrency: currency.USD, + PairFormatDisplay: ¤cy.PairFormat{ + Delimiter: "-", + Uppercase: true, + }, + Verbose: false, + LogSyncUpdateEvents: true, + LogSwitchProtocolEvents: true, + LogInitialSyncEvents: true, + } +} + +// CheckSyncManagerConfig checks config for valid values +// sets defaults if values are invalid +func (c *Config) CheckSyncManagerConfig() { + m.Lock() + defer m.Unlock() + if c.SyncManagerConfig == (SyncManagerConfig{}) { + c.SyncManagerConfig = GetDefaultSyncManagerConfig() + return + } + if c.SyncManagerConfig.TimeoutWebsocket <= 0 { + log.Warnf(log.ConfigMgr, "invalid sync manager websocket timeout value %v, defaulting to %v\n", c.SyncManagerConfig.TimeoutWebsocket, DefaultSyncerTimeoutWebsocket) + c.SyncManagerConfig.TimeoutWebsocket = DefaultSyncerTimeoutWebsocket + } + if c.SyncManagerConfig.PairFormatDisplay == nil { + log.Warnf(log.ConfigMgr, "invalid sync manager pair format value %v, defaulting to %v\n", c.SyncManagerConfig.PairFormatDisplay, c.CurrencyPairFormat) + c.SyncManagerConfig.PairFormatDisplay = c.CurrencyPairFormat + } + if c.SyncManagerConfig.TimeoutREST <= 0 { + log.Warnf(log.ConfigMgr, "invalid sync manager REST timeout value %v, defaulting to %v\n", c.SyncManagerConfig.PairFormatDisplay, c.CurrencyPairFormat) + c.SyncManagerConfig.TimeoutREST = DefaultSyncerTimeoutREST + } + if c.SyncManagerConfig.NumWorkers <= 0 { + log.Warnf(log.ConfigMgr, "invalid sync manager worker count value %v, defaulting to %v\n", c.SyncManagerConfig.PairFormatDisplay, DefaultSyncerWorkers) + c.SyncManagerConfig.NumWorkers = DefaultSyncerWorkers + } +} + // GetEnabledPairs returns a list of currency pairs for a specific exchange func (c *Config) GetEnabledPairs(exchName string, assetType asset.Item) (currency.Pairs, error) { exchCfg, err := c.GetExchangeConfig(exchName) @@ -1730,6 +1780,7 @@ func (c *Config) CheckConfig() error { c.CheckClientBankAccounts() c.CheckBankAccountConfig() c.CheckRemoteControlConfig() + c.CheckSyncManagerConfig() err = c.CheckCurrencyConfigValues() if err != nil { diff --git a/config/config_test.go b/config/config_test.go index a8dda946194..ed7ce1092ec 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -2299,3 +2299,14 @@ func TestExchangeConfigValidate(t *testing.T) { t.Fatalf("received: '%v' but expected: '%v'", err, nil) } } + +func TestGetDefaultSyncManagerConfig(t *testing.T) { + t.Parallel() + cfg := GetDefaultSyncManagerConfig() + if cfg == (SyncManagerConfig{}) { + t.Error("expected config") + } + if cfg.TimeoutREST != DefaultSyncerTimeoutREST { + t.Errorf("expected %v, received %v", DefaultSyncerTimeoutREST, cfg.TimeoutREST) + } +} diff --git a/config/config_types.go b/config/config_types.go index 88e0f4a5d79..4f853eef918 100644 --- a/config/config_types.go +++ b/config/config_types.go @@ -41,6 +41,12 @@ const ( defaultCurrencyStateManagerDelay = time.Minute defaultMaxJobsPerCycle = 5 DefaultOrderbookPublishPeriod = time.Second * 10 + // DefaultSyncerWorkers limits the number of sync workers + DefaultSyncerWorkers = 15 + // DefaultSyncerTimeoutREST the default time to switch from REST to websocket protocols without a response + DefaultSyncerTimeoutREST = time.Second * 15 + // DefaultSyncerTimeoutWebsocket the default time to switch from websocket to REST protocols without a response + DefaultSyncerTimeoutWebsocket = time.Minute ) // Constants here hold some messages @@ -81,6 +87,7 @@ type Config struct { GlobalHTTPTimeout time.Duration `json:"globalHTTPTimeout"` Database database.Config `json:"database"` Logging log.Config `json:"logging"` + SyncManagerConfig SyncManagerConfig `json:"syncManager"` ConnectionMonitor ConnectionMonitorConfig `json:"connectionMonitor"` OrderManager OrderManager `json:"orderManager"` DataHistoryManager DataHistoryManager `json:"dataHistoryManager"` @@ -130,6 +137,25 @@ type CurrencyStateManager struct { Delay time.Duration `json:"delay"` } +// SyncManagerConfig stores the currency pair synchronization manager config +type SyncManagerConfig struct { + Enabled bool `json:"enabled"` + SynchronizeTicker bool `json:"synchronizeTicker"` + SynchronizeOrderbook bool `json:"synchronizeOrderbook"` + SynchronizeTrades bool `json:"synchronizeTrades"` + SynchronizeContinuously bool `json:"synchronizeContinuously"` + TimeoutREST time.Duration `json:"timeoutREST"` + TimeoutWebsocket time.Duration `json:"timeoutWebsocket"` + NumWorkers int `json:"numWorkers"` + FiatDisplayCurrency currency.Code `json:"fiatDisplayCurrency"` + PairFormatDisplay *currency.PairFormat `json:"pairFormatDisplay,omitempty"` + // log events + Verbose bool `json:"verbose"` + LogSyncUpdateEvents bool `json:"logSyncUpdateEvents"` + LogSwitchProtocolEvents bool `json:"logSwitchProtocolEvents"` + LogInitialSyncEvents bool `json:"logInitialSyncEvents"` +} + // ConnectionMonitorConfig defines the connection monitor variables to ensure // that there is internet connectivity type ConnectionMonitorConfig struct { diff --git a/engine/engine.go b/engine/engine.go index 2331f6c324c..4329b53f6a7 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -491,21 +491,36 @@ func (bot *Engine) Start() error { } if bot.Settings.EnableExchangeSyncManager { - exchangeSyncCfg := &SyncManagerConfig{ - SynchronizeTicker: bot.Settings.EnableTickerSyncing, - SynchronizeOrderbook: bot.Settings.EnableOrderbookSyncing, - SynchronizeTrades: bot.Settings.EnableTradeSyncing, - SynchronizeContinuously: bot.Settings.SyncContinuously, - TimeoutREST: bot.Settings.SyncTimeoutREST, - TimeoutWebsocket: bot.Settings.SyncTimeoutWebsocket, - NumWorkers: bot.Settings.SyncWorkersCount, - Verbose: bot.Settings.Verbose, - FiatDisplayCurrency: bot.Config.Currency.FiatDisplayCurrency, - PairFormatDisplay: bot.Config.Currency.CurrencyPairFormat, + cfg := bot.Config.SyncManagerConfig + if !bot.Settings.EnableTickerSyncing { + cfg.SynchronizeTicker = false + } + if !bot.Settings.EnableOrderbookSyncing { + cfg.SynchronizeOrderbook = false + } + if !bot.Settings.EnableTradeSyncing { + cfg.SynchronizeTrades = false + } + if !bot.Settings.SyncContinuously { + cfg.SynchronizeContinuously = false + } + if cfg.TimeoutREST != bot.Settings.SyncTimeoutREST && + bot.Settings.SyncTimeoutREST != config.DefaultSyncerTimeoutREST { + cfg.TimeoutREST = bot.Settings.SyncTimeoutREST + } + if cfg.TimeoutWebsocket != bot.Settings.SyncTimeoutWebsocket && + bot.Settings.SyncTimeoutWebsocket != config.DefaultSyncerTimeoutWebsocket { + cfg.TimeoutWebsocket = bot.Settings.SyncTimeoutWebsocket + } + if cfg.NumWorkers != bot.Settings.SyncWorkersCount && + bot.Settings.SyncWorkersCount != config.DefaultSyncerWorkers { + cfg.NumWorkers = bot.Settings.SyncWorkersCount + } + if bot.Settings.Verbose { + cfg.Verbose = true } - if s, err := setupSyncManager( - exchangeSyncCfg, + &cfg, bot.ExchangeManager, &bot.Config.RemoteControl, bot.Settings.EnableWebsocketRoutine, diff --git a/engine/helpers.go b/engine/helpers.go index 1b8a420b13a..905356ea868 100644 --- a/engine/helpers.go +++ b/engine/helpers.go @@ -185,19 +185,33 @@ func (bot *Engine) SetSubsystem(subSystemName string, enable bool) error { case SyncManagerName: if enable { if bot.currencyPairSyncer == nil { - exchangeSyncCfg := &SyncManagerConfig{ - SynchronizeTicker: bot.Settings.EnableTickerSyncing, - SynchronizeOrderbook: bot.Settings.EnableOrderbookSyncing, - SynchronizeTrades: bot.Settings.EnableTradeSyncing, - SynchronizeContinuously: bot.Settings.SyncContinuously, - TimeoutREST: bot.Settings.SyncTimeoutREST, - TimeoutWebsocket: bot.Settings.SyncTimeoutWebsocket, - NumWorkers: bot.Settings.SyncWorkersCount, - FiatDisplayCurrency: bot.Config.Currency.FiatDisplayCurrency, - Verbose: bot.Settings.Verbose, + cfg := bot.Config.SyncManagerConfig + if !bot.Settings.EnableTickerSyncing { + cfg.SynchronizeTicker = false + } + if !bot.Settings.EnableOrderbookSyncing { + cfg.SynchronizeOrderbook = false + } + if !bot.Settings.EnableTradeSyncing { + cfg.SynchronizeTrades = false + } + if !bot.Settings.SyncContinuously { + cfg.SynchronizeContinuously = false + } + if cfg.TimeoutREST != bot.Settings.SyncTimeoutREST && bot.Settings.SyncTimeoutREST != config.DefaultSyncerTimeoutREST { + cfg.TimeoutREST = bot.Settings.SyncTimeoutREST + } + if cfg.TimeoutWebsocket != bot.Settings.SyncTimeoutWebsocket && bot.Settings.SyncTimeoutWebsocket != config.DefaultSyncerTimeoutWebsocket { + cfg.TimeoutWebsocket = bot.Settings.SyncTimeoutWebsocket + } + if cfg.NumWorkers != bot.Settings.SyncWorkersCount && bot.Settings.SyncWorkersCount != config.DefaultSyncerWorkers { + cfg.NumWorkers = bot.Settings.SyncWorkersCount + } + if bot.Settings.Verbose { + cfg.Verbose = true } bot.currencyPairSyncer, err = setupSyncManager( - exchangeSyncCfg, + &cfg, bot.ExchangeManager, &bot.Config.RemoteControl, bot.Settings.EnableWebsocketRoutine) diff --git a/engine/sync_manager.go b/engine/sync_manager.go index d4b8d2b7579..7e40a022f53 100644 --- a/engine/sync_manager.go +++ b/engine/sync_manager.go @@ -32,21 +32,15 @@ const ( ) var ( - createdCounter = 0 - removedCounter = 0 - // DefaultSyncerWorkers limits the number of sync workers - DefaultSyncerWorkers = 15 - // DefaultSyncerTimeoutREST the default time to switch from REST to websocket protocols without a response - DefaultSyncerTimeoutREST = time.Second * 15 - // DefaultSyncerTimeoutWebsocket the default time to switch from websocket to REST protocols without a response - DefaultSyncerTimeoutWebsocket = time.Minute - errNoSyncItemsEnabled = errors.New("no sync items enabled") - errUnknownSyncItem = errors.New("unknown sync item") - errCouldNotSyncNewData = errors.New("could not sync new data") + createdCounter = 0 + removedCounter = 0 + errNoSyncItemsEnabled = errors.New("no sync items enabled") + errUnknownSyncItem = errors.New("unknown sync item") + errCouldNotSyncNewData = errors.New("could not sync new data") ) // setupSyncManager starts a new CurrencyPairSyncer -func setupSyncManager(c *SyncManagerConfig, exchangeManager iExchangeManager, remoteConfig *config.RemoteControlConfig, websocketRoutineManagerEnabled bool) (*syncManager, error) { +func setupSyncManager(c *config.SyncManagerConfig, exchangeManager iExchangeManager, remoteConfig *config.RemoteControlConfig, websocketRoutineManagerEnabled bool) (*syncManager, error) { if c == nil { return nil, fmt.Errorf("%T %w", c, common.ErrNilPointer) } @@ -62,15 +56,15 @@ func setupSyncManager(c *SyncManagerConfig, exchangeManager iExchangeManager, re } if c.NumWorkers <= 0 { - c.NumWorkers = DefaultSyncerWorkers + c.NumWorkers = config.DefaultSyncerWorkers } if c.TimeoutREST <= time.Duration(0) { - c.TimeoutREST = DefaultSyncerTimeoutREST + c.TimeoutREST = config.DefaultSyncerTimeoutREST } if c.TimeoutWebsocket <= time.Duration(0) { - c.TimeoutWebsocket = DefaultSyncerTimeoutWebsocket + c.TimeoutWebsocket = config.DefaultSyncerTimeoutWebsocket } if c.FiatDisplayCurrency.IsEmpty() { @@ -196,19 +190,25 @@ func (m *syncManager) Start() error { } if atomic.CompareAndSwapInt32(&m.initSyncStarted, 0, 1) { - log.Debugf(log.SyncMgr, - "Exchange CurrencyPairSyncer initial sync started. %d items to process.", - createdCounter) + if m.config.LogInitialSyncEvents { + log.Debugf(log.SyncMgr, + "Exchange CurrencyPairSyncer initial sync started. %d items to process.", + createdCounter) + } m.initSyncStartTime = time.Now() } go func() { m.initSyncWG.Wait() if atomic.CompareAndSwapInt32(&m.initSyncCompleted, 0, 1) { - log.Debugf(log.SyncMgr, "Exchange CurrencyPairSyncer initial sync is complete.") + if m.config.LogInitialSyncEvents { + log.Debugf(log.SyncMgr, "Exchange CurrencyPairSyncer initial sync is complete.") + } completedTime := time.Now() - log.Debugf(log.SyncMgr, "Exchange CurrencyPairSyncer initial sync took %v [%v sync items].", - completedTime.Sub(m.initSyncStartTime), createdCounter) + if m.config.LogInitialSyncEvents { + log.Debugf(log.SyncMgr, "Exchange CurrencyPairSyncer initial sync took %v [%v sync items].", + completedTime.Sub(m.initSyncStartTime), createdCounter) + } if !m.config.SynchronizeContinuously { log.Debugln(log.SyncMgr, "Exchange CurrencyPairSyncer stopping.") @@ -382,13 +382,15 @@ func (m *syncManager) WebsocketUpdate(exchangeName string, p currency.Pair, a as if !s.IsUsingWebsocket { s.IsUsingWebsocket = true s.IsUsingREST = false - log.Warnf(log.SyncMgr, - "%s %s %s: %s Websocket re-enabled, switching from rest to websocket", - c.Exchange, - m.FormatCurrency(c.Pair), - strings.ToUpper(c.AssetType.String()), - syncType, - ) + if m.config.LogSwitchProtocolEvents { + log.Warnf(log.SyncMgr, + "%s %s %s: %s Websocket re-enabled, switching from rest to websocket", + c.Exchange, + m.FormatCurrency(c.Pair), + strings.ToUpper(c.AssetType.String()), + syncType, + ) + } } return m.update(c, syncType, err) @@ -410,12 +412,14 @@ func (m *syncManager) update(c *currencyPairSyncAgent, syncType syncItemType, er s.HaveData = true if atomic.LoadInt32(&m.initSyncCompleted) != 1 && !origHadData { removedCounter++ - log.Debugf(log.SyncMgr, "%s %s sync complete %v [%d/%d].", - c.Exchange, - syncType, - m.FormatCurrency(c.Pair), - removedCounter, - createdCounter) + if m.config.LogInitialSyncEvents { + log.Debugf(log.SyncMgr, "%s %s sync complete %v [%d/%d].", + c.Exchange, + syncType, + m.FormatCurrency(c.Pair), + removedCounter, + createdCounter) + } m.initSyncWG.Done() } @@ -532,13 +536,15 @@ func (m *syncManager) syncTicker(c *currencyPairSyncAgent, e exchange.IBotExchan // Downgrade to REST s.IsUsingWebsocket = false s.IsUsingREST = true - log.Warnf(log.SyncMgr, - "%s %s %s: No ticker update after %s, switching from websocket to rest", - c.Exchange, - m.FormatCurrency(c.Pair), - strings.ToUpper(c.AssetType.String()), - m.config.TimeoutWebsocket, - ) + if m.config.LogSwitchProtocolEvents { + log.Warnf(log.SyncMgr, + "%s %s %s: No ticker update after %s, switching from websocket to rest", + c.Exchange, + m.FormatCurrency(c.Pair), + strings.ToUpper(c.AssetType.String()), + m.config.TimeoutWebsocket, + ) + } } if s.IsUsingREST && time.Since(s.LastUpdated) > m.config.TimeoutREST { @@ -605,13 +611,15 @@ func (m *syncManager) syncOrderbook(c *currencyPairSyncAgent, e exchange.IBotExc // Downgrade to REST s.IsUsingWebsocket = false s.IsUsingREST = true - log.Warnf(log.SyncMgr, - "%s %s %s: No orderbook update after %s, switching from websocket to rest", - c.Exchange, - m.FormatCurrency(c.Pair).String(), - strings.ToUpper(c.AssetType.String()), - m.config.TimeoutWebsocket, - ) + if m.config.LogSwitchProtocolEvents { + log.Warnf(log.SyncMgr, + "%s %s %s: No orderbook update after %s, switching from websocket to rest", + c.Exchange, + m.FormatCurrency(c.Pair).String(), + strings.ToUpper(c.AssetType.String()), + m.config.TimeoutWebsocket, + ) + } } if s.IsUsingREST && time.Since(s.LastUpdated) > m.config.TimeoutREST { @@ -706,6 +714,9 @@ func (m *syncManager) PrintTickerSummary(result *ticker.Price, protocol string, // ignoring error as not all tickers have volume populated and error is not actionable _ = stats.Add(result.ExchangeName, result.Pair, result.AssetType, result.Last, result.Volume) + if !m.config.LogSyncUpdateEvents { + return + } if result.Pair.Quote.IsFiatCurrency() && !result.Pair.Quote.Equal(m.fiatDisplayCurrency) && @@ -795,7 +806,9 @@ func (m *syncManager) PrintOrderbookSummary(result *orderbook.Base, protocol str err) return } - + if !m.config.LogSyncUpdateEvents { + return + } bidsAmount, bidsValue := result.TotalBidsAmount() asksAmount, asksValue := result.TotalAsksAmount() diff --git a/engine/sync_manager_test.go b/engine/sync_manager_test.go index 2dfe02106ac..adf49e761d6 100644 --- a/engine/sync_manager_test.go +++ b/engine/sync_manager_test.go @@ -20,37 +20,37 @@ func TestSetupSyncManager(t *testing.T) { t.Errorf("error '%v', expected '%v'", err, common.ErrNilPointer) } - _, err = setupSyncManager(&SyncManagerConfig{}, nil, nil, false) + _, err = setupSyncManager(&config.SyncManagerConfig{}, nil, nil, false) if !errors.Is(err, errNoSyncItemsEnabled) { t.Errorf("error '%v', expected '%v'", err, errNoSyncItemsEnabled) } - _, err = setupSyncManager(&SyncManagerConfig{SynchronizeTrades: true}, nil, nil, false) + _, err = setupSyncManager(&config.SyncManagerConfig{SynchronizeTrades: true}, nil, nil, false) if !errors.Is(err, errNilExchangeManager) { t.Errorf("error '%v', expected '%v'", err, errNilExchangeManager) } - _, err = setupSyncManager(&SyncManagerConfig{SynchronizeTrades: true}, &ExchangeManager{}, nil, false) + _, err = setupSyncManager(&config.SyncManagerConfig{SynchronizeTrades: true}, &ExchangeManager{}, nil, false) if !errors.Is(err, errNilConfig) { t.Errorf("error '%v', expected '%v'", err, errNilConfig) } - _, err = setupSyncManager(&SyncManagerConfig{SynchronizeTrades: true}, &ExchangeManager{}, &config.RemoteControlConfig{}, true) + _, err = setupSyncManager(&config.SyncManagerConfig{SynchronizeTrades: true}, &ExchangeManager{}, &config.RemoteControlConfig{}, true) if !errors.Is(err, currency.ErrCurrencyCodeEmpty) { t.Errorf("error '%v', expected '%v'", err, currency.ErrCurrencyCodeEmpty) } - _, err = setupSyncManager(&SyncManagerConfig{SynchronizeTrades: true, FiatDisplayCurrency: currency.BTC}, &ExchangeManager{}, &config.RemoteControlConfig{}, true) + _, err = setupSyncManager(&config.SyncManagerConfig{SynchronizeTrades: true, FiatDisplayCurrency: currency.BTC}, &ExchangeManager{}, &config.RemoteControlConfig{}, true) if !errors.Is(err, currency.ErrFiatDisplayCurrencyIsNotFiat) { t.Errorf("error '%v', expected '%v'", err, currency.ErrFiatDisplayCurrencyIsNotFiat) } - _, err = setupSyncManager(&SyncManagerConfig{SynchronizeTrades: true, FiatDisplayCurrency: currency.USD}, &ExchangeManager{}, &config.RemoteControlConfig{}, true) + _, err = setupSyncManager(&config.SyncManagerConfig{SynchronizeTrades: true, FiatDisplayCurrency: currency.USD}, &ExchangeManager{}, &config.RemoteControlConfig{}, true) if !errors.Is(err, common.ErrNilPointer) { t.Errorf("error '%v', expected '%v'", err, common.ErrNilPointer) } - m, err := setupSyncManager(&SyncManagerConfig{SynchronizeTrades: true, FiatDisplayCurrency: currency.USD, PairFormatDisplay: ¤cy.EMPTYFORMAT}, &ExchangeManager{}, &config.RemoteControlConfig{}, true) + m, err := setupSyncManager(&config.SyncManagerConfig{SynchronizeTrades: true, FiatDisplayCurrency: currency.USD, PairFormatDisplay: ¤cy.EMPTYFORMAT}, &ExchangeManager{}, &config.RemoteControlConfig{}, true) if !errors.Is(err, nil) { t.Errorf("error '%v', expected '%v'", err, nil) } @@ -61,7 +61,7 @@ func TestSetupSyncManager(t *testing.T) { func TestSyncManagerStart(t *testing.T) { t.Parallel() - m, err := setupSyncManager(&SyncManagerConfig{SynchronizeTrades: true, FiatDisplayCurrency: currency.USD, PairFormatDisplay: ¤cy.EMPTYFORMAT}, &ExchangeManager{}, &config.RemoteControlConfig{}, true) + m, err := setupSyncManager(&config.SyncManagerConfig{SynchronizeTrades: true, FiatDisplayCurrency: currency.USD, PairFormatDisplay: ¤cy.EMPTYFORMAT}, &ExchangeManager{}, &config.RemoteControlConfig{}, true) if !errors.Is(err, nil) { t.Errorf("error '%v', expected '%v'", err, nil) } @@ -112,7 +112,7 @@ func TestSyncManagerStop(t *testing.T) { if !errors.Is(err, nil) { t.Fatalf("received: '%v' but expected: '%v'", err, nil) } - m, err = setupSyncManager(&SyncManagerConfig{SynchronizeTrades: true, SynchronizeContinuously: true, FiatDisplayCurrency: currency.USD, PairFormatDisplay: ¤cy.EMPTYFORMAT}, em, &config.RemoteControlConfig{}, false) + m, err = setupSyncManager(&config.SyncManagerConfig{SynchronizeTrades: true, SynchronizeContinuously: true, FiatDisplayCurrency: currency.USD, PairFormatDisplay: ¤cy.EMPTYFORMAT}, em, &config.RemoteControlConfig{}, false) if !errors.Is(err, nil) { t.Errorf("error '%v', expected '%v'", err, nil) } @@ -163,7 +163,7 @@ func TestPrintTickerSummary(t *testing.T) { if !errors.Is(err, nil) { t.Fatalf("received: '%v' but expected: '%v'", err, nil) } - m, err = setupSyncManager(&SyncManagerConfig{SynchronizeTrades: true, SynchronizeContinuously: true, FiatDisplayCurrency: currency.USD, PairFormatDisplay: ¤cy.EMPTYFORMAT}, em, &config.RemoteControlConfig{}, false) + m, err = setupSyncManager(&config.SyncManagerConfig{SynchronizeTrades: true, SynchronizeContinuously: true, FiatDisplayCurrency: currency.USD, PairFormatDisplay: ¤cy.EMPTYFORMAT}, em, &config.RemoteControlConfig{}, false) if !errors.Is(err, nil) { t.Errorf("error '%v', expected '%v'", err, nil) } @@ -205,7 +205,7 @@ func TestPrintOrderbookSummary(t *testing.T) { if !errors.Is(err, nil) { t.Fatalf("received: '%v' but expected: '%v'", err, nil) } - m, err = setupSyncManager(&SyncManagerConfig{SynchronizeTrades: true, SynchronizeContinuously: true, FiatDisplayCurrency: currency.USD, PairFormatDisplay: ¤cy.EMPTYFORMAT}, em, &config.RemoteControlConfig{}, false) + m, err = setupSyncManager(&config.SyncManagerConfig{SynchronizeTrades: true, SynchronizeContinuously: true, FiatDisplayCurrency: currency.USD, PairFormatDisplay: ¤cy.EMPTYFORMAT}, em, &config.RemoteControlConfig{}, false) if !errors.Is(err, nil) { t.Errorf("error '%v', expected '%v'", err, nil) } diff --git a/engine/sync_manager_types.go b/engine/sync_manager_types.go index fdd41178b02..aed9509ca78 100644 --- a/engine/sync_manager_types.go +++ b/engine/sync_manager_types.go @@ -33,20 +33,6 @@ type currencyPairSyncAgent struct { locks []sync.Mutex } -// SyncManagerConfig stores the currency pair synchronization manager config -type SyncManagerConfig struct { - SynchronizeTicker bool - SynchronizeOrderbook bool - SynchronizeTrades bool - SynchronizeContinuously bool - TimeoutREST time.Duration - TimeoutWebsocket time.Duration - NumWorkers int - FiatDisplayCurrency currency.Code - PairFormatDisplay *currency.PairFormat - Verbose bool -} - // syncManager stores the exchange currency pair syncer object type syncManager struct { initSyncCompleted int32 @@ -65,6 +51,6 @@ type syncManager struct { tickerBatchLastRequested map[string]time.Time remoteConfig *config.RemoteControlConfig - config SyncManagerConfig + config config.SyncManagerConfig exchangeManager iExchangeManager } diff --git a/main.go b/main.go index d0b1f545313..9712df26173 100644 --- a/main.go +++ b/main.go @@ -66,11 +66,11 @@ func main() { flag.BoolVar(&settings.EnableTickerSyncing, "tickersync", true, "enables ticker syncing for all enabled exchanges") flag.BoolVar(&settings.EnableOrderbookSyncing, "orderbooksync", true, "enables orderbook syncing for all enabled exchanges") flag.BoolVar(&settings.EnableTradeSyncing, "tradesync", false, "enables trade syncing for all enabled exchanges") - flag.IntVar(&settings.SyncWorkersCount, "syncworkers", engine.DefaultSyncerWorkers, "the amount of workers (goroutines) to use for syncing exchange data") + flag.IntVar(&settings.SyncWorkersCount, "syncworkers", config.DefaultSyncerWorkers, "the amount of workers (goroutines) to use for syncing exchange data") flag.BoolVar(&settings.SyncContinuously, "synccontinuously", true, "whether to sync exchange data continuously (ticker, orderbook and trade history info") - flag.DurationVar(&settings.SyncTimeoutREST, "synctimeoutrest", engine.DefaultSyncerTimeoutREST, + flag.DurationVar(&settings.SyncTimeoutREST, "synctimeoutrest", config.DefaultSyncerTimeoutREST, "the amount of time before the syncer will switch from rest protocol to the streaming protocol (e.g. from REST to websocket)") - flag.DurationVar(&settings.SyncTimeoutWebsocket, "synctimeoutwebsocket", engine.DefaultSyncerTimeoutWebsocket, + flag.DurationVar(&settings.SyncTimeoutWebsocket, "synctimeoutwebsocket", config.DefaultSyncerTimeoutWebsocket, "the amount of time before the syncer will switch from the websocket protocol to REST protocol (e.g. from websocket to REST)") // Forex provider settings From 9a0481886deaff7568375f17a07db77b267114c6 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 22 Aug 2023 16:30:34 +1000 Subject: [PATCH 02/13] config-example add --- config_example.json | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/config_example.json b/config_example.json index 2251d2a929b..9b5ca286731 100644 --- a/config_example.json +++ b/config_example.json @@ -37,6 +37,25 @@ } } }, + "syncManager": { + "enabled": true, + "synchronizeTicker": true, + "synchronizeOrderbook": true, + "synchronizeTrades": true, + "synchronizeContinuously": true, + "timeoutREST": 15000000000, + "timeoutWebsocket": 60000000000, + "numWorkers": 15, + "fiatDisplayCurrency": "USD", + "pairFormatDisplay": { + "uppercase": true, + "delimiter": "-" + }, + "verbose": false, + "logSyncUpdateEvents": true, + "logSwitchProtocolEvents": true, + "logInitialSyncEvents": true + }, "connectionMonitor": { "preferredDNSList": [ "8.8.8.8", From e46f1f965d10700aef9734c8745d5a75821fcf9e Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 22 Aug 2023 16:49:52 +1000 Subject: [PATCH 03/13] who doesnt like more coverage? --- config/config_test.go | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/config/config_test.go b/config/config_test.go index ed7ce1092ec..451dd0b2a29 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -2310,3 +2310,35 @@ func TestGetDefaultSyncManagerConfig(t *testing.T) { t.Errorf("expected %v, received %v", DefaultSyncerTimeoutREST, cfg.TimeoutREST) } } + +func TestCheckSyncManagerConfig(t *testing.T) { + t.Parallel() + c := Config{} + if c.SyncManagerConfig != (SyncManagerConfig{}) { + t.Error("expected empty config") + } + c.CheckSyncManagerConfig() + if c.SyncManagerConfig.TimeoutREST != DefaultSyncerTimeoutREST { + t.Error("expected default config") + } + c.SyncManagerConfig.TimeoutWebsocket = -1 + c.SyncManagerConfig.PairFormatDisplay = nil + c.SyncManagerConfig.TimeoutREST = -1 + c.SyncManagerConfig.NumWorkers = -1 + c.CurrencyPairFormat = ¤cy.PairFormat{ + Uppercase: true, + } + c.CheckSyncManagerConfig() + if c.SyncManagerConfig.TimeoutWebsocket != DefaultSyncerTimeoutWebsocket { + t.Errorf("received %v expected %v", c.SyncManagerConfig.TimeoutWebsocket, DefaultSyncerTimeoutWebsocket) + } + if c.SyncManagerConfig.PairFormatDisplay == nil { + t.Errorf("received %v expected %v", c.SyncManagerConfig.PairFormatDisplay, c.CurrencyPairFormat) + } + if c.SyncManagerConfig.TimeoutREST != DefaultSyncerTimeoutREST { + t.Errorf("received %v expected %v", c.SyncManagerConfig.TimeoutREST, DefaultSyncerTimeoutREST) + } + if c.SyncManagerConfig.NumWorkers != DefaultSyncerWorkers { + t.Errorf("received %v expected %v", c.SyncManagerConfig.NumWorkers, DefaultSyncerWorkers) + } +} From 46c69f575f9042b4b4187d2ba829f34a40f198be Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 22 Aug 2023 16:53:52 +1000 Subject: [PATCH 04/13] ensures you can actually disable it via config el oh el --- engine/engine.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/engine/engine.go b/engine/engine.go index 4329b53f6a7..c712eecc2d3 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -490,7 +490,7 @@ func (bot *Engine) Start() error { } } - if bot.Settings.EnableExchangeSyncManager { + if bot.Config.SyncManagerConfig.Enabled && bot.Settings.EnableExchangeSyncManager { cfg := bot.Config.SyncManagerConfig if !bot.Settings.EnableTickerSyncing { cfg.SynchronizeTicker = false From a2ed2cc0240d850244bff71fd32301da3b6115df Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 22 Aug 2023 16:57:41 +1000 Subject: [PATCH 05/13] less ifs, better control --- engine/engine.go | 21 ++++++--------------- engine/helpers.go | 30 ++++++++++++------------------ 2 files changed, 18 insertions(+), 33 deletions(-) diff --git a/engine/engine.go b/engine/engine.go index c712eecc2d3..5fc403a2253 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -492,18 +492,12 @@ func (bot *Engine) Start() error { if bot.Config.SyncManagerConfig.Enabled && bot.Settings.EnableExchangeSyncManager { cfg := bot.Config.SyncManagerConfig - if !bot.Settings.EnableTickerSyncing { - cfg.SynchronizeTicker = false - } - if !bot.Settings.EnableOrderbookSyncing { - cfg.SynchronizeOrderbook = false - } - if !bot.Settings.EnableTradeSyncing { - cfg.SynchronizeTrades = false - } - if !bot.Settings.SyncContinuously { - cfg.SynchronizeContinuously = false - } + cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing && cfg.SynchronizeTicker + cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing && cfg.SynchronizeOrderbook + cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing && cfg.SynchronizeTrades + cfg.SynchronizeContinuously = bot.Settings.SyncContinuously && cfg.SynchronizeContinuously + cfg.Verbose = bot.Settings.Verbose && cfg.Verbose + if cfg.TimeoutREST != bot.Settings.SyncTimeoutREST && bot.Settings.SyncTimeoutREST != config.DefaultSyncerTimeoutREST { cfg.TimeoutREST = bot.Settings.SyncTimeoutREST @@ -516,9 +510,6 @@ func (bot *Engine) Start() error { bot.Settings.SyncWorkersCount != config.DefaultSyncerWorkers { cfg.NumWorkers = bot.Settings.SyncWorkersCount } - if bot.Settings.Verbose { - cfg.Verbose = true - } if s, err := setupSyncManager( &cfg, bot.ExchangeManager, diff --git a/engine/helpers.go b/engine/helpers.go index 905356ea868..cfb8279f4b6 100644 --- a/engine/helpers.go +++ b/engine/helpers.go @@ -186,30 +186,24 @@ func (bot *Engine) SetSubsystem(subSystemName string, enable bool) error { if enable { if bot.currencyPairSyncer == nil { cfg := bot.Config.SyncManagerConfig - if !bot.Settings.EnableTickerSyncing { - cfg.SynchronizeTicker = false - } - if !bot.Settings.EnableOrderbookSyncing { - cfg.SynchronizeOrderbook = false - } - if !bot.Settings.EnableTradeSyncing { - cfg.SynchronizeTrades = false - } - if !bot.Settings.SyncContinuously { - cfg.SynchronizeContinuously = false - } - if cfg.TimeoutREST != bot.Settings.SyncTimeoutREST && bot.Settings.SyncTimeoutREST != config.DefaultSyncerTimeoutREST { + cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing && cfg.SynchronizeTicker + cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing && cfg.SynchronizeOrderbook + cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing && cfg.SynchronizeTrades + cfg.SynchronizeContinuously = bot.Settings.SyncContinuously && cfg.SynchronizeContinuously + cfg.Verbose = bot.Settings.Verbose && cfg.Verbose + + if cfg.TimeoutREST != bot.Settings.SyncTimeoutREST && + bot.Settings.SyncTimeoutREST != config.DefaultSyncerTimeoutREST { cfg.TimeoutREST = bot.Settings.SyncTimeoutREST } - if cfg.TimeoutWebsocket != bot.Settings.SyncTimeoutWebsocket && bot.Settings.SyncTimeoutWebsocket != config.DefaultSyncerTimeoutWebsocket { + if cfg.TimeoutWebsocket != bot.Settings.SyncTimeoutWebsocket && + bot.Settings.SyncTimeoutWebsocket != config.DefaultSyncerTimeoutWebsocket { cfg.TimeoutWebsocket = bot.Settings.SyncTimeoutWebsocket } - if cfg.NumWorkers != bot.Settings.SyncWorkersCount && bot.Settings.SyncWorkersCount != config.DefaultSyncerWorkers { + if cfg.NumWorkers != bot.Settings.SyncWorkersCount && + bot.Settings.SyncWorkersCount != config.DefaultSyncerWorkers { cfg.NumWorkers = bot.Settings.SyncWorkersCount } - if bot.Settings.Verbose { - cfg.Verbose = true - } bot.currencyPairSyncer, err = setupSyncManager( &cfg, bot.ExchangeManager, From ccb1bc0f4ef3f4fbbf51bcb9b44e90d43cba34d5 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 22 Aug 2023 16:59:42 +1000 Subject: [PATCH 06/13] fix verbose --- engine/engine.go | 2 +- engine/helpers.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/engine/engine.go b/engine/engine.go index 5fc403a2253..e209759d0bc 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -496,7 +496,7 @@ func (bot *Engine) Start() error { cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing && cfg.SynchronizeOrderbook cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing && cfg.SynchronizeTrades cfg.SynchronizeContinuously = bot.Settings.SyncContinuously && cfg.SynchronizeContinuously - cfg.Verbose = bot.Settings.Verbose && cfg.Verbose + cfg.Verbose = bot.Settings.Verbose || cfg.Verbose if cfg.TimeoutREST != bot.Settings.SyncTimeoutREST && bot.Settings.SyncTimeoutREST != config.DefaultSyncerTimeoutREST { diff --git a/engine/helpers.go b/engine/helpers.go index cfb8279f4b6..6f6c974ec6e 100644 --- a/engine/helpers.go +++ b/engine/helpers.go @@ -190,7 +190,7 @@ func (bot *Engine) SetSubsystem(subSystemName string, enable bool) error { cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing && cfg.SynchronizeOrderbook cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing && cfg.SynchronizeTrades cfg.SynchronizeContinuously = bot.Settings.SyncContinuously && cfg.SynchronizeContinuously - cfg.Verbose = bot.Settings.Verbose && cfg.Verbose + cfg.Verbose = bot.Settings.Verbose || cfg.Verbose if cfg.TimeoutREST != bot.Settings.SyncTimeoutREST && bot.Settings.SyncTimeoutREST != config.DefaultSyncerTimeoutREST { From 2e485fd5f1f406b4079924abde3a531eebfdeb22 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 23 Aug 2023 07:15:08 +1000 Subject: [PATCH 07/13] sync trades default false --- config/config.go | 2 +- engine/engine.go | 2 +- engine/helpers.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/config/config.go b/config/config.go index 7674abef211..1d8ab02b191 100644 --- a/config/config.go +++ b/config/config.go @@ -702,7 +702,7 @@ func GetDefaultSyncManagerConfig() SyncManagerConfig { Enabled: true, SynchronizeTicker: true, SynchronizeOrderbook: true, - SynchronizeTrades: true, + SynchronizeTrades: false, SynchronizeContinuously: true, TimeoutREST: DefaultSyncerTimeoutREST, TimeoutWebsocket: DefaultSyncerTimeoutWebsocket, diff --git a/engine/engine.go b/engine/engine.go index e209759d0bc..a9520ff7c03 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -494,8 +494,8 @@ func (bot *Engine) Start() error { cfg := bot.Config.SyncManagerConfig cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing && cfg.SynchronizeTicker cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing && cfg.SynchronizeOrderbook - cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing && cfg.SynchronizeTrades cfg.SynchronizeContinuously = bot.Settings.SyncContinuously && cfg.SynchronizeContinuously + cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing || cfg.SynchronizeTrades cfg.Verbose = bot.Settings.Verbose || cfg.Verbose if cfg.TimeoutREST != bot.Settings.SyncTimeoutREST && diff --git a/engine/helpers.go b/engine/helpers.go index 6f6c974ec6e..ac65ee1c9c4 100644 --- a/engine/helpers.go +++ b/engine/helpers.go @@ -188,8 +188,8 @@ func (bot *Engine) SetSubsystem(subSystemName string, enable bool) error { cfg := bot.Config.SyncManagerConfig cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing && cfg.SynchronizeTicker cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing && cfg.SynchronizeOrderbook - cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing && cfg.SynchronizeTrades cfg.SynchronizeContinuously = bot.Settings.SyncContinuously && cfg.SynchronizeContinuously + cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing || cfg.SynchronizeTrades cfg.Verbose = bot.Settings.Verbose || cfg.Verbose if cfg.TimeoutREST != bot.Settings.SyncTimeoutREST && From 56242f8571e100cc64076df2036c884ac48d3691 Mon Sep 17 00:00:00 2001 From: Scott Date: Wed, 23 Aug 2023 17:50:11 +1000 Subject: [PATCH 08/13] fix summary being printed when not enabled --- engine/sync_manager.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/engine/sync_manager.go b/engine/sync_manager.go index 7e40a022f53..84ddb4e36b8 100644 --- a/engine/sync_manager.go +++ b/engine/sync_manager.go @@ -114,6 +114,11 @@ func (m *syncManager) Start() error { if !atomic.CompareAndSwapInt32(&m.started, 0, 1) { return ErrSubSystemAlreadyStarted } + if !m.config.SynchronizeTicker && + !m.config.SynchronizeOrderbook && + !m.config.SynchronizeTrades { + return errNoSyncItemsEnabled + } m.shutdown = make(chan bool) m.initSyncWG.Add(1) m.inService.Done() @@ -699,6 +704,9 @@ func (m *syncManager) PrintTickerSummary(result *ticker.Price, protocol string, if m == nil || atomic.LoadInt32(&m.started) == 0 { return } + if !m.config.SynchronizeTicker { + return + } if err != nil { if err == common.ErrNotYetImplemented { log.Warnf(log.SyncMgr, "Failed to get %s ticker. Error: %s", @@ -782,6 +790,9 @@ func (m *syncManager) PrintOrderbookSummary(result *orderbook.Base, protocol str if m == nil || atomic.LoadInt32(&m.started) == 0 { return } + if !m.config.SynchronizeOrderbook { + return + } if err != nil { if result == nil { log.Errorf(log.OrderBook, "Failed to get %s orderbook. Error: %s", From c373c59559a88ad0b3cb68661c1c4cbd6bbd815b Mon Sep 17 00:00:00 2001 From: Scott Date: Thu, 24 Aug 2023 09:33:06 +1000 Subject: [PATCH 09/13] fixes config checker and output --- config/config.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/config/config.go b/config/config.go index 1d8ab02b191..6627de75c4e 100644 --- a/config/config.go +++ b/config/config.go @@ -733,15 +733,18 @@ func (c *Config) CheckSyncManagerConfig() { c.SyncManagerConfig.TimeoutWebsocket = DefaultSyncerTimeoutWebsocket } if c.SyncManagerConfig.PairFormatDisplay == nil { - log.Warnf(log.ConfigMgr, "invalid sync manager pair format value %v, defaulting to %v\n", c.SyncManagerConfig.PairFormatDisplay, c.CurrencyPairFormat) - c.SyncManagerConfig.PairFormatDisplay = c.CurrencyPairFormat + log.Warnf(log.ConfigMgr, "invalid sync manager pair format value %v, using default format eg BTC-USD\n", c.SyncManagerConfig.PairFormatDisplay) + c.SyncManagerConfig.PairFormatDisplay = ¤cy.PairFormat{ + Uppercase: true, + Delimiter: currency.DashDelimiter, + } } if c.SyncManagerConfig.TimeoutREST <= 0 { - log.Warnf(log.ConfigMgr, "invalid sync manager REST timeout value %v, defaulting to %v\n", c.SyncManagerConfig.PairFormatDisplay, c.CurrencyPairFormat) + log.Warnf(log.ConfigMgr, "invalid sync manager REST timeout value %v, defaulting to %v\n", c.SyncManagerConfig.TimeoutREST, DefaultSyncerTimeoutREST) c.SyncManagerConfig.TimeoutREST = DefaultSyncerTimeoutREST } if c.SyncManagerConfig.NumWorkers <= 0 { - log.Warnf(log.ConfigMgr, "invalid sync manager worker count value %v, defaulting to %v\n", c.SyncManagerConfig.PairFormatDisplay, DefaultSyncerWorkers) + log.Warnf(log.ConfigMgr, "invalid sync manager worker count value %v, defaulting to %v\n", c.SyncManagerConfig.NumWorkers, DefaultSyncerWorkers) c.SyncManagerConfig.NumWorkers = DefaultSyncerWorkers } } From f706f153dfc48655e82e90992d76b25ac4d43fe3 Mon Sep 17 00:00:00 2001 From: Scott Date: Thu, 24 Aug 2023 14:29:07 +1000 Subject: [PATCH 10/13] nits --- config/config.go | 4 ++++ engine/engine.go | 6 +++--- engine/helpers.go | 6 +++--- main.go | 8 ++++---- 4 files changed, 14 insertions(+), 10 deletions(-) diff --git a/config/config.go b/config/config.go index 6627de75c4e..e2f5763755b 100644 --- a/config/config.go +++ b/config/config.go @@ -747,6 +747,10 @@ func (c *Config) CheckSyncManagerConfig() { log.Warnf(log.ConfigMgr, "invalid sync manager worker count value %v, defaulting to %v\n", c.SyncManagerConfig.NumWorkers, DefaultSyncerWorkers) c.SyncManagerConfig.NumWorkers = DefaultSyncerWorkers } + if c.SyncManagerConfig.FiatDisplayCurrency.IsEmpty() { + log.Warnf(log.ConfigMgr, "invalid sync manager fiat display currency value, defaulting to %v\n", currency.USD) + c.SyncManagerConfig.FiatDisplayCurrency = currency.USD + } } // GetEnabledPairs returns a list of currency pairs for a specific exchange diff --git a/engine/engine.go b/engine/engine.go index a9520ff7c03..2510ee5b275 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -492,9 +492,9 @@ func (bot *Engine) Start() error { if bot.Config.SyncManagerConfig.Enabled && bot.Settings.EnableExchangeSyncManager { cfg := bot.Config.SyncManagerConfig - cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing && cfg.SynchronizeTicker - cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing && cfg.SynchronizeOrderbook - cfg.SynchronizeContinuously = bot.Settings.SyncContinuously && cfg.SynchronizeContinuously + cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing || cfg.SynchronizeTicker + cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing || cfg.SynchronizeOrderbook + cfg.SynchronizeContinuously = bot.Settings.SyncContinuously || cfg.SynchronizeContinuously cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing || cfg.SynchronizeTrades cfg.Verbose = bot.Settings.Verbose || cfg.Verbose diff --git a/engine/helpers.go b/engine/helpers.go index ac65ee1c9c4..fb1def47ef4 100644 --- a/engine/helpers.go +++ b/engine/helpers.go @@ -186,9 +186,9 @@ func (bot *Engine) SetSubsystem(subSystemName string, enable bool) error { if enable { if bot.currencyPairSyncer == nil { cfg := bot.Config.SyncManagerConfig - cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing && cfg.SynchronizeTicker - cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing && cfg.SynchronizeOrderbook - cfg.SynchronizeContinuously = bot.Settings.SyncContinuously && cfg.SynchronizeContinuously + cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing || cfg.SynchronizeTicker + cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing || cfg.SynchronizeOrderbook + cfg.SynchronizeContinuously = bot.Settings.SyncContinuously || cfg.SynchronizeContinuously cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing || cfg.SynchronizeTrades cfg.Verbose = bot.Settings.Verbose || cfg.Verbose diff --git a/main.go b/main.go index 9712df26173..2475c205f21 100644 --- a/main.go +++ b/main.go @@ -63,11 +63,11 @@ func main() { flag.IntVar(&settings.DispatchJobsLimit, "dispatchjobslimit", dispatch.DefaultJobsLimit, "sets the dispatch package max jobs limit") // Exchange syncer settings - flag.BoolVar(&settings.EnableTickerSyncing, "tickersync", true, "enables ticker syncing for all enabled exchanges") - flag.BoolVar(&settings.EnableOrderbookSyncing, "orderbooksync", true, "enables orderbook syncing for all enabled exchanges") - flag.BoolVar(&settings.EnableTradeSyncing, "tradesync", false, "enables trade syncing for all enabled exchanges") + flag.BoolVar(&settings.EnableTickerSyncing, "tickersync", false, "enables ticker syncing for all enabled exchanges, overriding false config value") + flag.BoolVar(&settings.EnableOrderbookSyncing, "orderbooksync", false, "enables orderbook syncing for all enabled exchanges, overriding false config value") + flag.BoolVar(&settings.EnableTradeSyncing, "tradesync", false, "enables trade syncing for all enabled exchanges, overriding false config value") flag.IntVar(&settings.SyncWorkersCount, "syncworkers", config.DefaultSyncerWorkers, "the amount of workers (goroutines) to use for syncing exchange data") - flag.BoolVar(&settings.SyncContinuously, "synccontinuously", true, "whether to sync exchange data continuously (ticker, orderbook and trade history info") + flag.BoolVar(&settings.SyncContinuously, "synccontinuously", false, "whether to sync exchange data continuously (ticker, orderbook and trade history info), overriding false config value") flag.DurationVar(&settings.SyncTimeoutREST, "synctimeoutrest", config.DefaultSyncerTimeoutREST, "the amount of time before the syncer will switch from rest protocol to the streaming protocol (e.g. from REST to websocket)") flag.DurationVar(&settings.SyncTimeoutWebsocket, "synctimeoutwebsocket", config.DefaultSyncerTimeoutWebsocket, From 27346d5ec673e0a83cbae536c6be680dad786382 Mon Sep 17 00:00:00 2001 From: Scott Date: Thu, 24 Aug 2023 14:52:13 +1000 Subject: [PATCH 11/13] I can put this behind me now --- engine/engine.go | 16 +++++++++++----- engine/helpers.go | 8 ++++---- main.go | 2 +- 3 files changed, 16 insertions(+), 10 deletions(-) diff --git a/engine/engine.go b/engine/engine.go index 2510ee5b275..28f609036ed 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -179,6 +179,12 @@ func validateSettings(b *Engine, s *Settings, flagSet FlagSet) { flagSet.WithBool("currencystatemanager", &b.Settings.EnableCurrencyStateManager, b.Config.CurrencyStateManager.Enabled != nil && *b.Config.CurrencyStateManager.Enabled) flagSet.WithBool("gctscriptmanager", &b.Settings.EnableGCTScriptManager, b.Config.GCTScript.Enabled) + flagSet.WithBool("tickersync", &b.Settings.EnableTickerSyncing, b.Config.SyncManagerConfig.SynchronizeTicker) + flagSet.WithBool("orderbooksync", &b.Settings.EnableOrderbookSyncing, b.Config.SyncManagerConfig.SynchronizeOrderbook) + flagSet.WithBool("tradesync", &b.Settings.EnableTradeSyncing, b.Config.SyncManagerConfig.SynchronizeTrades) + flagSet.WithBool("synccontinuously", &b.Settings.SyncContinuously, b.Config.SyncManagerConfig.SynchronizeContinuously) + flagSet.WithBool("syncmanager", &b.Settings.EnableExchangeSyncManager, b.Config.SyncManagerConfig.Enabled) + if b.Settings.EnablePortfolioManager && b.Settings.PortfolioManagerDelay <= 0 { b.Settings.PortfolioManagerDelay = PortfolioSleepDelay @@ -490,12 +496,12 @@ func (bot *Engine) Start() error { } } - if bot.Config.SyncManagerConfig.Enabled && bot.Settings.EnableExchangeSyncManager { + if bot.Settings.EnableExchangeSyncManager { cfg := bot.Config.SyncManagerConfig - cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing || cfg.SynchronizeTicker - cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing || cfg.SynchronizeOrderbook - cfg.SynchronizeContinuously = bot.Settings.SyncContinuously || cfg.SynchronizeContinuously - cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing || cfg.SynchronizeTrades + cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing + cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing + cfg.SynchronizeContinuously = bot.Settings.SyncContinuously + cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing cfg.Verbose = bot.Settings.Verbose || cfg.Verbose if cfg.TimeoutREST != bot.Settings.SyncTimeoutREST && diff --git a/engine/helpers.go b/engine/helpers.go index fb1def47ef4..2467af9969f 100644 --- a/engine/helpers.go +++ b/engine/helpers.go @@ -186,10 +186,10 @@ func (bot *Engine) SetSubsystem(subSystemName string, enable bool) error { if enable { if bot.currencyPairSyncer == nil { cfg := bot.Config.SyncManagerConfig - cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing || cfg.SynchronizeTicker - cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing || cfg.SynchronizeOrderbook - cfg.SynchronizeContinuously = bot.Settings.SyncContinuously || cfg.SynchronizeContinuously - cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing || cfg.SynchronizeTrades + cfg.SynchronizeTicker = bot.Settings.EnableTickerSyncing + cfg.SynchronizeOrderbook = bot.Settings.EnableOrderbookSyncing + cfg.SynchronizeContinuously = bot.Settings.SyncContinuously + cfg.SynchronizeTrades = bot.Settings.EnableTradeSyncing cfg.Verbose = bot.Settings.Verbose || cfg.Verbose if cfg.TimeoutREST != bot.Settings.SyncTimeoutREST && diff --git a/main.go b/main.go index 2475c205f21..fd0900eef91 100644 --- a/main.go +++ b/main.go @@ -46,7 +46,7 @@ func main() { flag.BoolVar(&settings.EnableCommsRelayer, "enablecommsrelayer", true, "enables available communications relayer") flag.BoolVar(&settings.Verbose, "verbose", false, "increases logging verbosity for GoCryptoTrader") flag.BoolVar(&settings.EnableFuturesTracking, "enablefuturestracking", true, "tracks futures orders PNL is supported by the exchange") - flag.BoolVar(&settings.EnableExchangeSyncManager, "syncmanager", true, "enables to exchange sync manager") + flag.BoolVar(&settings.EnableExchangeSyncManager, "syncmanager", false, "enables to exchange sync manager") flag.BoolVar(&settings.EnableWebsocketRoutine, "websocketroutine", true, "enables the websocket routine for all loaded exchanges") flag.BoolVar(&settings.EnableCoinmarketcapAnalysis, "coinmarketcap", false, "overrides config and runs currency analysis") flag.BoolVar(&settings.EnableEventManager, "eventmanager", true, "enables the event manager") From 8a82a775cd76c1866cfcd51a77c37eb400ea8829 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 29 Aug 2023 15:01:55 +1000 Subject: [PATCH 12/13] Fixed logCaSiNg Co-authored-by: Adrian Gallagher --- config/config.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/config/config.go b/config/config.go index e2f5763755b..440f2e16838 100644 --- a/config/config.go +++ b/config/config.go @@ -729,26 +729,26 @@ func (c *Config) CheckSyncManagerConfig() { return } if c.SyncManagerConfig.TimeoutWebsocket <= 0 { - log.Warnf(log.ConfigMgr, "invalid sync manager websocket timeout value %v, defaulting to %v\n", c.SyncManagerConfig.TimeoutWebsocket, DefaultSyncerTimeoutWebsocket) + log.Warnf(log.ConfigMgr, "Invalid sync manager websocket timeout value %v, defaulting to %v\n", c.SyncManagerConfig.TimeoutWebsocket, DefaultSyncerTimeoutWebsocket) c.SyncManagerConfig.TimeoutWebsocket = DefaultSyncerTimeoutWebsocket } if c.SyncManagerConfig.PairFormatDisplay == nil { - log.Warnf(log.ConfigMgr, "invalid sync manager pair format value %v, using default format eg BTC-USD\n", c.SyncManagerConfig.PairFormatDisplay) + log.Warnf(log.ConfigMgr, "Invalid sync manager pair format value %v, using default format eg BTC-USD\n", c.SyncManagerConfig.PairFormatDisplay) c.SyncManagerConfig.PairFormatDisplay = ¤cy.PairFormat{ Uppercase: true, Delimiter: currency.DashDelimiter, } } if c.SyncManagerConfig.TimeoutREST <= 0 { - log.Warnf(log.ConfigMgr, "invalid sync manager REST timeout value %v, defaulting to %v\n", c.SyncManagerConfig.TimeoutREST, DefaultSyncerTimeoutREST) + log.Warnf(log.ConfigMgr, "Invalid sync manager REST timeout value %v, defaulting to %v\n", c.SyncManagerConfig.TimeoutREST, DefaultSyncerTimeoutREST) c.SyncManagerConfig.TimeoutREST = DefaultSyncerTimeoutREST } if c.SyncManagerConfig.NumWorkers <= 0 { - log.Warnf(log.ConfigMgr, "invalid sync manager worker count value %v, defaulting to %v\n", c.SyncManagerConfig.NumWorkers, DefaultSyncerWorkers) + log.Warnf(log.ConfigMgr, "Invalid sync manager worker count value %v, defaulting to %v\n", c.SyncManagerConfig.NumWorkers, DefaultSyncerWorkers) c.SyncManagerConfig.NumWorkers = DefaultSyncerWorkers } if c.SyncManagerConfig.FiatDisplayCurrency.IsEmpty() { - log.Warnf(log.ConfigMgr, "invalid sync manager fiat display currency value, defaulting to %v\n", currency.USD) + log.Warnf(log.ConfigMgr, "Invalid sync manager fiat display currency value, defaulting to %v\n", currency.USD) c.SyncManagerConfig.FiatDisplayCurrency = currency.USD } } From d2d274df40fca17b5af8610cc63e3666311e15a3 Mon Sep 17 00:00:00 2001 From: Scott Date: Tue, 29 Aug 2023 15:13:11 +1000 Subject: [PATCH 13/13] combines if statements --- engine/sync_manager.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/engine/sync_manager.go b/engine/sync_manager.go index 84ddb4e36b8..5c4d829f000 100644 --- a/engine/sync_manager.go +++ b/engine/sync_manager.go @@ -208,11 +208,8 @@ func (m *syncManager) Start() error { if atomic.CompareAndSwapInt32(&m.initSyncCompleted, 0, 1) { if m.config.LogInitialSyncEvents { log.Debugf(log.SyncMgr, "Exchange CurrencyPairSyncer initial sync is complete.") - } - completedTime := time.Now() - if m.config.LogInitialSyncEvents { log.Debugf(log.SyncMgr, "Exchange CurrencyPairSyncer initial sync took %v [%v sync items].", - completedTime.Sub(m.initSyncStartTime), createdCounter) + time.Since(m.initSyncStartTime), createdCounter) } if !m.config.SynchronizeContinuously {