From e9da0e65d4d7911c5d64f99697f13616f430dd7b Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Thu, 2 Jul 2020 17:07:02 +0200 Subject: [PATCH 1/6] Relax requirement on Core URL flags They shouldn't be required when captive core is used --- services/horizon/cmd/db.go | 16 +++++++----- services/horizon/cmd/ingest.go | 45 ++++++++++++++++++++-------------- services/horizon/cmd/root.go | 22 ++++++++++++----- 3 files changed, 53 insertions(+), 30 deletions(-) diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 0ecdf6a420..95b80ba7d8 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -204,26 +204,30 @@ var dbReingestRangeCmd = &cobra.Command{ initRootConfig() - coreSession, err := db.Open("postgres", config.StellarCoreDatabaseURL) - if err != nil { - log.Fatalf("cannot open Core DB: %v", err) - } - horizonSession, err := db.Open("postgres", config.DatabaseURL) if err != nil { log.Fatalf("cannot open Horizon DB: %v", err) } ingestConfig := expingest.Config{ - CoreSession: coreSession, NetworkPassphrase: config.NetworkPassphrase, HistorySession: horizonSession, HistoryArchiveURL: config.HistoryArchiveURLs[0], MaxReingestRetries: int(retries), ReingesRetryBackoffSeconds: int(retryBackoffSeconds), } + if config.EnableCaptiveCoreIngestion { ingestConfig.StellarCorePath = config.StellarCoreBinaryPath + } else { + if config.StellarCoreDatabaseURL == "" { + log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName) + } + coreSession, err := db.Open("postgres", config.StellarCoreDatabaseURL) + if err != nil { + log.Fatalf("cannot open Core DB: %v", err) + } + ingestConfig.CoreSession = coreSession } if parallelWorkers < 2 { diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go index efcbc0eabd..0b2c8b4eb6 100644 --- a/services/horizon/cmd/ingest.go +++ b/services/horizon/cmd/ingest.go @@ -24,7 +24,7 @@ var ingestVerifyFrom, ingestVerifyTo, ingestVerifyDebugServerPort uint32 var ingestVerifyState bool var ingestVerifyRangeCmdOpts = []*support.ConfigOption{ - &support.ConfigOption{ + { Name: "from", ConfigKey: &ingestVerifyFrom, OptType: types.Uint32, @@ -32,7 +32,7 @@ var ingestVerifyRangeCmdOpts = []*support.ConfigOption{ FlagDefault: uint32(0), Usage: "first ledger of the range to ingest", }, - &support.ConfigOption{ + { Name: "to", ConfigKey: &ingestVerifyTo, OptType: types.Uint32, @@ -40,7 +40,7 @@ var ingestVerifyRangeCmdOpts = []*support.ConfigOption{ FlagDefault: uint32(0), Usage: "last ledger of the range to ingest", }, - &support.ConfigOption{ + { Name: "verify-state", ConfigKey: &ingestVerifyState, OptType: types.Bool, @@ -48,7 +48,7 @@ var ingestVerifyRangeCmdOpts = []*support.ConfigOption{ FlagDefault: false, Usage: "[optional] verifies state at the last ledger of the range when true", }, - &support.ConfigOption{ + { Name: "debug-server-port", ConfigKey: &ingestVerifyDebugServerPort, OptType: types.Uint32, @@ -83,11 +83,6 @@ var ingestVerifyRangeCmd = &cobra.Command{ }() } - coreSession, err := db.Open("postgres", config.StellarCoreDatabaseURL) - if err != nil { - log.Fatalf("cannot open Core DB: %v", err) - } - horizonSession, err := db.Open("postgres", config.DatabaseURL) if err != nil { log.Fatalf("cannot open Horizon DB: %v", err) @@ -102,13 +97,22 @@ var ingestVerifyRangeCmd = &cobra.Command{ } ingestConfig := expingest.Config{ - CoreSession: coreSession, NetworkPassphrase: config.NetworkPassphrase, HistorySession: horizonSession, HistoryArchiveURL: config.HistoryArchiveURLs[0], } if config.EnableCaptiveCoreIngestion { ingestConfig.StellarCorePath = config.StellarCoreBinaryPath + } else { + if config.StellarCoreDatabaseURL == "" { + log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName) + } + + coreSession, err := db.Open("postgres", config.StellarCoreDatabaseURL) + if err != nil { + log.Fatalf("cannot open Core DB: %v", err) + } + ingestConfig.CoreSession = coreSession } system, err := expingest.NewSystem(ingestConfig) @@ -132,7 +136,7 @@ var ingestVerifyRangeCmd = &cobra.Command{ var stressTestNumTransactions, stressTestChangesPerTransaction int var stressTestCmdOpts = []*support.ConfigOption{ - &support.ConfigOption{ + { Name: "transactions", ConfigKey: &stressTestNumTransactions, OptType: types.Int, @@ -140,7 +144,7 @@ var stressTestCmdOpts = []*support.ConfigOption{ FlagDefault: int(1000), Usage: "total number of transactions to ingest (at most 1000)", }, - &support.ConfigOption{ + { Name: "changes", ConfigKey: &stressTestChangesPerTransaction, OptType: types.Int, @@ -162,11 +166,6 @@ var ingestStressTestCmd = &cobra.Command{ initRootConfig() - coreSession, err := db.Open("postgres", config.StellarCoreDatabaseURL) - if err != nil { - log.Fatalf("cannot open Core DB: %v", err) - } - horizonSession, err := db.Open("postgres", config.DatabaseURL) if err != nil { log.Fatalf("cannot open Horizon DB: %v", err) @@ -181,13 +180,23 @@ var ingestStressTestCmd = &cobra.Command{ } ingestConfig := expingest.Config{ - CoreSession: coreSession, NetworkPassphrase: config.NetworkPassphrase, HistorySession: horizonSession, HistoryArchiveURL: config.HistoryArchiveURLs[0], } + if config.EnableCaptiveCoreIngestion { ingestConfig.StellarCorePath = config.StellarCoreBinaryPath + } else { + if config.StellarCoreDatabaseURL == "" { + log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName) + } + + coreSession, err := db.Open("postgres", config.StellarCoreDatabaseURL) + if err != nil { + log.Fatalf("cannot open Core DB: %v", err) + } + ingestConfig.CoreSession = coreSession } system, err := expingest.NewSystem(ingestConfig) diff --git a/services/horizon/cmd/root.go b/services/horizon/cmd/root.go index 8ed3975f6c..c68ff32349 100644 --- a/services/horizon/cmd/root.go +++ b/services/horizon/cmd/root.go @@ -20,6 +20,12 @@ import ( "github.com/stellar/throttled" ) +const ( + maxDBPingAttempts = 30 + stellarCoreDBURLFlagName = "stellar-core-db-url" + stellarCoreURLFlagName = "stellar-core-url" +) + var ( config horizon.Config @@ -28,13 +34,19 @@ var ( Short: "client-facing api server for the stellar network", Long: "client-facing api server for the stellar network. It acts as the interface between Stellar Core and applications that want to access the Stellar network. It allows you to submit transactions to the network, check the status of accounts, subscribe to event streams and more.", Run: func(cmd *cobra.Command, args []string) { + if config.Ingest { + if config.StellarCoreURL == "" { + log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName) + } + if config.StellarCoreDatabaseURL == "" { + log.Fatalf("flag --%s cannot be empty", stellarCoreURLFlagName) + } + } initApp().Serve() }, } ) -const maxDBPingAttempts = 30 - // validateBothOrNeither ensures that both options are provided, if either is provided. func validateBothOrNeither(option1, option2 string) { arg1, arg2 := viper.GetString(option1), viper.GetString(option2) @@ -127,18 +139,16 @@ var configOpts = support.ConfigOptions{ ConfigKey: &config.EnableCaptiveCoreIngestion, }, &support.ConfigOption{ - Name: "stellar-core-db-url", + Name: stellarCoreDBURLFlagName, EnvVar: "STELLAR_CORE_DATABASE_URL", ConfigKey: &config.StellarCoreDatabaseURL, OptType: types.String, - Required: true, Usage: "stellar-core postgres database to connect with", }, &support.ConfigOption{ - Name: "stellar-core-url", + Name: stellarCoreURLFlagName, ConfigKey: &config.StellarCoreURL, OptType: types.String, - Required: true, Usage: "stellar-core to connect with (for http commands)", }, &support.ConfigOption{ From 4469aacd69bc25d84f0cc0082dd9e94b63f196f1 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Thu, 2 Jul 2020 17:40:37 +0200 Subject: [PATCH 2/6] Appease linter go vet --- services/horizon/cmd/db.go | 6 +++--- services/horizon/cmd/ingest.go | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 95b80ba7d8..5b90289693 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -223,9 +223,9 @@ var dbReingestRangeCmd = &cobra.Command{ if config.StellarCoreDatabaseURL == "" { log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName) } - coreSession, err := db.Open("postgres", config.StellarCoreDatabaseURL) - if err != nil { - log.Fatalf("cannot open Core DB: %v", err) + coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL) + if dbErr != nil { + log.Fatalf("cannot open Core DB: %v", dbErr) } ingestConfig.CoreSession = coreSession } diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go index 0b2c8b4eb6..e6e6c860bb 100644 --- a/services/horizon/cmd/ingest.go +++ b/services/horizon/cmd/ingest.go @@ -108,9 +108,9 @@ var ingestVerifyRangeCmd = &cobra.Command{ log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName) } - coreSession, err := db.Open("postgres", config.StellarCoreDatabaseURL) - if err != nil { - log.Fatalf("cannot open Core DB: %v", err) + coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL) + if dbErr != nil { + log.Fatalf("cannot open Core DB: %v", dbErr) } ingestConfig.CoreSession = coreSession } @@ -192,9 +192,9 @@ var ingestStressTestCmd = &cobra.Command{ log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName) } - coreSession, err := db.Open("postgres", config.StellarCoreDatabaseURL) - if err != nil { - log.Fatalf("cannot open Core DB: %v", err) + coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL) + if dbErr != nil { + log.Fatalf("cannot open Core DB: %v", dbErr) } ingestConfig.CoreSession = coreSession } From 8902fd161d87b837208319dcba51ddf63044a58e Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Fri, 3 Jul 2020 12:51:38 +0200 Subject: [PATCH 3/6] Address review feedback --- services/horizon/cmd/root.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/horizon/cmd/root.go b/services/horizon/cmd/root.go index c68ff32349..fa4da269df 100644 --- a/services/horizon/cmd/root.go +++ b/services/horizon/cmd/root.go @@ -34,10 +34,10 @@ var ( Short: "client-facing api server for the stellar network", Long: "client-facing api server for the stellar network. It acts as the interface between Stellar Core and applications that want to access the Stellar network. It allows you to submit transactions to the network, check the status of accounts, subscribe to event streams and more.", Run: func(cmd *cobra.Command, args []string) { + if config.StellarCoreURL == "" { + log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName) + } if config.Ingest { - if config.StellarCoreURL == "" { - log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName) - } if config.StellarCoreDatabaseURL == "" { log.Fatalf("flag --%s cannot be empty", stellarCoreURLFlagName) } From 3faaff2478a0312138822272ee6528db303a1da1 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Fri, 3 Jul 2020 12:55:46 +0200 Subject: [PATCH 4/6] Do not use the core DB session if captive is used --- services/horizon/internal/expingest/main.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/horizon/internal/expingest/main.go b/services/horizon/internal/expingest/main.go index b673d41c47..b7fedeb544 100644 --- a/services/horizon/internal/expingest/main.go +++ b/services/horizon/internal/expingest/main.go @@ -149,9 +149,6 @@ func NewSystem(config Config) (System, error) { return nil, errors.Wrap(err, "error creating history archive") } - coreSession := config.CoreSession.Clone() - coreSession.Ctx = ctx - var ledgerBackend ledgerbackend.LedgerBackend if len(config.StellarCorePath) > 0 { ledgerBackend = ledgerbackend.NewCaptive( @@ -160,6 +157,8 @@ func NewSystem(config Config) (System, error) { []string{config.HistoryArchiveURL}, ) } else { + coreSession := config.CoreSession.Clone() + coreSession.Ctx = ctx ledgerBackend, err = ledgerbackend.NewDatabaseBackendFromSession(coreSession, config.NetworkPassphrase) if err != nil { cancel() From fc3312612c8f7645926dedd20a8856d8fd408f58 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Fri, 3 Jul 2020 14:50:21 +0200 Subject: [PATCH 5/6] Fix typo --- services/horizon/cmd/root.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/horizon/cmd/root.go b/services/horizon/cmd/root.go index fa4da269df..64fdc987b9 100644 --- a/services/horizon/cmd/root.go +++ b/services/horizon/cmd/root.go @@ -35,11 +35,11 @@ var ( Long: "client-facing api server for the stellar network. It acts as the interface between Stellar Core and applications that want to access the Stellar network. It allows you to submit transactions to the network, check the status of accounts, subscribe to event streams and more.", Run: func(cmd *cobra.Command, args []string) { if config.StellarCoreURL == "" { - log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName) + log.Fatalf("flag --%s cannot be empty", stellarCoreURLFlagName) } if config.Ingest { if config.StellarCoreDatabaseURL == "" { - log.Fatalf("flag --%s cannot be empty", stellarCoreURLFlagName) + log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName) } } initApp().Serve() From a3bf665c3151a6f6557defe6db366e950a6a50a7 Mon Sep 17 00:00:00 2001 From: Alfonso Acosta Date: Fri, 3 Jul 2020 15:00:46 +0200 Subject: [PATCH 6/6] Validate Core URL arguments for all app-based commands --- services/horizon/cmd/root.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/services/horizon/cmd/root.go b/services/horizon/cmd/root.go index 64fdc987b9..2312150e3b 100644 --- a/services/horizon/cmd/root.go +++ b/services/horizon/cmd/root.go @@ -34,14 +34,6 @@ var ( Short: "client-facing api server for the stellar network", Long: "client-facing api server for the stellar network. It acts as the interface between Stellar Core and applications that want to access the Stellar network. It allows you to submit transactions to the network, check the status of accounts, subscribe to event streams and more.", Run: func(cmd *cobra.Command, args []string) { - if config.StellarCoreURL == "" { - log.Fatalf("flag --%s cannot be empty", stellarCoreURLFlagName) - } - if config.Ingest { - if config.StellarCoreDatabaseURL == "" { - log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName) - } - } initApp().Serve() }, } @@ -380,6 +372,15 @@ func init() { func initApp() *horizon.App { initRootConfig() + // Validate app-specific arguments + if config.StellarCoreURL == "" { + log.Fatalf("flag --%s cannot be empty", stellarCoreURLFlagName) + } + if config.Ingest { + if config.StellarCoreDatabaseURL == "" { + log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName) + } + } return horizon.NewApp(config) }