From 74113539514a8089afe8d29d743b55e6ea2c1c4c Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 1 Nov 2023 16:22:05 -0400 Subject: [PATCH 01/19] Remove stellar-core-db-url --- services/horizon/cmd/db.go | 3 --- services/horizon/cmd/ingest.go | 16 ---------------- services/horizon/internal/flags.go | 18 ------------------ .../horizon/internal/integration/db_test.go | 2 -- .../internal/integration/parameters_test.go | 18 +++++++----------- .../internal/test/integration/integration.go | 18 ++++++------------ 6 files changed, 13 insertions(+), 62 deletions(-) diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index e23c224012..3875714975 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -423,9 +423,6 @@ func runDBReingestRange(ledgerRanges []history.LedgerRange, reingestForce bool, } if !config.EnableCaptiveCoreIngestion { - if config.StellarCoreDatabaseURL == "" { - return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName) - } if ingestConfig.CoreSession, err = db.Open("postgres", config.StellarCoreDatabaseURL); err != nil { ingestConfig.HistorySession.Close() return fmt.Errorf("cannot open Core DB: %v", err) diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go index 1c44e064d9..fd48e6d7c6 100644 --- a/services/horizon/cmd/ingest.go +++ b/services/horizon/cmd/ingest.go @@ -140,10 +140,6 @@ var ingestVerifyRangeCmd = &cobra.Command{ } if !ingestConfig.EnableCaptiveCore { - if globalConfig.StellarCoreDatabaseURL == "" { - return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName) - } - coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) if dbErr != nil { return fmt.Errorf("cannot open Core DB: %v", dbErr) @@ -233,10 +229,6 @@ var ingestStressTestCmd = &cobra.Command{ ingestConfig.RemoteCaptiveCoreURL = globalConfig.RemoteCaptiveCoreURL ingestConfig.CaptiveCoreConfigUseDB = globalConfig.CaptiveCoreConfigUseDB } else { - if globalConfig.StellarCoreDatabaseURL == "" { - return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName) - } - coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) if dbErr != nil { return fmt.Errorf("cannot open Core DB: %v", dbErr) @@ -325,10 +317,6 @@ var ingestInitGenesisStateCmd = &cobra.Command{ ingestConfig.CaptiveCoreBinaryPath = globalConfig.CaptiveCoreBinaryPath ingestConfig.CaptiveCoreConfigUseDB = globalConfig.CaptiveCoreConfigUseDB } else { - if globalConfig.StellarCoreDatabaseURL == "" { - return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName) - } - coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) if dbErr != nil { return fmt.Errorf("cannot open Core DB: %v", dbErr) @@ -405,10 +393,6 @@ var ingestBuildStateCmd = &cobra.Command{ if !ingestBuildStateSkipChecks { if !ingestConfig.EnableCaptiveCore { - if globalConfig.StellarCoreDatabaseURL == "" { - return fmt.Errorf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName) - } - coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) if dbErr != nil { return fmt.Errorf("cannot open Core DB: %v", dbErr) diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 4f18aacc63..02a59ae478 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -29,8 +29,6 @@ const ( DatabaseURLFlagName = "db-url" // IngestFlagName is the command line flag for enabling ingestion on the Horizon instance IngestFlagName = "ingest" - // StellarCoreDBURLFlagName is the command line flag for configuring the postgres Stellar Core URL - StellarCoreDBURLFlagName = "stellar-core-db-url" // StellarCoreURLFlagName is the command line flag for configuring the URL fore Stellar Core HTTP endpoint StellarCoreURLFlagName = "stellar-core-url" // StellarCoreBinaryPathName is the command line flag for configuring the path to the stellar core binary @@ -322,15 +320,6 @@ func Flags() (*Config, support.ConfigOptions) { ConfigKey: &config.CaptiveCoreTomlParams.PeerPort, UsedInCommands: IngestionCommands, }, - &support.ConfigOption{ - Name: StellarCoreDBURLFlagName, - EnvVar: "STELLAR_CORE_DATABASE_URL", - ConfigKey: &config.StellarCoreDatabaseURL, - OptType: types.String, - Required: false, - Usage: "stellar-core postgres database to connect with", - UsedInCommands: IngestionCommands, - }, &support.ConfigOption{ Name: StellarCoreURLFlagName, ConfigKey: &config.StellarCoreURL, @@ -724,9 +713,6 @@ func NewAppFromFlags(config *Config, flags support.ConfigOptions) (*App, error) if config.StellarCoreURL == "" { return nil, fmt.Errorf("flag --%s cannot be empty", StellarCoreURLFlagName) } - if config.Ingest && !config.EnableCaptiveCoreIngestion && config.StellarCoreDatabaseURL == "" { - return nil, fmt.Errorf("flag --%s cannot be empty", StellarCoreDBURLFlagName) - } log.Infof("Initializing horizon...") app, err := NewApp(*config) @@ -931,10 +917,6 @@ func ApplyFlags(config *Config, flags support.ConfigOptions, options ApplyOption return fmt.Errorf("invalid config: one or more captive core params passed (--%s or --%s) but --ingest not set"+captiveCoreMigrationHint, StellarCoreBinaryPathName, captiveCoreConfigFlag) } - if config.StellarCoreDatabaseURL != "" { - return fmt.Errorf("invalid config: --%s passed but --ingest not set"+ - "", StellarCoreDBURLFlagName) - } } // Configure log file diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index 020d934930..3ed112325d 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -558,8 +558,6 @@ func command(t *testing.T, horizonConfig horizon.Config, args ...string) []strin horizonConfig.HistoryArchiveURLs[0], "--db-url", horizonConfig.DatabaseURL, - "--stellar-core-db-url", - horizonConfig.StellarCoreDatabaseURL, "--stellar-core-binary-path", horizonConfig.CaptiveCoreBinaryPath, "--captive-core-config-path", diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go index 661a90ec72..341c0746b7 100644 --- a/services/horizon/internal/integration/parameters_test.go +++ b/services/horizon/internal/integration/parameters_test.go @@ -28,7 +28,6 @@ import ( var defaultCaptiveCoreParameters = map[string]string{ horizon.StellarCoreBinaryPathName: os.Getenv("CAPTIVE_CORE_BIN"), horizon.StellarCoreURLFlagName: "", - horizon.StellarCoreDBURLFlagName: "", } var networkParamArgs = map[string]string{ @@ -400,10 +399,9 @@ func TestIngestionFilteringAlwaysDefaultingToTrue(t *testing.T) { func TestDisableTxSub(t *testing.T) { t.Run("require stellar-core-url when both DISABLE_TX_SUB=false and INGEST=false", func(t *testing.T) { localParams := integration.MergeMaps(networkParamArgs, map[string]string{ - horizon.NetworkFlagName: "testnet", - horizon.IngestFlagName: "false", - horizon.DisableTxSubFlagName: "false", - horizon.StellarCoreDBURLFlagName: "", + horizon.NetworkFlagName: "testnet", + horizon.IngestFlagName: "false", + horizon.DisableTxSubFlagName: "false", }) testConfig := integration.GetTestConfig() testConfig.HorizonIngestParameters = localParams @@ -414,13 +412,11 @@ func TestDisableTxSub(t *testing.T) { test.Shutdown() }) t.Run("horizon starts successfully when DISABLE_TX_SUB=false, INGEST=false and stellar-core-url is provided", func(t *testing.T) { - // TODO: Remove explicit mention of stellar-core-db-url once this issue is done: https://github.com/stellar/go/issues/4855 localParams := integration.MergeMaps(networkParamArgs, map[string]string{ - horizon.NetworkFlagName: "testnet", - horizon.IngestFlagName: "false", - horizon.DisableTxSubFlagName: "false", - horizon.StellarCoreDBURLFlagName: "", - horizon.StellarCoreURLFlagName: "http://localhost:11626", + horizon.NetworkFlagName: "testnet", + horizon.IngestFlagName: "false", + horizon.DisableTxSubFlagName: "false", + horizon.StellarCoreURLFlagName: "http://localhost:11626", }) testConfig := integration.GetTestConfig() testConfig.HorizonIngestParameters = localParams diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go index e0f2e13864..9777f5a90c 100644 --- a/services/horizon/internal/test/integration/integration.go +++ b/services/horizon/internal/test/integration/integration.go @@ -447,18 +447,12 @@ func (i *Test) getDefaultIngestArgs(postgres *dbtest.DB) map[string]string { "port": "8001", "enable-captive-core-ingestion": strconv.FormatBool(len(i.coreConfig.binaryPath) > 0), "db-url": postgres.DSN, - "stellar-core-db-url": fmt.Sprintf( - "postgres://postgres:%s@%s:%d/stellar?sslmode=disable", - stellarCorePostgresPassword, - "localhost", - stellarCorePostgresPort, - ), - "stellar-core-binary-path": i.coreConfig.binaryPath, - "captive-core-config-path": i.coreConfig.configPath, - "captive-core-http-port": "21626", - "captive-core-use-db": strconv.FormatBool(i.coreConfig.useDB), - "captive-core-storage-path": i.coreConfig.storagePath, - "ingest": "true"}) + "stellar-core-binary-path": i.coreConfig.binaryPath, + "captive-core-config-path": i.coreConfig.configPath, + "captive-core-http-port": "21626", + "captive-core-use-db": strconv.FormatBool(i.coreConfig.useDB), + "captive-core-storage-path": i.coreConfig.storagePath, + "ingest": "true"}) } func (i *Test) createWebCommand(webConfig *horizon.Config, webConfigOpts config.ConfigOptions) *cobra.Command { From 0c7f69b98be1b176c28340edabb6756aa12f236a Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 1 Nov 2023 16:39:03 -0400 Subject: [PATCH 02/19] Remove stellar-core-db-url - 2 --- services/horizon/cmd/db.go | 7 ---- services/horizon/cmd/ingest.go | 44 +++-------------------- services/horizon/internal/config.go | 3 +- services/horizon/internal/helpers_test.go | 4 +-- services/horizon/internal/init.go | 4 --- 5 files changed, 7 insertions(+), 55 deletions(-) diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 3875714975..19d79dd2e8 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -422,13 +422,6 @@ func runDBReingestRange(ledgerRanges []history.LedgerRange, reingestForce bool, return fmt.Errorf("cannot open Horizon DB: %v", err) } - if !config.EnableCaptiveCoreIngestion { - if ingestConfig.CoreSession, err = db.Open("postgres", config.StellarCoreDatabaseURL); err != nil { - ingestConfig.HistorySession.Close() - return fmt.Errorf("cannot open Core DB: %v", err) - } - } - if parallelWorkers > 1 { system, systemErr := ingest.NewParallelSystems(ingestConfig, parallelWorkers) if systemErr != nil { diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go index fd48e6d7c6..23dfb4faee 100644 --- a/services/horizon/cmd/ingest.go +++ b/services/horizon/cmd/ingest.go @@ -139,14 +139,6 @@ var ingestVerifyRangeCmd = &cobra.Command{ EnableIngestionFiltering: globalConfig.EnableIngestionFiltering, } - if !ingestConfig.EnableCaptiveCore { - coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) - if dbErr != nil { - return fmt.Errorf("cannot open Core DB: %v", dbErr) - } - ingestConfig.CoreSession = coreSession - } - system, err := ingest.NewSystem(ingestConfig) if err != nil { return err @@ -224,17 +216,9 @@ var ingestStressTestCmd = &cobra.Command{ RoundingSlippageFilter: globalConfig.RoundingSlippageFilter, } - if globalConfig.EnableCaptiveCoreIngestion { - ingestConfig.CaptiveCoreBinaryPath = globalConfig.CaptiveCoreBinaryPath - ingestConfig.RemoteCaptiveCoreURL = globalConfig.RemoteCaptiveCoreURL - ingestConfig.CaptiveCoreConfigUseDB = globalConfig.CaptiveCoreConfigUseDB - } else { - coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) - if dbErr != nil { - return fmt.Errorf("cannot open Core DB: %v", dbErr) - } - ingestConfig.CoreSession = coreSession - } + ingestConfig.CaptiveCoreBinaryPath = globalConfig.CaptiveCoreBinaryPath + ingestConfig.RemoteCaptiveCoreURL = globalConfig.RemoteCaptiveCoreURL + ingestConfig.CaptiveCoreConfigUseDB = globalConfig.CaptiveCoreConfigUseDB system, err := ingest.NewSystem(ingestConfig) if err != nil { @@ -313,16 +297,8 @@ var ingestInitGenesisStateCmd = &cobra.Command{ EnableIngestionFiltering: globalConfig.EnableIngestionFiltering, } - if globalConfig.EnableCaptiveCoreIngestion { - ingestConfig.CaptiveCoreBinaryPath = globalConfig.CaptiveCoreBinaryPath - ingestConfig.CaptiveCoreConfigUseDB = globalConfig.CaptiveCoreConfigUseDB - } else { - coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) - if dbErr != nil { - return fmt.Errorf("cannot open Core DB: %v", dbErr) - } - ingestConfig.CoreSession = coreSession - } + ingestConfig.CaptiveCoreBinaryPath = globalConfig.CaptiveCoreBinaryPath + ingestConfig.CaptiveCoreConfigUseDB = globalConfig.CaptiveCoreConfigUseDB system, err := ingest.NewSystem(ingestConfig) if err != nil { @@ -391,16 +367,6 @@ var ingestBuildStateCmd = &cobra.Command{ EnableIngestionFiltering: globalConfig.EnableIngestionFiltering, } - if !ingestBuildStateSkipChecks { - if !ingestConfig.EnableCaptiveCore { - coreSession, dbErr := db.Open("postgres", globalConfig.StellarCoreDatabaseURL) - if dbErr != nil { - return fmt.Errorf("cannot open Core DB: %v", dbErr) - } - ingestConfig.CoreSession = coreSession - } - } - system, err := ingest.NewSystem(ingestConfig) if err != nil { return err diff --git a/services/horizon/internal/config.go b/services/horizon/internal/config.go index fb6853ea72..d7ae6c44d4 100644 --- a/services/horizon/internal/config.go +++ b/services/horizon/internal/config.go @@ -30,8 +30,7 @@ type Config struct { CaptiveCoreReuseStoragePath bool CaptiveCoreConfigUseDB bool - StellarCoreDatabaseURL string - StellarCoreURL string + StellarCoreURL string // MaxDBConnections has a priority over all 4 values below. MaxDBConnections int diff --git a/services/horizon/internal/helpers_test.go b/services/horizon/internal/helpers_test.go index a5c1dfc0d2..9a78598430 100644 --- a/services/horizon/internal/helpers_test.go +++ b/services/horizon/internal/helpers_test.go @@ -8,7 +8,6 @@ import ( "github.com/stellar/go/network" "github.com/stellar/go/services/horizon/internal/test" - tdb "github.com/stellar/go/services/horizon/internal/test/db" supportLog "github.com/stellar/go/support/log" ) @@ -22,8 +21,7 @@ func NewTestApp(dsn string) *App { func NewTestConfig(dsn string) Config { return Config{ - DatabaseURL: dsn, - StellarCoreDatabaseURL: tdb.StellarCoreURL(), + DatabaseURL: dsn, RateQuota: &throttled.RateQuota{ MaxRate: throttled.PerHour(1000), MaxBurst: 100, diff --git a/services/horizon/internal/init.go b/services/horizon/internal/init.go index e735a2d2c8..6d44b62eaa 100644 --- a/services/horizon/internal/init.go +++ b/services/horizon/internal/init.go @@ -92,10 +92,6 @@ func mustInitHorizonDB(app *App) { func initIngester(app *App) { var err error var coreSession db.SessionInterface - if !app.config.EnableCaptiveCoreIngestion { - coreSession = mustNewDBSession( - db.CoreSubservice, app.config.StellarCoreDatabaseURL, ingest.MaxDBConnections, ingest.MaxDBConnections, app.prometheusRegistry) - } app.ingester, err = ingest.NewSystem(ingest.Config{ CoreSession: coreSession, HistorySession: mustNewDBSession( From a0a0a0e87c575d503b3682d5c186b537f61ef662 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 1 Nov 2023 16:45:22 -0400 Subject: [PATCH 03/19] Remove enable-captive-core-ingestion support --- services/horizon/cmd/db.go | 1 - services/horizon/cmd/ingest.go | 4 --- services/horizon/internal/config.go | 1 - services/horizon/internal/flags.go | 21 +++--------- services/horizon/internal/flags_test.go | 1 - services/horizon/internal/init.go | 1 - .../horizon/internal/integration/db_test.go | 1 - .../internal/integration/parameters_test.go | 18 +++++----- .../internal/test/integration/integration.go | 34 +++++++++---------- 9 files changed, 28 insertions(+), 54 deletions(-) diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go index 19d79dd2e8..45ecd90489 100644 --- a/services/horizon/cmd/db.go +++ b/services/horizon/cmd/db.go @@ -406,7 +406,6 @@ func runDBReingestRange(ledgerRanges []history.LedgerRange, reingestForce bool, ReingestEnabled: true, MaxReingestRetries: int(retries), ReingestRetryBackoffSeconds: int(retryBackoffSeconds), - EnableCaptiveCore: config.EnableCaptiveCoreIngestion, CaptiveCoreBinaryPath: config.CaptiveCoreBinaryPath, CaptiveCoreConfigUseDB: config.CaptiveCoreConfigUseDB, RemoteCaptiveCoreURL: config.RemoteCaptiveCoreURL, diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go index 23dfb4faee..ff4e634201 100644 --- a/services/horizon/cmd/ingest.go +++ b/services/horizon/cmd/ingest.go @@ -128,7 +128,6 @@ var ingestVerifyRangeCmd = &cobra.Command{ NetworkPassphrase: globalConfig.NetworkPassphrase, HistorySession: horizonSession, HistoryArchiveURLs: globalConfig.HistoryArchiveURLs, - EnableCaptiveCore: globalConfig.EnableCaptiveCoreIngestion, CaptiveCoreBinaryPath: globalConfig.CaptiveCoreBinaryPath, CaptiveCoreConfigUseDB: globalConfig.CaptiveCoreConfigUseDB, RemoteCaptiveCoreURL: globalConfig.RemoteCaptiveCoreURL, @@ -212,7 +211,6 @@ var ingestStressTestCmd = &cobra.Command{ NetworkPassphrase: globalConfig.NetworkPassphrase, HistorySession: horizonSession, HistoryArchiveURLs: globalConfig.HistoryArchiveURLs, - EnableCaptiveCore: globalConfig.EnableCaptiveCoreIngestion, RoundingSlippageFilter: globalConfig.RoundingSlippageFilter, } @@ -291,7 +289,6 @@ var ingestInitGenesisStateCmd = &cobra.Command{ NetworkPassphrase: globalConfig.NetworkPassphrase, HistorySession: horizonSession, HistoryArchiveURLs: globalConfig.HistoryArchiveURLs, - EnableCaptiveCore: globalConfig.EnableCaptiveCoreIngestion, CheckpointFrequency: globalConfig.CheckpointFrequency, RoundingSlippageFilter: globalConfig.RoundingSlippageFilter, EnableIngestionFiltering: globalConfig.EnableIngestionFiltering, @@ -356,7 +353,6 @@ var ingestBuildStateCmd = &cobra.Command{ NetworkPassphrase: globalConfig.NetworkPassphrase, HistorySession: horizonSession, HistoryArchiveURLs: globalConfig.HistoryArchiveURLs, - EnableCaptiveCore: globalConfig.EnableCaptiveCoreIngestion, CaptiveCoreBinaryPath: globalConfig.CaptiveCoreBinaryPath, CaptiveCoreConfigUseDB: globalConfig.CaptiveCoreConfigUseDB, RemoteCaptiveCoreURL: globalConfig.RemoteCaptiveCoreURL, diff --git a/services/horizon/internal/config.go b/services/horizon/internal/config.go index d7ae6c44d4..1cc14b4900 100644 --- a/services/horizon/internal/config.go +++ b/services/horizon/internal/config.go @@ -19,7 +19,6 @@ type Config struct { Port uint AdminPort uint - EnableCaptiveCoreIngestion bool EnableIngestionFiltering bool CaptiveCoreBinaryPath string RemoteCaptiveCoreURL string diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 02a59ae478..1f73171fda 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -43,8 +43,6 @@ const ( CaptiveCoreConfigUseDB = "captive-core-use-db" // CaptiveCoreHTTPPortFlagName is the commandline flag for specifying captive core HTTP port CaptiveCoreHTTPPortFlagName = "captive-core-http-port" - // EnableCaptiveCoreIngestionFlagName is the commandline flag for enabling captive core ingestion - EnableCaptiveCoreIngestionFlagName = "enable-captive-core-ingestion" // NetworkPassphraseFlagName is the command line flag for specifying the network passphrase NetworkPassphraseFlagName = "network-passphrase" // HistoryArchiveURLsFlagName is the command line flag for specifying the history archive URLs @@ -248,15 +246,6 @@ func Flags() (*Config, support.ConfigOptions) { ConfigKey: &config.CaptiveCoreConfigUseDB, UsedInCommands: IngestionCommands, }, - &support.ConfigOption{ - Name: "enable-captive-core-ingestion", - OptType: types.Bool, - FlagDefault: true, - Required: false, - Usage: "causes Horizon to ingest from a Captive Stellar Core process instead of a persistent Stellar Core database", - ConfigKey: &config.EnableCaptiveCoreIngestion, - UsedInCommands: IngestionCommands, - }, &support.ConfigOption{ Name: EnableIngestionFilteringFlagName, OptType: types.String, @@ -902,14 +891,12 @@ func ApplyFlags(config *Config, flags support.ConfigOptions, options ApplyOption return err } - if config.EnableCaptiveCoreIngestion { - err := setCaptiveCoreConfiguration(config, options) - if err != nil { - return errors.Wrap(err, "error generating captive core configuration") - } + err := setCaptiveCoreConfiguration(config, options) + if err != nil { + return errors.Wrap(err, "error generating captive core configuration") } } else { - if config.EnableCaptiveCoreIngestion && (config.CaptiveCoreBinaryPath != "" || config.CaptiveCoreConfigPath != "") { + if config.CaptiveCoreBinaryPath != "" || config.CaptiveCoreConfigPath != "" { captiveCoreConfigFlag := captiveCoreConfigAppendPathName if viper.GetString(CaptiveCoreConfigPathName) != "" { captiveCoreConfigFlag = CaptiveCoreConfigPathName diff --git a/services/horizon/internal/flags_test.go b/services/horizon/internal/flags_test.go index eab97b1beb..7b5b2b6e4c 100644 --- a/services/horizon/internal/flags_test.go +++ b/services/horizon/internal/flags_test.go @@ -252,7 +252,6 @@ func TestEnvironmentVariables(t *testing.T) { assert.Equal(t, config.StellarCoreURL, "http://localhost:11626") assert.Equal(t, config.NetworkPassphrase, "Standalone Network ; February 2017") assert.Equal(t, config.ApplyMigrations, true) - assert.Equal(t, config.EnableCaptiveCoreIngestion, false) assert.Equal(t, config.CheckpointFrequency, uint32(8)) assert.Equal(t, config.MaxDBConnections, 50) assert.Equal(t, config.AdminPort, uint(6060)) diff --git a/services/horizon/internal/init.go b/services/horizon/internal/init.go index 6d44b62eaa..5d38c86ccf 100644 --- a/services/horizon/internal/init.go +++ b/services/horizon/internal/init.go @@ -107,7 +107,6 @@ func initIngester(app *App) { CaptiveCoreConfigUseDB: app.config.CaptiveCoreConfigUseDB, CaptiveCoreToml: app.config.CaptiveCoreToml, RemoteCaptiveCoreURL: app.config.RemoteCaptiveCoreURL, - EnableCaptiveCore: app.config.EnableCaptiveCoreIngestion, DisableStateVerification: app.config.IngestDisableStateVerification, StateVerificationCheckpointFrequency: uint32(app.config.IngestStateVerificationCheckpointFrequency), StateVerificationTimeout: app.config.IngestStateVerificationTimeout, diff --git a/services/horizon/internal/integration/db_test.go b/services/horizon/internal/integration/db_test.go index 3ed112325d..2d48e1deb8 100644 --- a/services/horizon/internal/integration/db_test.go +++ b/services/horizon/internal/integration/db_test.go @@ -564,7 +564,6 @@ func command(t *testing.T, horizonConfig horizon.Config, args ...string) []strin horizonConfig.CaptiveCoreConfigPath, "--captive-core-use-db=" + strconv.FormatBool(horizonConfig.CaptiveCoreConfigUseDB), - "--enable-captive-core-ingestion=" + strconv.FormatBool(horizonConfig.EnableCaptiveCoreIngestion), "--network-passphrase", horizonConfig.NetworkPassphrase, // due to ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go index 341c0746b7..a5c7619e48 100644 --- a/services/horizon/internal/integration/parameters_test.go +++ b/services/horizon/internal/integration/parameters_test.go @@ -31,13 +31,12 @@ var defaultCaptiveCoreParameters = map[string]string{ } var networkParamArgs = map[string]string{ - horizon.EnableCaptiveCoreIngestionFlagName: "", - horizon.CaptiveCoreConfigPathName: "", - horizon.CaptiveCoreHTTPPortFlagName: "", - horizon.StellarCoreBinaryPathName: "", - horizon.StellarCoreURLFlagName: "", - horizon.HistoryArchiveURLsFlagName: "", - horizon.NetworkPassphraseFlagName: "", + horizon.CaptiveCoreConfigPathName: "", + horizon.CaptiveCoreHTTPPortFlagName: "", + horizon.StellarCoreBinaryPathName: "", + horizon.StellarCoreURLFlagName: "", + horizon.HistoryArchiveURLsFlagName: "", + horizon.NetworkPassphraseFlagName: "", } const ( @@ -164,9 +163,8 @@ func TestInvalidNetworkParameters(t *testing.T) { for _, testCase := range testCases { t.Run(testCase.name, func(t *testing.T) { localParams := integration.MergeMaps(networkParamArgs, map[string]string{ - horizon.NetworkFlagName: testCase.networkValue, - horizon.EnableCaptiveCoreIngestionFlagName: "true", - testCase.param: testCase.param, // set any value + horizon.NetworkFlagName: testCase.networkValue, + testCase.param: testCase.param, // set any value }) testConfig := integration.GetTestConfig() testConfig.SkipCoreContainerCreation = true diff --git a/services/horizon/internal/test/integration/integration.go b/services/horizon/internal/test/integration/integration.go index 9777f5a90c..9ef5798487 100644 --- a/services/horizon/internal/test/integration/integration.go +++ b/services/horizon/internal/test/integration/integration.go @@ -422,14 +422,13 @@ func (i *Test) getDefaultArgs(postgres *dbtest.DB) map[string]string { // TODO: Ideally, we'd be pulling host/port information from the Docker // Compose YAML file itself rather than hardcoding it. return map[string]string{ - "ingest": "false", - "history-archive-urls": fmt.Sprintf("http://%s:%d", "localhost", historyArchivePort), - "db-url": postgres.RO_DSN, - "stellar-core-url": i.coreClient.URL, - "network-passphrase": i.passPhrase, - "apply-migrations": "true", - "enable-captive-core-ingestion": "false", - "port": horizonDefaultPort, + "ingest": "false", + "history-archive-urls": fmt.Sprintf("http://%s:%d", "localhost", historyArchivePort), + "db-url": postgres.RO_DSN, + "stellar-core-url": i.coreClient.URL, + "network-passphrase": i.passPhrase, + "apply-migrations": "true", + "port": horizonDefaultPort, // due to ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING "checkpoint-frequency": "8", "per-hour-rate-limit": "0", // disable rate limiting @@ -443,16 +442,15 @@ func (i *Test) getDefaultWebArgs(postgres *dbtest.DB) map[string]string { func (i *Test) getDefaultIngestArgs(postgres *dbtest.DB) map[string]string { return MergeMaps(i.getDefaultArgs(postgres), map[string]string{ - "admin-port": strconv.Itoa(i.AdminPort()), - "port": "8001", - "enable-captive-core-ingestion": strconv.FormatBool(len(i.coreConfig.binaryPath) > 0), - "db-url": postgres.DSN, - "stellar-core-binary-path": i.coreConfig.binaryPath, - "captive-core-config-path": i.coreConfig.configPath, - "captive-core-http-port": "21626", - "captive-core-use-db": strconv.FormatBool(i.coreConfig.useDB), - "captive-core-storage-path": i.coreConfig.storagePath, - "ingest": "true"}) + "admin-port": strconv.Itoa(i.AdminPort()), + "port": "8001", + "db-url": postgres.DSN, + "stellar-core-binary-path": i.coreConfig.binaryPath, + "captive-core-config-path": i.coreConfig.configPath, + "captive-core-http-port": "21626", + "captive-core-use-db": strconv.FormatBool(i.coreConfig.useDB), + "captive-core-storage-path": i.coreConfig.storagePath, + "ingest": "true"}) } func (i *Test) createWebCommand(webConfig *horizon.Config, webConfigOpts config.ConfigOptions) *cobra.Command { From 992aa3d9ed561a0c890e7471b13038b2992536e5 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 1 Nov 2023 16:58:57 -0400 Subject: [PATCH 04/19] Remove enable-captive-core-ingestion support - 2 --- services/horizon/internal/ingest/main.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/services/horizon/internal/ingest/main.go b/services/horizon/internal/ingest/main.go index 6535a3e097..ee83c211b8 100644 --- a/services/horizon/internal/ingest/main.go +++ b/services/horizon/internal/ingest/main.go @@ -76,7 +76,6 @@ type Config struct { CoreSession db.SessionInterface StellarCoreURL string StellarCoreCursor string - EnableCaptiveCore bool CaptiveCoreBinaryPath string CaptiveCoreStoragePath string CaptiveCoreToml *ledgerbackend.CaptiveCoreToml @@ -108,16 +107,14 @@ type Config struct { // LocalCaptiveCoreEnabled returns true if configured to run // a local captive core instance for ingestion. func (c Config) LocalCaptiveCoreEnabled() bool { - // c.EnableCaptiveCore is true for both local and remote captive core - // and c.RemoteCaptiveCoreURL is always empty when running - // local captive core. - return c.EnableCaptiveCore && c.RemoteCaptiveCoreURL == "" + // c.RemoteCaptiveCoreURL is always empty when running local captive core. + return c.RemoteCaptiveCoreURL == "" } // RemoteCaptiveCoreEnabled returns true if configured to run // a remote captive core instance for ingestion. func (c Config) RemoteCaptiveCoreEnabled() bool { - return c.EnableCaptiveCore && c.RemoteCaptiveCoreURL != "" + return c.RemoteCaptiveCoreURL != "" } const ( @@ -741,7 +738,7 @@ func (s *system) resetStateVerificationErrors() { } func (s *system) updateCursor(ledgerSequence uint32) error { - if s.stellarCoreClient == nil || s.config.EnableCaptiveCore { + if s.stellarCoreClient == nil { return nil } From 6a74ad92aed02f521d6a4acb84494730d87e6ec2 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 2 Nov 2023 10:51:11 -0400 Subject: [PATCH 05/19] Update flags_test.go --- services/horizon/internal/flags_test.go | 27 ++++++++++++------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/services/horizon/internal/flags_test.go b/services/horizon/internal/flags_test.go index 7b5b2b6e4c..b2e617bc00 100644 --- a/services/horizon/internal/flags_test.go +++ b/services/horizon/internal/flags_test.go @@ -208,20 +208,19 @@ func Test_createCaptiveCoreConfig(t *testing.T) { func TestEnvironmentVariables(t *testing.T) { environmentVars := map[string]string{ - "INGEST": "false", - "HISTORY_ARCHIVE_URLS": "http://localhost:1570", - "DATABASE_URL": "postgres://postgres@localhost/test_332cb65e6b00?sslmode=disable&timezone=UTC", - "STELLAR_CORE_URL": "http://localhost:11626", - "NETWORK_PASSPHRASE": "Standalone Network ; February 2017", - "APPLY_MIGRATIONS": "true", - "ENABLE_CAPTIVE_CORE_INGESTION": "false", - "CHECKPOINT_FREQUENCY": "8", - "MAX_DB_CONNECTIONS": "50", - "ADMIN_PORT": "6060", - "PORT": "8001", - "CAPTIVE_CORE_BINARY_PATH": os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN"), - "CAPTIVE_CORE_CONFIG_PATH": "../docker/captive-core-classic-integration-tests.cfg", - "CAPTIVE_CORE_USE_DB": "true", + "INGEST": "false", + "HISTORY_ARCHIVE_URLS": "http://localhost:1570", + "DATABASE_URL": "postgres://postgres@localhost/test_332cb65e6b00?sslmode=disable&timezone=UTC", + "STELLAR_CORE_URL": "http://localhost:11626", + "NETWORK_PASSPHRASE": "Standalone Network ; February 2017", + "APPLY_MIGRATIONS": "true", + "CHECKPOINT_FREQUENCY": "8", + "MAX_DB_CONNECTIONS": "50", + "ADMIN_PORT": "6060", + "PORT": "8001", + "CAPTIVE_CORE_BINARY_PATH": os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN"), + "CAPTIVE_CORE_CONFIG_PATH": "../docker/captive-core-classic-integration-tests.cfg", + "CAPTIVE_CORE_USE_DB": "true", } envManager := test.NewEnvironmentManager() From 48340055c4b506ddf86c1a449a874c1b32c4724e Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 2 Nov 2023 10:53:27 -0400 Subject: [PATCH 06/19] Update start --- services/horizon/docker/verify-range/start | 2 -- 1 file changed, 2 deletions(-) diff --git a/services/horizon/docker/verify-range/start b/services/horizon/docker/verify-range/start index ff0e33ca29..0d70e3ca8f 100644 --- a/services/horizon/docker/verify-range/start +++ b/services/horizon/docker/verify-range/start @@ -73,8 +73,6 @@ export NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" export HISTORY_ARCHIVE_URLS="https://s3-eu-west-1.amazonaws.com/history.stellar.org/prd/core-live/core_live_001" export DATABASE_URL="postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable" export CAPTIVE_CORE_CONFIG_APPEND_PATH="/captive-core-pubnet.cfg" -export STELLAR_CORE_BINARY_PATH="/usr/bin/stellar-core" -export ENABLE_CAPTIVE_CORE_INGESTION="true" cd stellar-go git pull origin From 7cebd1b724078693e28539ec366728fef4e8583d Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 2 Nov 2023 11:48:28 -0400 Subject: [PATCH 07/19] Misc changes - 1 --- services/horizon/docker/verify-range/start | 2 +- services/horizon/internal/flags.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/services/horizon/docker/verify-range/start b/services/horizon/docker/verify-range/start index 0d70e3ca8f..40a68a998d 100644 --- a/services/horizon/docker/verify-range/start +++ b/services/horizon/docker/verify-range/start @@ -72,7 +72,7 @@ dump_horizon_db() { export NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" export HISTORY_ARCHIVE_URLS="https://s3-eu-west-1.amazonaws.com/history.stellar.org/prd/core-live/core_live_001" export DATABASE_URL="postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable" -export CAPTIVE_CORE_CONFIG_APPEND_PATH="/captive-core-pubnet.cfg" +export CAPTIVE_CORE_CONFIG_APPEND_PATH="/captive-core-pubnet.cfg" cd stellar-go git pull origin diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 1f73171fda..231f90f9a7 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -901,7 +901,7 @@ func ApplyFlags(config *Config, flags support.ConfigOptions, options ApplyOption if viper.GetString(CaptiveCoreConfigPathName) != "" { captiveCoreConfigFlag = CaptiveCoreConfigPathName } - return fmt.Errorf("invalid config: one or more captive core params passed (--%s or --%s) but --ingest not set"+captiveCoreMigrationHint, + return fmt.Errorf("invalid config: one or more captive core params passed (--%s or --%s) but --ingest not set. "+captiveCoreMigrationHint, StellarCoreBinaryPathName, captiveCoreConfigFlag) } } From 1d95c7acdecda58626f5ac8b0b06d98ec77db2c1 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 2 Nov 2023 12:19:27 -0400 Subject: [PATCH 08/19] Update parameters_test.go --- services/horizon/internal/integration/parameters_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go index a5c7619e48..348358e399 100644 --- a/services/horizon/internal/integration/parameters_test.go +++ b/services/horizon/internal/integration/parameters_test.go @@ -112,7 +112,10 @@ func TestEnvironmentPreserved(t *testing.T) { confName, _, cleanup := createCaptiveCoreConfig(SIMPLE_CAPTIVE_CORE_TOML) defer cleanup() testConfig := integration.GetTestConfig() - testConfig.HorizonEnvironment = map[string]string{"CAPTIVE_CORE_CONFIG_PATH": confName} + testConfig.HorizonEnvironment = map[string]string{ + "CAPTIVE_CORE_CONFIG_PATH": confName, + "CAPTIVE_CORE_BINARY_PATH": os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN"), + } test := integration.NewTest(t, *testConfig) err = test.StartHorizon() From d54fa7c32ae98c02d32dc4b3ed30831fa950aef6 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 2 Nov 2023 13:21:53 -0400 Subject: [PATCH 09/19] Update parameters_test.go --- .../internal/integration/parameters_test.go | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go index 348358e399..d7ba8819a8 100644 --- a/services/horizon/internal/integration/parameters_test.go +++ b/services/horizon/internal/integration/parameters_test.go @@ -53,6 +53,8 @@ const ( PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS" ADDRESS="localhost" QUALITY="MEDIUM"` + + STELLAR_CORE_URL = "http://localhost:11626" ) var ( @@ -97,24 +99,21 @@ func TestEnvironmentPreserved(t *testing.T) { // running an integration test. // Note that we ALSO need to make sure we don't modify parent env state. - value, isSet := os.LookupEnv("CAPTIVE_CORE_CONFIG_PATH") + value, isSet := os.LookupEnv("STELLAR_CORE_URL") defer func() { if isSet { - _ = os.Setenv("CAPTIVE_CORE_CONFIG_PATH", value) + _ = os.Setenv("STELLAR_CORE_URL", value) } else { - _ = os.Unsetenv("CAPTIVE_CORE_CONFIG_PATH") + _ = os.Unsetenv("STELLAR_CORE_URL") } }() - err := os.Setenv("CAPTIVE_CORE_CONFIG_PATH", "original value") + err := os.Setenv("STELLAR_CORE_URL", "original value") assert.NoError(t, err) - confName, _, cleanup := createCaptiveCoreConfig(SIMPLE_CAPTIVE_CORE_TOML) - defer cleanup() testConfig := integration.GetTestConfig() testConfig.HorizonEnvironment = map[string]string{ - "CAPTIVE_CORE_CONFIG_PATH": confName, - "CAPTIVE_CORE_BINARY_PATH": os.Getenv("HORIZON_INTEGRATION_TESTS_CAPTIVE_CORE_BIN"), + "STELLAR_CORE_URL": STELLAR_CORE_URL, } test := integration.NewTest(t, *testConfig) @@ -122,12 +121,12 @@ func TestEnvironmentPreserved(t *testing.T) { assert.NoError(t, err) test.WaitForHorizon() - envValue := os.Getenv("CAPTIVE_CORE_CONFIG_PATH") - assert.Equal(t, confName, envValue) + envValue := os.Getenv("STELLAR_CORE_URL") + assert.Equal(t, STELLAR_CORE_URL, envValue) test.Shutdown() - envValue = os.Getenv("CAPTIVE_CORE_CONFIG_PATH") + envValue = os.Getenv("STELLAR_CORE_URL") assert.Equal(t, "original value", envValue) } From 8a5946ec5dac6e19c317bedcda4a28a10c6bcc30 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 3 Nov 2023 12:37:00 -0400 Subject: [PATCH 10/19] Update parameters_test.go --- .../horizon/internal/integration/parameters_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go index d7ba8819a8..87f7f2774a 100644 --- a/services/horizon/internal/integration/parameters_test.go +++ b/services/horizon/internal/integration/parameters_test.go @@ -40,7 +40,7 @@ var networkParamArgs = map[string]string{ } const ( - SIMPLE_CAPTIVE_CORE_TOML = ` + SimpleCaptiveCoreToml = ` PEER_PORT=11725 ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true @@ -54,7 +54,7 @@ const ( ADDRESS="localhost" QUALITY="MEDIUM"` - STELLAR_CORE_URL = "http://localhost:11626" + StellarCoreUrl = "http://localhost:11626" ) var ( @@ -73,7 +73,7 @@ func TestBucketDirDisallowed(t *testing.T) { } config := `BUCKET_DIR_PATH="/tmp" - ` + SIMPLE_CAPTIVE_CORE_TOML + ` + SimpleCaptiveCoreToml confName, _, cleanup := createCaptiveCoreConfig(config) defer cleanup() @@ -113,7 +113,7 @@ func TestEnvironmentPreserved(t *testing.T) { testConfig := integration.GetTestConfig() testConfig.HorizonEnvironment = map[string]string{ - "STELLAR_CORE_URL": STELLAR_CORE_URL, + "STELLAR_CORE_URL": StellarCoreUrl, } test := integration.NewTest(t, *testConfig) @@ -122,7 +122,7 @@ func TestEnvironmentPreserved(t *testing.T) { test.WaitForHorizon() envValue := os.Getenv("STELLAR_CORE_URL") - assert.Equal(t, STELLAR_CORE_URL, envValue) + assert.Equal(t, StellarCoreUrl, envValue) test.Shutdown() @@ -281,7 +281,7 @@ func TestCaptiveCoreConfigFilesystemState(t *testing.T) { t.Skip() // explained above } - confName, storagePath, cleanup := createCaptiveCoreConfig(SIMPLE_CAPTIVE_CORE_TOML) + confName, storagePath, cleanup := createCaptiveCoreConfig(SimpleCaptiveCoreToml) defer cleanup() localParams := integration.MergeMaps(defaultCaptiveCoreParameters, map[string]string{ From 1f4bc00a5af1b19dc1d25aba86802360283dd6a5 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 3 Nov 2023 13:47:11 -0400 Subject: [PATCH 11/19] Update start --- services/horizon/docker/verify-range/start | 2 ++ 1 file changed, 2 insertions(+) diff --git a/services/horizon/docker/verify-range/start b/services/horizon/docker/verify-range/start index 40a68a998d..bed3f76b24 100644 --- a/services/horizon/docker/verify-range/start +++ b/services/horizon/docker/verify-range/start @@ -73,6 +73,8 @@ export NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" export HISTORY_ARCHIVE_URLS="https://s3-eu-west-1.amazonaws.com/history.stellar.org/prd/core-live/core_live_001" export DATABASE_URL="postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable" export CAPTIVE_CORE_CONFIG_APPEND_PATH="/captive-core-pubnet.cfg" +export CAPTIVE_CORE_CONFIG_APPEND_PATH="/captive-core-pubnet.cfg" +export STELLAR_CORE_BINARY_PATH="/usr/bin/stellar-core" cd stellar-go git pull origin From 7efd402d3a724bc71d09a34ba66498fce33d8b61 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 3 Nov 2023 13:47:45 -0400 Subject: [PATCH 12/19] Update start --- services/horizon/docker/verify-range/start | 1 - 1 file changed, 1 deletion(-) diff --git a/services/horizon/docker/verify-range/start b/services/horizon/docker/verify-range/start index bed3f76b24..8da48db6cc 100644 --- a/services/horizon/docker/verify-range/start +++ b/services/horizon/docker/verify-range/start @@ -73,7 +73,6 @@ export NETWORK_PASSPHRASE="Public Global Stellar Network ; September 2015" export HISTORY_ARCHIVE_URLS="https://s3-eu-west-1.amazonaws.com/history.stellar.org/prd/core-live/core_live_001" export DATABASE_URL="postgres://postgres:postgres@localhost:5432/horizon?sslmode=disable" export CAPTIVE_CORE_CONFIG_APPEND_PATH="/captive-core-pubnet.cfg" -export CAPTIVE_CORE_CONFIG_APPEND_PATH="/captive-core-pubnet.cfg" export STELLAR_CORE_BINARY_PATH="/usr/bin/stellar-core" cd stellar-go From 0d1ba227b6c96407a4c5f12890b2bbdcb1607168 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 6 Nov 2023 14:45:42 -0500 Subject: [PATCH 13/19] Deprecate the flags instead of removing --- services/horizon/cmd/ingest.go | 7 ++-- services/horizon/internal/flags.go | 40 +++++++++++++++++++ .../internal/integration/parameters_test.go | 6 +-- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go index ff4e634201..36d7decf0f 100644 --- a/services/horizon/cmd/ingest.go +++ b/services/horizon/cmd/ingest.go @@ -212,12 +212,11 @@ var ingestStressTestCmd = &cobra.Command{ HistorySession: horizonSession, HistoryArchiveURLs: globalConfig.HistoryArchiveURLs, RoundingSlippageFilter: globalConfig.RoundingSlippageFilter, + CaptiveCoreBinaryPath: globalConfig.CaptiveCoreBinaryPath, + RemoteCaptiveCoreURL: globalConfig.RemoteCaptiveCoreURL, + CaptiveCoreConfigUseDB: globalConfig.CaptiveCoreConfigUseDB, } - ingestConfig.CaptiveCoreBinaryPath = globalConfig.CaptiveCoreBinaryPath - ingestConfig.RemoteCaptiveCoreURL = globalConfig.RemoteCaptiveCoreURL - ingestConfig.CaptiveCoreConfigUseDB = globalConfig.CaptiveCoreConfigUseDB - system, err := ingest.NewSystem(ingestConfig) if err != nil { return err diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 231f90f9a7..141a4e98e3 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -29,6 +29,8 @@ const ( DatabaseURLFlagName = "db-url" // IngestFlagName is the command line flag for enabling ingestion on the Horizon instance IngestFlagName = "ingest" + // StellarCoreDBURLFlagName is the command line flag for configuring the postgres Stellar Core URL + StellarCoreDBURLFlagName = "stellar-core-db-url" // StellarCoreURLFlagName is the command line flag for configuring the URL fore Stellar Core HTTP endpoint StellarCoreURLFlagName = "stellar-core-url" // StellarCoreBinaryPathName is the command line flag for configuring the path to the stellar core binary @@ -43,6 +45,8 @@ const ( CaptiveCoreConfigUseDB = "captive-core-use-db" // CaptiveCoreHTTPPortFlagName is the commandline flag for specifying captive core HTTP port CaptiveCoreHTTPPortFlagName = "captive-core-http-port" + // EnableCaptiveCoreIngestionFlagName is the commandline flag for enabling captive core ingestion + EnableCaptiveCoreIngestionFlagName = "enable-captive-core-ingestion" // NetworkPassphraseFlagName is the command line flag for specifying the network passphrase NetworkPassphraseFlagName = "network-passphrase" // HistoryArchiveURLsFlagName is the command line flag for specifying the history archive URLs @@ -246,6 +250,24 @@ func Flags() (*Config, support.ConfigOptions) { ConfigKey: &config.CaptiveCoreConfigUseDB, UsedInCommands: IngestionCommands, }, + &support.ConfigOption{ + Name: EnableCaptiveCoreIngestionFlagName, + OptType: types.String, + FlagDefault: "", + Required: false, + Hidden: true, + CustomSetValue: func(opt *support.ConfigOption) error { + if val := viper.GetString(opt.Name); val != "" { + stdLog.Printf( + "DEPRECATED - The usage of the flag --enable-captive-core-ingestion has been deprecated. " + + "Horizon now uses Captive-Core ingestion by default and this flag will soon be removed in " + + "the future.", + ) + } + return nil + }, + UsedInCommands: IngestionCommands, + }, &support.ConfigOption{ Name: EnableIngestionFilteringFlagName, OptType: types.String, @@ -309,6 +331,24 @@ func Flags() (*Config, support.ConfigOptions) { ConfigKey: &config.CaptiveCoreTomlParams.PeerPort, UsedInCommands: IngestionCommands, }, + &support.ConfigOption{ + Name: StellarCoreDBURLFlagName, + EnvVar: "STELLAR_CORE_DATABASE_URL", + OptType: types.String, + Required: false, + Hidden: true, + CustomSetValue: func(opt *support.ConfigOption) error { + if val := viper.GetString(opt.Name); val != "" { + stdLog.Printf( + "DEPRECATED - The usage of the flag --stellar-core-db-url has been deprecated. " + + "Horizon now uses Captive-Core ingestion by default and this flag will soon be removed in " + + "the future.", + ) + } + return nil + }, + UsedInCommands: IngestionCommands, + }, &support.ConfigOption{ Name: StellarCoreURLFlagName, ConfigKey: &config.StellarCoreURL, diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go index 87f7f2774a..01df7e3f3a 100644 --- a/services/horizon/internal/integration/parameters_test.go +++ b/services/horizon/internal/integration/parameters_test.go @@ -54,7 +54,7 @@ const ( ADDRESS="localhost" QUALITY="MEDIUM"` - StellarCoreUrl = "http://localhost:11626" + StellarCoreURL = "http://localhost:11626" ) var ( @@ -113,7 +113,7 @@ func TestEnvironmentPreserved(t *testing.T) { testConfig := integration.GetTestConfig() testConfig.HorizonEnvironment = map[string]string{ - "STELLAR_CORE_URL": StellarCoreUrl, + "STELLAR_CORE_URL": StellarCoreURL, } test := integration.NewTest(t, *testConfig) @@ -122,7 +122,7 @@ func TestEnvironmentPreserved(t *testing.T) { test.WaitForHorizon() envValue := os.Getenv("STELLAR_CORE_URL") - assert.Equal(t, StellarCoreUrl, envValue) + assert.Equal(t, StellarCoreURL, envValue) test.Shutdown() From c10d203442b115f30640d60dac643081de64e92f Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 6 Nov 2023 15:04:55 -0500 Subject: [PATCH 14/19] Remove ingestion ELSE condition --- services/horizon/CHANGELOG.md | 3 ++- services/horizon/internal/flags.go | 9 --------- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/services/horizon/CHANGELOG.md b/services/horizon/CHANGELOG.md index 84b897e2e6..f9122401f9 100644 --- a/services/horizon/CHANGELOG.md +++ b/services/horizon/CHANGELOG.md @@ -12,11 +12,12 @@ file. This project adheres to [Semantic Versioning](http://semver.org/). ### Added - Added new command-line flag `--network` to specify the Stellar network (pubnet or testnet), aiming at simplifying the configuration process by automatically configuring the following parameters based on the chosen network: `--history-archive-urls`, `--network-passphrase`, and `--captive-core-config-path` ([4949](https://github.com/stellar/go/pull/4949)). +- Add a deprecation warning for using command-line flags when running Horizon ([5051](https://github.com/stellar/go/pull/5051)) +- Deprecate configuration flags related to legacy non-captive core ingestion ([5100](https://github.com/stellar/go/pull/5100)) ### Fixed - The same slippage calculation from the [`v2.26.1`](#2261) hotfix now properly excludes spikes for smoother trade aggregation plots ([4999](https://github.com/stellar/go/pull/4999)). - Limit the display of global flags on command line help `-h` output ([5077](https://github.com/stellar/go/pull/5077)). -- Add a deprecation warning for using command-line flags when running Horizon ([5051](https://github.com/stellar/go/pull/5051)) ### DB Schema Migration - Drop unused indices from the Horizon database. For the database with full history, the migration is anticipated to take up to an hour and is expected to free up approximately 1.3TB of storage ([5081](https://github.com/stellar/go/pull/5081)). diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 141a4e98e3..1996e5b741 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -935,15 +935,6 @@ func ApplyFlags(config *Config, flags support.ConfigOptions, options ApplyOption if err != nil { return errors.Wrap(err, "error generating captive core configuration") } - } else { - if config.CaptiveCoreBinaryPath != "" || config.CaptiveCoreConfigPath != "" { - captiveCoreConfigFlag := captiveCoreConfigAppendPathName - if viper.GetString(CaptiveCoreConfigPathName) != "" { - captiveCoreConfigFlag = CaptiveCoreConfigPathName - } - return fmt.Errorf("invalid config: one or more captive core params passed (--%s or --%s) but --ingest not set. "+captiveCoreMigrationHint, - StellarCoreBinaryPathName, captiveCoreConfigFlag) - } } // Configure log file From f573408a8770c0a656d22b81190a350f82175ea0 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 6 Nov 2023 15:11:47 -0500 Subject: [PATCH 15/19] Update flags.go --- services/horizon/internal/flags.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 1996e5b741..56fd9f4883 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -57,8 +57,7 @@ const ( EnableIngestionFilteringFlagName = "exp-enable-ingestion-filtering" // DisableTxSubFlagName is the command line flag for disabling transaction submission feature of Horizon DisableTxSubFlagName = "disable-tx-sub" - - captiveCoreMigrationHint = "If you are migrating from Horizon 1.x.y, start with the Migration Guide here: https://developers.stellar.org/docs/run-api-server/migrating/" + // StellarPubnet is a constant representing the Stellar public network StellarPubnet = "pubnet" // StellarTestnet is a constant representing the Stellar test network From 58870a204fd5b4211a26cafff22f75bc85cfb41c Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 6 Nov 2023 16:16:59 -0500 Subject: [PATCH 16/19] Update flags.go --- services/horizon/internal/flags.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/horizon/internal/flags.go b/services/horizon/internal/flags.go index 56fd9f4883..6480b9f009 100644 --- a/services/horizon/internal/flags.go +++ b/services/horizon/internal/flags.go @@ -57,7 +57,7 @@ const ( EnableIngestionFilteringFlagName = "exp-enable-ingestion-filtering" // DisableTxSubFlagName is the command line flag for disabling transaction submission feature of Horizon DisableTxSubFlagName = "disable-tx-sub" - + // StellarPubnet is a constant representing the Stellar public network StellarPubnet = "pubnet" // StellarTestnet is a constant representing the Stellar test network From f169de8ed3d9b8ad25b5e2bad85d118ae3d56851 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 6 Nov 2023 16:37:52 -0500 Subject: [PATCH 17/19] Update parameters_test.go --- .../internal/integration/parameters_test.go | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go index 01df7e3f3a..527323b6d7 100644 --- a/services/horizon/internal/integration/parameters_test.go +++ b/services/horizon/internal/integration/parameters_test.go @@ -527,6 +527,45 @@ func TestDeprecatedOutputs(t *testing.T) { "Configuring section in the developer documentation on how to use them - "+ "https://developers.stellar.org/docs/run-api-server/configuring") }) + t.Run("deprecated output for --stellar-core-db-url and --enable-captive-core-ingestion", func(t *testing.T) { + originalStderr := os.Stderr + r, w, _ := os.Pipe() + os.Stderr = w + stdLog.SetOutput(os.Stderr) + + testConfig := integration.GetTestConfig() + testConfig.HorizonIngestParameters = map[string]string{ + "stellar-core-db-url": "temp-url", + "enable-captive-core-ingestion": "true", + } + test := integration.NewTest(t, *testConfig) + err := test.StartHorizon() + assert.NoError(t, err) + test.WaitForHorizon() + + // Use a wait group to wait for the goroutine to finish before proceeding + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + if err := w.Close(); err != nil { + t.Errorf("Failed to close Stdout") + return + } + }() + + outputBytes, _ := io.ReadAll(r) + wg.Wait() // Wait for the goroutine to finish before proceeding + _ = r.Close() + os.Stderr = originalStderr + + assert.Contains(t, string(outputBytes), "DEPRECATED - The usage of the flag --stellar-core-db-url has been deprecated. "+ + "Horizon now uses Captive-Core ingestion by default and this flag will soon be removed in "+ + "the future.") + assert.Contains(t, string(outputBytes), "DEPRECATED - The usage of the flag --enable-captive-core-ingestion has been deprecated. "+ + "Horizon now uses Captive-Core ingestion by default and this flag will soon be removed in "+ + "the future.") + }) } func TestGlobalFlagsOutput(t *testing.T) { From 196815c69fa39471af97c673fee08f5eb6a2717a Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 6 Nov 2023 17:48:04 -0500 Subject: [PATCH 18/19] Update parameters_test.go --- .../internal/integration/parameters_test.go | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/services/horizon/internal/integration/parameters_test.go b/services/horizon/internal/integration/parameters_test.go index 527323b6d7..75cf5bd1c6 100644 --- a/services/horizon/internal/integration/parameters_test.go +++ b/services/horizon/internal/integration/parameters_test.go @@ -559,6 +559,45 @@ func TestDeprecatedOutputs(t *testing.T) { _ = r.Close() os.Stderr = originalStderr + assert.Contains(t, string(outputBytes), "DEPRECATED - The usage of the flag --stellar-core-db-url has been deprecated. "+ + "Horizon now uses Captive-Core ingestion by default and this flag will soon be removed in "+ + "the future.") + assert.Contains(t, string(outputBytes), "DEPRECATED - The usage of the flag --enable-captive-core-ingestion has been deprecated. "+ + "Horizon now uses Captive-Core ingestion by default and this flag will soon be removed in "+ + "the future.") + }) + t.Run("deprecated output for env vars STELLAR_CORE_DATABASE_URL and ENABLE_CAPTIVE_CORE_INGESTION", func(t *testing.T) { + originalStderr := os.Stderr + r, w, _ := os.Pipe() + os.Stderr = w + stdLog.SetOutput(os.Stderr) + + testConfig := integration.GetTestConfig() + testConfig.HorizonEnvironment = map[string]string{ + "STELLAR_CORE_DATABASE_URL": "temp-url", + "ENABLE_CAPTIVE_CORE_INGESTION": "true", + } + test := integration.NewTest(t, *testConfig) + err := test.StartHorizon() + assert.NoError(t, err) + test.WaitForHorizon() + + // Use a wait group to wait for the goroutine to finish before proceeding + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + if err := w.Close(); err != nil { + t.Errorf("Failed to close Stdout") + return + } + }() + + outputBytes, _ := io.ReadAll(r) + wg.Wait() // Wait for the goroutine to finish before proceeding + _ = r.Close() + os.Stderr = originalStderr + assert.Contains(t, string(outputBytes), "DEPRECATED - The usage of the flag --stellar-core-db-url has been deprecated. "+ "Horizon now uses Captive-Core ingestion by default and this flag will soon be removed in "+ "the future.") From 45fcc9fc12bbf0f090dee9a38dd904e3068a6d7c Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 7 Nov 2023 10:40:24 -0500 Subject: [PATCH 19/19] Update ingest.go --- services/horizon/cmd/ingest.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go index 36d7decf0f..e2d38977ab 100644 --- a/services/horizon/cmd/ingest.go +++ b/services/horizon/cmd/ingest.go @@ -291,11 +291,10 @@ var ingestInitGenesisStateCmd = &cobra.Command{ CheckpointFrequency: globalConfig.CheckpointFrequency, RoundingSlippageFilter: globalConfig.RoundingSlippageFilter, EnableIngestionFiltering: globalConfig.EnableIngestionFiltering, + CaptiveCoreBinaryPath: globalConfig.CaptiveCoreBinaryPath, + CaptiveCoreConfigUseDB: globalConfig.CaptiveCoreConfigUseDB, } - ingestConfig.CaptiveCoreBinaryPath = globalConfig.CaptiveCoreBinaryPath - ingestConfig.CaptiveCoreConfigUseDB = globalConfig.CaptiveCoreConfigUseDB - system, err := ingest.NewSystem(ingestConfig) if err != nil { return err