From 40aaf54b02ec00863e1c386b71c255d70667a50e Mon Sep 17 00:00:00 2001 From: Tamir Sen Date: Thu, 7 May 2020 18:34:20 +0200 Subject: [PATCH 1/2] Create separate db connections for ingestion --- services/horizon/internal/init.go | 65 ++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 14 deletions(-) diff --git a/services/horizon/internal/init.go b/services/horizon/internal/init.go index d134a77e3e..55d889589e 100644 --- a/services/horizon/internal/init.go +++ b/services/horizon/internal/init.go @@ -19,33 +19,70 @@ import ( "github.com/stellar/go/support/log" ) -func mustInitHorizonDB(app *App) { - session, err := db.Open("postgres", app.config.DatabaseURL) +const maxIngestionDBConnections = 2 + +func mustNewDBSession(databaseURL string, maxIdle, maxOpen int) *db.Session { + session, err := db.Open("postgres", databaseURL) if err != nil { log.Fatalf("cannot open Horizon DB: %v", err) } - session.DB.SetMaxIdleConns(app.config.HorizonDBMaxIdleConnections) - session.DB.SetMaxOpenConns(app.config.HorizonDBMaxOpenConnections) - app.historyQ = &history.Q{session} + session.DB.SetMaxIdleConns(maxIdle) + session.DB.SetMaxOpenConns(maxOpen) + return session } -func mustInitCoreDB(app *App) { - session, err := db.Open("postgres", app.config.StellarCoreDatabaseURL) - if err != nil { - log.Fatalf("cannot open Core DB: %v", err) +func mustInitHorizonDB(app *App) { + maxIdle := app.config.HorizonDBMaxIdleConnections + maxOpen := app.config.HorizonDBMaxOpenConnections + // There are two connection pools, one for serving requests to horizon + // and another pool for ingestion. Ideally the total connections in both + // pools should be bounded by HorizonDBMaxOpenConnections. But, if the ingestion + // pool consumes a significant quota of HorizonDBMaxOpenConnections then we will + // allow a total of HorizonDBMaxOpenConnections + maxIngestionDBConnections connections. + if maxIdle >= maxIngestionDBConnections*2 { + maxIdle -= maxIngestionDBConnections } + if maxOpen >= maxIngestionDBConnections*2 { + maxOpen -= maxIngestionDBConnections + } + app.historyQ = &history.Q{mustNewDBSession( + app.config.DatabaseURL, + maxIdle, + maxOpen, + )} +} - session.DB.SetMaxIdleConns(app.config.CoreDBMaxIdleConnections) - session.DB.SetMaxOpenConns(app.config.CoreDBMaxOpenConnections) - app.coreQ = &core.Q{session} +func mustInitCoreDB(app *App) { + maxIdle := app.config.CoreDBMaxIdleConnections + maxOpen := app.config.CoreDBMaxOpenConnections + // There are two connection pools, one for serving requests to horizon + // and another pool for ingestion. Ideally the total connections in both + // pools should be bounded by CoreDBMaxOpenConnections. But, if the ingestion + // pool consumes a significant quota of CoreDBMaxOpenConnections then we will + // allow a total of CoreDBMaxOpenConnections + maxIngestionDBConnections connections. + if maxIdle >= maxIngestionDBConnections*2 { + maxIdle -= maxIngestionDBConnections + } + if maxOpen >= maxIngestionDBConnections*2 { + maxOpen -= maxIngestionDBConnections + } + app.coreQ = &core.Q{mustNewDBSession( + app.config.StellarCoreDatabaseURL, + maxIdle, + maxOpen, + )} } func initExpIngester(app *App, orderBookGraph *orderbook.OrderBookGraph) { var err error app.expingester, err = expingest.NewSystem(expingest.Config{ - CoreSession: app.CoreSession(context.Background()), - HistorySession: app.HorizonSession(context.Background()), + CoreSession: mustNewDBSession( + app.config.StellarCoreDatabaseURL, maxIngestionDBConnections, maxIngestionDBConnections, + ), + HistorySession: mustNewDBSession( + app.config.DatabaseURL, maxIngestionDBConnections, maxIngestionDBConnections, + ), NetworkPassphrase: app.config.NetworkPassphrase, // TODO: // Use the first archive for now. We don't have a mechanism to From 8caef3af651fc835ec45cdc149230808554c22b0 Mon Sep 17 00:00:00 2001 From: Tamir Sen Date: Fri, 8 May 2020 18:28:14 +0200 Subject: [PATCH 2/2] Code review feedback --- services/horizon/internal/expingest/main.go | 3 ++ services/horizon/internal/init.go | 46 ++++++++++----------- 2 files changed, 25 insertions(+), 24 deletions(-) diff --git a/services/horizon/internal/expingest/main.go b/services/horizon/internal/expingest/main.go index fdf6bdd014..34d4e76c8c 100644 --- a/services/horizon/internal/expingest/main.go +++ b/services/horizon/internal/expingest/main.go @@ -44,6 +44,9 @@ const ( // everything else). CurrentVersion = 10 + // MaxDBConnections is the size of the postgres connection pool dedicated to Horizon ingestion + MaxDBConnections = 2 + defaultCoreCursorName = "HORIZON" stateVerificationErrorThreshold = 3 ) diff --git a/services/horizon/internal/init.go b/services/horizon/internal/init.go index 55d889589e..5754010a97 100644 --- a/services/horizon/internal/init.go +++ b/services/horizon/internal/init.go @@ -19,8 +19,6 @@ import ( "github.com/stellar/go/support/log" ) -const maxIngestionDBConnections = 2 - func mustNewDBSession(databaseURL string, maxIdle, maxOpen int) *db.Session { session, err := db.Open("postgres", databaseURL) if err != nil { @@ -35,17 +33,17 @@ func mustNewDBSession(databaseURL string, maxIdle, maxOpen int) *db.Session { func mustInitHorizonDB(app *App) { maxIdle := app.config.HorizonDBMaxIdleConnections maxOpen := app.config.HorizonDBMaxOpenConnections - // There are two connection pools, one for serving requests to horizon - // and another pool for ingestion. Ideally the total connections in both - // pools should be bounded by HorizonDBMaxOpenConnections. But, if the ingestion - // pool consumes a significant quota of HorizonDBMaxOpenConnections then we will - // allow a total of HorizonDBMaxOpenConnections + maxIngestionDBConnections connections. - if maxIdle >= maxIngestionDBConnections*2 { - maxIdle -= maxIngestionDBConnections - } - if maxOpen >= maxIngestionDBConnections*2 { - maxOpen -= maxIngestionDBConnections + if app.config.Ingest || app.config.IngestInMemoryOnly { + maxIdle -= expingest.MaxDBConnections + maxOpen -= expingest.MaxDBConnections + if maxIdle <= 0 { + log.Fatalf("max idle connections to horizon db must be greater than %d", expingest.MaxDBConnections) + } + if maxOpen <= 0 { + log.Fatalf("max open connections to horizon db must be greater than %d", expingest.MaxDBConnections) + } } + app.historyQ = &history.Q{mustNewDBSession( app.config.DatabaseURL, maxIdle, @@ -56,17 +54,17 @@ func mustInitHorizonDB(app *App) { func mustInitCoreDB(app *App) { maxIdle := app.config.CoreDBMaxIdleConnections maxOpen := app.config.CoreDBMaxOpenConnections - // There are two connection pools, one for serving requests to horizon - // and another pool for ingestion. Ideally the total connections in both - // pools should be bounded by CoreDBMaxOpenConnections. But, if the ingestion - // pool consumes a significant quota of CoreDBMaxOpenConnections then we will - // allow a total of CoreDBMaxOpenConnections + maxIngestionDBConnections connections. - if maxIdle >= maxIngestionDBConnections*2 { - maxIdle -= maxIngestionDBConnections - } - if maxOpen >= maxIngestionDBConnections*2 { - maxOpen -= maxIngestionDBConnections + if app.config.Ingest || app.config.IngestInMemoryOnly { + maxIdle -= expingest.MaxDBConnections + maxOpen -= expingest.MaxDBConnections + if maxIdle <= 0 { + log.Fatalf("max idle connections to stellar-core db must be greater than %d", expingest.MaxDBConnections) + } + if maxOpen <= 0 { + log.Fatalf("max open connections to stellar-core db must be greater than %d", expingest.MaxDBConnections) + } } + app.coreQ = &core.Q{mustNewDBSession( app.config.StellarCoreDatabaseURL, maxIdle, @@ -78,10 +76,10 @@ func initExpIngester(app *App, orderBookGraph *orderbook.OrderBookGraph) { var err error app.expingester, err = expingest.NewSystem(expingest.Config{ CoreSession: mustNewDBSession( - app.config.StellarCoreDatabaseURL, maxIngestionDBConnections, maxIngestionDBConnections, + app.config.StellarCoreDatabaseURL, expingest.MaxDBConnections, expingest.MaxDBConnections, ), HistorySession: mustNewDBSession( - app.config.DatabaseURL, maxIngestionDBConnections, maxIngestionDBConnections, + app.config.DatabaseURL, expingest.MaxDBConnections, expingest.MaxDBConnections, ), NetworkPassphrase: app.config.NetworkPassphrase, // TODO: