services/horizon: Use local Horizon in integration tests (#3106)

The integration tests require that you have a compiled horizon binary present on the machine. The tests then copy that binary into the Stellar quickstart docker container. This PR removes this step. Instead of relying on Horizon running within docker, the test setup will create a new Horizon app and start serving requests. In the test tear down, the Horizon app is shut down.
stellar · Oct 9, 2020 · b0ec1a1 · b0ec1a1
1 parent 00b2fec
commit b0ec1a1
Show file tree

Hide file tree

Showing 13 changed files with 676 additions and 527 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -219,6 +219,10 @@ commands:
         type: boolean
         default: false
     steps:
+      - unless:
+          condition: << parameters.enable-captive-core >>
+          steps:
+            - run: docker run -d --env POSTGRES_HOST_AUTH_METHOD=trust -p 5432:5432 circleci/postgres:9.6.5-alpine
       - run:
           name: Run Horizon integration tests <<# parameters.enable-captive-core >>(With captive core)<</ parameters.enable-captive-core >>
           # Currently all integration tests are in a single directory.

diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go
@@ -11,6 +11,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
+	horizon "github.com/stellar/go/services/horizon/internal"
 	"github.com/stellar/go/services/horizon/internal/db2/schema"
 	"github.com/stellar/go/services/horizon/internal/expingest"
 	support "github.com/stellar/go/support/config"
@@ -24,6 +25,15 @@ var dbCmd = &cobra.Command{
 	Short: "commands to manage horizon's postgres db",
 }
 
+var dbURLConfigOption = support.ConfigOption{
+	Name:      "db-url",
+	EnvVar:    "DATABASE_URL",
+	ConfigKey: &config.DatabaseURL,
+	OptType:   types.String,
+	Required:  true,
+	Usage:     "horizon postgres database to connect with",
+}
+
 var dbInitCmd = &cobra.Command{
 	Use:   "init",
 	Short: "install schema",
@@ -79,13 +89,12 @@ var dbMigrateCmd = &cobra.Command{
 		dbURLConfigOption.Require()
 		dbURLConfigOption.SetValue()
 
-		db, err := sql.Open("postgres", viper.GetString("db-url"))
+		dbConn, err := db.Open("postgres", viper.GetString("db-url"))
 		if err != nil {
 			log.Fatal(err)
 		}
-		pingDB(db)
 
-		numMigrationsRun, err := schema.Migrate(db, dir, count)
+		numMigrationsRun, err := schema.Migrate(dbConn.DB.DB, dir, count)
 		if err != nil {
 			log.Fatal(err)
 		}
@@ -103,7 +112,7 @@ var dbReapCmd = &cobra.Command{
 	Short: "reaps (i.e. removes) any reapable history data",
 	Long:  "reap removes any historical data that is earlier than the configured retention cutoff",
 	Run: func(cmd *cobra.Command, args []string) {
-		app := initApp()
+		app := horizon.NewAppFromFlags(config, flags)
 		app.UpdateLedgerState()
 		err := app.DeleteUnretainedHistory()
 		if err != nil {
@@ -202,7 +211,7 @@ var dbReingestRangeCmd = &cobra.Command{
 			argsInt32[i] = uint32(seq)
 		}
 
-		initRootConfig()
+		horizon.ApplyFlags(config, flags)
 
 		horizonSession, err := db.Open("postgres", config.DatabaseURL)
 		if err != nil {
@@ -222,7 +231,7 @@ var dbReingestRangeCmd = &cobra.Command{
 
 		if !ingestConfig.EnableCaptiveCore {
 			if config.StellarCoreDatabaseURL == "" {
-				log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName)
+				log.Fatalf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName)
 			}
 			coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL)
 			if dbErr != nil {

diff --git a/services/horizon/cmd/ingest.go b/services/horizon/cmd/ingest.go
@@ -9,6 +9,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 	"github.com/stellar/go/historyarchive"
+	horizon "github.com/stellar/go/services/horizon/internal"
 	"github.com/stellar/go/services/horizon/internal/db2/history"
 	"github.com/stellar/go/services/horizon/internal/expingest"
 	support "github.com/stellar/go/support/config"
@@ -69,7 +70,7 @@ var ingestVerifyRangeCmd = &cobra.Command{
 			co.SetValue()
 		}
 
-		initRootConfig()
+		horizon.ApplyFlags(config, flags)
 
 		if ingestVerifyDebugServerPort != 0 {
 			go func() {
@@ -108,7 +109,7 @@ var ingestVerifyRangeCmd = &cobra.Command{
 
 		if !ingestConfig.EnableCaptiveCore {
 			if config.StellarCoreDatabaseURL == "" {
-				log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName)
+				log.Fatalf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName)
 			}
 
 			coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL)
@@ -167,7 +168,7 @@ var ingestStressTestCmd = &cobra.Command{
 			co.SetValue()
 		}
 
-		initRootConfig()
+		horizon.ApplyFlags(config, flags)
 
 		horizonSession, err := db.Open("postgres", config.DatabaseURL)
 		if err != nil {
@@ -193,7 +194,7 @@ var ingestStressTestCmd = &cobra.Command{
 			ingestConfig.RemoteCaptiveCoreURL = config.RemoteCaptiveCoreURL
 		} else {
 			if config.StellarCoreDatabaseURL == "" {
-				log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName)
+				log.Fatalf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName)
 			}
 
 			coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL)
@@ -224,7 +225,7 @@ var ingestTriggerStateRebuildCmd = &cobra.Command{
 	Use:   "trigger-state-rebuild",
 	Short: "updates a database to trigger state rebuild, state will be rebuilt by a running Horizon instance, DO NOT RUN production DB, some endpoints will be unavailable until state is rebuilt",
 	Run: func(cmd *cobra.Command, args []string) {
-		initRootConfig()
+		horizon.ApplyFlags(config, flags)
 
 		horizonSession, err := db.Open("postgres", config.DatabaseURL)
 		if err != nil {
@@ -245,7 +246,7 @@ var ingestInitGenesisStateCmd = &cobra.Command{
 	Use:   "init-genesis-state",
 	Short: "ingests genesis state (ledger 1)",
 	Run: func(cmd *cobra.Command, args []string) {
-		initRootConfig()
+		horizon.ApplyFlags(config, flags)
 
 		horizonSession, err := db.Open("postgres", config.DatabaseURL)
 		if err != nil {
@@ -273,7 +274,7 @@ var ingestInitGenesisStateCmd = &cobra.Command{
 			ingestConfig.StellarCoreBinaryPath = config.StellarCoreBinaryPath
 		} else {
 			if config.StellarCoreDatabaseURL == "" {
-				log.Fatalf("flag --%s cannot be empty", stellarCoreDBURLFlagName)
+				log.Fatalf("flag --%s cannot be empty", horizon.StellarCoreDBURLFlagName)
 			}
 
 			coreSession, dbErr := db.Open("postgres", config.StellarCoreDatabaseURL)