services/horizon/internal/test/integration: Run captive core integration tests

.circleci/config.yml

@@ -142,6 +142,18 @@ commands:
           name: Check dependencies
           command: ./gomod.sh
 
+  # install_stellar_core installs the latest unstable version of stellar core.
+  install_stellar_core:
+    steps:
+      - run:
+          name: Install latest version of Stellar Core
+          command: |
+            sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
+            sudo bash -c 'echo "deb https://apt.stellar.org xenial unstable" > /etc/apt/sources.list.d/SDF-unstable.list'
+            sudo apt-get update && sudo apt-get install -y stellar-core
+            echo "using stellar core version $(stellar-core version)"
+            echo "export CAPTIVE_CORE_BIN=/usr/bin/stellar-core" >> $BASH_ENV
+
   check_ingest_state:
     steps:
       - run:
@@ -197,40 +209,13 @@ commands:
           name: Build release artifacts
           command: go run ./support/scripts/build_release_artifacts/main.go
 
-  # build_horizon builds horizon binary
-  build_horizon:
-    steps:
-      - run:
-          name: Build horizon binary
-          command: |
-            go build -v ./services/horizon
-
   # send_coverage_report sends coverage report to codecov.io
   send_coverage_report:
     steps:
       - run:
           name: Send report to codecov.io
           command: bash <(curl -s https://codecov.io/bash)
 
-  # run_horizon_integration_tests runs Horizon's integration tests
-  run_horizon_integration_tests:
-    parameters:
-      enable-captive-core:
-        type: boolean
-        default: false
-    steps:
-      - unless:
-          condition: << parameters.enable-captive-core >>
-          steps:
-            - run: docker run -d --env POSTGRES_HOST_AUTH_METHOD=trust -p 5432:5432 circleci/postgres:9.6.5-alpine
-      - run:
-          name: Run Horizon integration tests <<# parameters.enable-captive-core >>(With captive core)<</ parameters.enable-captive-core >>
-          # Currently all integration tests are in a single directory.
-          # Pulling the image helps with test running time
-          command: |
-            cd ~/go/src/github.com/stellar/go
-            <<# parameters.enable-captive-core >>HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE=true<</ parameters.enable-captive-core >> go test -timeout 25m -v ./services/horizon/internal/integration/...
-
 #-----------------------------------------------------------------------------#
 # Jobs use the commands to accomplish a given task, and run through workflows #
 #-----------------------------------------------------------------------------#
@@ -422,28 +407,37 @@ jobs:
   # test_horizon_integration performs Horizon integration tests, it's using
   # decicated vm machine to be able to start arbitrary docker containers.
   test_horizon_integration:
+    parameters:
+      enable-captive-core:
+        type: boolean
+        default: false
     working_directory: ~/go/src/github.com/stellar/go
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202010-01
     steps:
       - checkout
       - run:
           name: Setting env variables
-          command: |
-            echo "export HORIZON_INTEGRATION_TESTS=true" >> $BASH_ENV
-            echo "export HORIZON_BIN_DIR=~/go/src/github.com/stellar/go" >> $BASH_ENV
+          command: echo "export HORIZON_INTEGRATION_TESTS=true" >> $BASH_ENV
       - run:
           name: Pull latest Stellar Core image
           command: docker pull stellar/stellar-core
       - install_golang
-      - build_horizon
-      # run the integration tests ...
-      #   ... without captive core
-      - run_horizon_integration_tests
-      #   ... and with captive core
-      # This isn't working at the moment
-      # - run_horizon_integration_tests:
-      #     enable-captive-core: true
+      - run:
+          name: Start Horizon Postgres DB
+          command: docker run -d --env POSTGRES_HOST_AUTH_METHOD=trust -p 5432:5432 circleci/postgres:9.6.5-alpine
+      - when:
+          condition: << parameters.enable-captive-core >>
+          steps:
+            - install_stellar_core
+      - run:
+          name: Run Horizon integration tests <<# parameters.enable-captive-core >>(With captive core)<</ parameters.enable-captive-core >>
+          # Currently all integration tests are in a single directory.
+          # Pulling the image helps with test running time
+          command: |
+            cd ~/go/src/github.com/stellar/go
+            <<# parameters.enable-captive-core >>HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE=true<</ parameters.enable-captive-core >> go test -timeout 25m -v ./services/horizon/internal/integration/...
+
 #-------------------------------------------------------------------------#
 # Workflows orchestrate jobs and make sure they run in the right sequence #
 #-------------------------------------------------------------------------#
@@ -458,8 +452,13 @@ workflows:
       - test_code_1_14_postgres10
       - test_code_1_15
       - test_code_1_15_postgres10
-      # TODO: remove once integration_tests_nightly is confirmed to be working
+      # run the integration tests ...
+      #   ... without captive core
       - test_horizon_integration
+      #   ... and with captive core
+      - test_horizon_integration:
+          name: test_horizon_integration_with_captive_core
+          enable-captive-core: true
       - publish_state_diff_docker_image:
           filters:
               branches:
@@ -504,16 +503,3 @@ workflows:
           # Pushing stellar/horizon:latest to docker hub requires manual approval
           requires:
             - hold
-
-  integration_tests_nightly:
-    triggers:
-      - schedule:
-          # Every day
-          cron: "0 0 * * *"
-          filters:
-            branches:
-              only:
-                - master
-                - /release-horizon-/
-    jobs:
-      - test_horizon_integration

ingest/ledgerbackend/captive_core_backend.go

@@ -19,7 +19,7 @@ var _ LedgerBackend = (*CaptiveStellarCore)(nil)
 
 func (c *CaptiveStellarCore) roundDownToFirstReplayAfterCheckpointStart(ledger uint32) uint32 {
 	r := c.checkpointManager.GetCheckpointRange(ledger)
-	if r.Low == 0 {
+	if r.Low <= 1 {
 		// Stellar-Core doesn't stream ledger 1
 		return 2
 	}

services/horizon/cmd/db.go

@@ -249,6 +249,7 @@ func RunDBReingestRange(from, to uint32, reingestForce bool, parallelWorkers uin
 		EnableCaptiveCore:           config.EnableCaptiveCoreIngestion,
 		CaptiveCoreBinaryPath:       config.CaptiveCoreBinaryPath,
 		RemoteCaptiveCoreURL:        config.RemoteCaptiveCoreURL,
+		CaptiveCoreConfigAppendPath: config.CaptiveCoreConfigAppendPath,
 	}
 
 	if !ingestConfig.EnableCaptiveCore {

services/horizon/docker/captive-core-integration-tests.cfg

@@ -0,0 +1,13 @@
+PEER_PORT=11725
+ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true
+
+UNSAFE_QUORUM=true
+FAILURE_SAFETY=0
+
+[[VALIDATORS]]
+NAME="local_core"
+HOME_DOMAIN="core.local"
+# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK"
+PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS"
+ADDRESS="localhost"
+QUALITY="MEDIUM"

services/horizon/docker/captive-core-reingest-range-integration-tests.cfg

@@ -0,0 +1,9 @@
+ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true
+
+[[VALIDATORS]]
+NAME="local_core"
+HOME_DOMAIN="core.local"
+# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK"
+PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS"
+ADDRESS="localhost"
+QUALITY="MEDIUM"

services/horizon/docker/stellar-core-integration-tests.cfg

@@ -1,6 +1,3 @@
-# simple configuration for a standalone test "network"
-# see stellar-core_example.cfg for a description of the configuration parameters
-
 RUN_STANDALONE=false
 ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true
 

services/horizon/internal/app.go

@@ -128,13 +128,22 @@ func (a *App) Serve() {
 	}
 
 	// configure shutdown signal handler
-	done := make(chan os.Signal, 1)
-	signal.Notify(done, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
+	signalChan := make(chan os.Signal, 1)
+	signal.Notify(signalChan, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
 	go func() {
-		<-done
-		a.Close()
+		select {
+		case <-signalChan:
+			a.Close()
+		case <-a.done:
+			return
+		}
+	}()
+
+	wg.Add(1)
+	go func() {
+		a.waitForDone()
+		wg.Done()
 	}()
-	go a.waitForDone()
 
 	err := a.webServer.Serve()
 	if err != nil && err != http.ErrServerClosed {

services/horizon/internal/ingest/fsm.go

@@ -656,6 +656,11 @@ func (h reingestHistoryRangeState) run(s *system) (transition, error) {
 		return stop(), errors.Errorf("invalid range: [%d, %d]", h.fromLedger, h.toLedger)
 	}
 
+	if h.fromLedger == 1 {
+		log.Warn("Ledger 1 is pregenerated and not available, starting from ledger 2.")
+		h.fromLedger = 2
+	}
+
 	log.WithFields(logpkg.F{
 		"from": h.fromLedger,
 		"to":   h.toLedger,
@@ -673,11 +678,6 @@ func (h reingestHistoryRangeState) run(s *system) (transition, error) {
 		"duration": time.Since(startTime).Seconds(),
 	}).Info("Range ready")
 
-	if h.fromLedger == 1 {
-		log.Warn("Ledger 1 is pregenerated and not available, starting from ledger 2.")
-		h.fromLedger = 2
-	}
-
 	startTime = time.Now()
 
 	if h.force {

services/horizon/internal/integration/db_test.go

@@ -1,11 +1,13 @@
 package integration
 
 import (
+	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/stretchr/testify/assert"
 
+	"github.com/stellar/go/historyarchive"
 	horizoncmd "github.com/stellar/go/services/horizon/cmd"
 	"github.com/stellar/go/services/horizon/internal/db2/schema"
 	"github.com/stellar/go/services/horizon/internal/test/integration"
@@ -56,8 +58,36 @@ func TestReingestDB(t *testing.T) {
 	_, err = schema.Migrate(dbConn.DB.DB, schema.MigrateUp, 0)
 	tt.NoError(err)
 
+	// cap reachedLedger to the nearest checkpoint ledger because reingest range cannot ingest past the most
+	// recent checkpoint ledger when using captive core
+	toLedger := uint32(reachedLedger)
+	archive, err := historyarchive.Connect(horizonConfig.HistoryArchiveURLs[0], historyarchive.ConnectOptions{
+		NetworkPassphrase:   horizonConfig.NetworkPassphrase,
+		CheckpointFrequency: horizonConfig.CheckpointFrequency,
+	})
+	tt.NoError(err)
+
+	// make sure a full checkpoint has elapsed otherwise there will be nothing to reingest
+	var latestCheckpoint uint32
+	publishedFirstCheckpoint := func() bool {
+		has, requestErr := archive.GetRootHAS()
+		tt.NoError(requestErr)
+		latestCheckpoint = has.CurrentLedger
+		return latestCheckpoint > 1
+	}
+	tt.Eventually(publishedFirstCheckpoint, 10*time.Second, time.Second)
+
+	if toLedger > latestCheckpoint {
+		toLedger = latestCheckpoint
+	}
+
+	horizonConfig.CaptiveCoreConfigAppendPath = filepath.Join(
+		filepath.Dir(horizonConfig.CaptiveCoreConfigAppendPath),
+		"captive-core-reingest-range-integration-tests.cfg",
+	)
+
 	// Reingest into the DB
-	err = horizoncmd.RunDBReingestRange(1, uint32(reachedLedger), false, 1, horizonConfig)
+	err = horizoncmd.RunDBReingestRange(1, toLedger, false, 1, horizonConfig)
 	tt.NoError(err)
 }
 
@@ -69,9 +99,8 @@ func TestResumeFromInitializedDB(t *testing.T) {
 	oldDBURL := itest.GetHorizonConfig().DatabaseURL
 	itestConfig := protocol15Config
 	itestConfig.PostgresURL = oldDBURL
-	itest.Shutdown()
 
-	itest = integration.NewTest(t, itestConfig)
+	itest.RestartHorizon()
 
 	successfullyResumed := func() bool {
 		root, err := itest.Client().Root()