Merge branch 'master' into release-horizon-v2.0.0-beta

stellar · Jan 6, 2021 · 0821039 · 0821039
2 parents ed3ba91 + 6cad160
commit 0821039
Show file tree

Hide file tree

Showing 31 changed files with 376 additions and 248 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -142,6 +142,18 @@ commands:
           name: Check dependencies
           command: ./gomod.sh
 
+  # install_stellar_core installs the latest unstable version of stellar core.
+  install_stellar_core:
+    steps:
+      - run:
+          name: Install latest version of Stellar Core
+          command: |
+            sudo wget -qO - https://apt.stellar.org/SDF.asc | APT_KEY_DONT_WARN_ON_DANGEROUS_USAGE=true sudo apt-key add -
+            sudo bash -c 'echo "deb https://apt.stellar.org xenial unstable" > /etc/apt/sources.list.d/SDF-unstable.list'
+            sudo apt-get update && sudo apt-get install -y stellar-core
+            echo "using stellar core version $(stellar-core version)"
+            echo "export CAPTIVE_CORE_BIN=/usr/bin/stellar-core" >> $BASH_ENV
+
   check_ingest_state:
     steps:
       - run:
@@ -197,40 +209,13 @@ commands:
           name: Build release artifacts
           command: go run ./support/scripts/build_release_artifacts/main.go
 
-  # build_horizon builds horizon binary
-  build_horizon:
-    steps:
-      - run:
-          name: Build horizon binary
-          command: |
-            go build -v ./services/horizon
-
   # send_coverage_report sends coverage report to codecov.io
   send_coverage_report:
     steps:
       - run:
           name: Send report to codecov.io
           command: bash <(curl -s https://codecov.io/bash)
 
-  # run_horizon_integration_tests runs Horizon's integration tests
-  run_horizon_integration_tests:
-    parameters:
-      enable-captive-core:
-        type: boolean
-        default: false
-    steps:
-      - unless:
-          condition: << parameters.enable-captive-core >>
-          steps:
-            - run: docker run -d --env POSTGRES_HOST_AUTH_METHOD=trust -p 5432:5432 circleci/postgres:9.6.5-alpine
-      - run:
-          name: Run Horizon integration tests <<# parameters.enable-captive-core >>(With captive core)<</ parameters.enable-captive-core >>
-          # Currently all integration tests are in a single directory.
-          # Pulling the image helps with test running time
-          command: |
-            cd ~/go/src/github.com/stellar/go
-            <<# parameters.enable-captive-core >>HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE=true<</ parameters.enable-captive-core >> go test -timeout 25m -v ./services/horizon/internal/integration/...
-
 #-----------------------------------------------------------------------------#
 # Jobs use the commands to accomplish a given task, and run through workflows #
 #-----------------------------------------------------------------------------#
@@ -422,28 +407,37 @@ jobs:
   # test_horizon_integration performs Horizon integration tests, it's using
   # decicated vm machine to be able to start arbitrary docker containers.
   test_horizon_integration:
+    parameters:
+      enable-captive-core:
+        type: boolean
+        default: false
     working_directory: ~/go/src/github.com/stellar/go
     machine:
-      image: ubuntu-1604:201903-01
+      image: ubuntu-1604:202010-01
     steps:
       - checkout
       - run:
           name: Setting env variables
-          command: |
-            echo "export HORIZON_INTEGRATION_TESTS=true" >> $BASH_ENV
-            echo "export HORIZON_BIN_DIR=~/go/src/github.com/stellar/go" >> $BASH_ENV
+          command: echo "export HORIZON_INTEGRATION_TESTS=true" >> $BASH_ENV
       - run:
           name: Pull latest Stellar Core image
           command: docker pull stellar/stellar-core
       - install_golang
-      - build_horizon
-      # run the integration tests ...
-      #   ... without captive core
-      - run_horizon_integration_tests
-      #   ... and with captive core
-      # This isn't working at the moment
-      # - run_horizon_integration_tests:
-      #     enable-captive-core: true
+      - run:
+          name: Start Horizon Postgres DB
+          command: docker run -d --env POSTGRES_HOST_AUTH_METHOD=trust -p 5432:5432 circleci/postgres:9.6.5-alpine
+      - when:
+          condition: << parameters.enable-captive-core >>
+          steps:
+            - install_stellar_core
+      - run:
+          name: Run Horizon integration tests <<# parameters.enable-captive-core >>(With captive core)<</ parameters.enable-captive-core >>
+          # Currently all integration tests are in a single directory.
+          # Pulling the image helps with test running time
+          command: |
+            cd ~/go/src/github.com/stellar/go
+            <<# parameters.enable-captive-core >>HORIZON_INTEGRATION_ENABLE_CAPTIVE_CORE=true<</ parameters.enable-captive-core >> go test -timeout 25m -v ./services/horizon/internal/integration/...
+
 #-------------------------------------------------------------------------#
 # Workflows orchestrate jobs and make sure they run in the right sequence #
 #-------------------------------------------------------------------------#
@@ -458,8 +452,13 @@ workflows:
       - test_code_1_14_postgres10
       - test_code_1_15
       - test_code_1_15_postgres10
-      # TODO: remove once integration_tests_nightly is confirmed to be working
+      # run the integration tests ...
+      #   ... without captive core
       - test_horizon_integration
+      #   ... and with captive core
+      - test_horizon_integration:
+          name: test_horizon_integration_with_captive_core
+          enable-captive-core: true
       - publish_state_diff_docker_image:
           filters:
               branches:
@@ -504,16 +503,3 @@ workflows:
           # Pushing stellar/horizon:latest to docker hub requires manual approval
           requires:
             - hold
-
-  integration_tests_nightly:
-    triggers:
-      - schedule:
-          # Every day
-          cron: "0 0 * * *"
-          filters:
-            branches:
-              only:
-                - master
-                - /release-horizon-/
-    jobs:
-      - test_horizon_integration
diff --git a/ingest/adapters/mocks.go b/ingest/adapters/mocks.go
diff --git a/ingest/ledgerbackend/captive_core_backend.go b/ingest/ledgerbackend/captive_core_backend.go
@@ -19,7 +19,7 @@ var _ LedgerBackend = (*CaptiveStellarCore)(nil)
 
 func (c *CaptiveStellarCore) roundDownToFirstReplayAfterCheckpointStart(ledger uint32) uint32 {
 	r := c.checkpointManager.GetCheckpointRange(ledger)
-	if r.Low == 0 {
+	if r.Low <= 1 {
 		// Stellar-Core doesn't stream ledger 1
 		return 2
 	}

diff --git a/ingest/ledgerbackend/captive_core_backend_test.go b/ingest/ledgerbackend/captive_core_backend_test.go
@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/hex"
 	"fmt"
-	"io"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -131,13 +130,6 @@ type testLedgerHeader struct {
 	previousLedgerHash string
 }
 
-func writeLedgerHeader(w io.Writer, header testLedgerHeader) {
-	err := xdr.MarshalFramed(w, buildLedgerCloseMeta(header))
-	if err != nil {
-		panic(err)
-	}
-}
-
 func TestCaptiveNew(t *testing.T) {
 	executablePath := "/etc/stellar-core"
 	configPath := "/etc/stellar-core.cfg"
@@ -612,7 +604,6 @@ func TestCaptiveGetLedger(t *testing.T) {
 }
 
 func TestCaptiveGetLedger_NextLedgerIsDifferentToLedgerFromBuffer(t *testing.T) {
-	tt := assert.New(t)
 	metaChan := make(chan metaResult, 100)
 
 	for i := 64; i <= 65; i++ {
@@ -633,30 +624,45 @@ func TestCaptiveGetLedger_NextLedgerIsDifferentToLedgerFromBuffer(t *testing.T)
 	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
 	mockRunner.On("context").Return(context.Background())
 	mockRunner.On("close").Return(nil)
+}
+
+func TestCaptiveStellarCore_PrepareRangeAfterClose(t *testing.T) {
+	executablePath := "/etc/stellar-core"
+	networkPassphrase := network.PublicNetworkPassphrase
+	historyURLs := []string{"http://localhost"}
+
+	captiveStellarCore, err := NewCaptive(
+		CaptiveCoreConfig{
+			BinaryPath:         executablePath,
+			NetworkPassphrase:  networkPassphrase,
+			HistoryArchiveURLs: historyURLs,
+		},
+	)
+	assert.NoError(t, err)
+
+	assert.NoError(t, captiveStellarCore.Close())
+
+	assert.EqualError(
+		t,
+		captiveStellarCore.PrepareRange(BoundedRange(65, 66)),
+		"error starting prepare range: opening subprocess: error getting latest checkpoint sequence: "+
+			"error getting root HAS: Get \"http://localhost/.well-known/stellar-history.json\": context canceled",
+	)
 
+	// even if the request to fetch the latest checkpoint succeeds, we should fail at creating the subprocess
 	mockArchive := &historyarchive.MockArchive{}
 	mockArchive.
 		On("GetRootHAS").
 		Return(historyarchive.HistoryArchiveState{
 			CurrentLedger: uint32(200),
 		}, nil)
-
-	captiveBackend := CaptiveStellarCore{
-		archive: mockArchive,
-		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
-			return mockRunner, nil
-		},
-		checkpointManager: historyarchive.NewCheckpointManager(64),
-	}
-
-	err := captiveBackend.PrepareRange(BoundedRange(65, 66))
-	assert.NoError(t, err)
-
-	_, _, err = captiveBackend.GetLedger(66)
-	tt.EqualError(err, "unexpected ledger sequence (expected=66 actual=68)")
-
+	captiveStellarCore.archive = mockArchive
+	assert.EqualError(
+		t,
+		captiveStellarCore.PrepareRange(BoundedRange(65, 66)),
+		"error starting prepare range: opening subprocess: error running stellar-core: context canceled",
+	)
 	mockArchive.AssertExpectations(t)
-	mockRunner.AssertExpectations(t)
 }
 
 func TestCaptiveGetLedger_ErrReadingMetaResult(t *testing.T) {
@@ -683,6 +689,7 @@ func TestCaptiveGetLedger_ErrReadingMetaResult(t *testing.T) {
 		cancel()
 	}).Once()
 
+	// even if the request to fetch the latest checkpoint succeeds, we should fail at creating the subprocess
 	mockArchive := &historyarchive.MockArchive{}
 	mockArchive.
 		On("GetRootHAS").
@@ -759,6 +766,60 @@ func TestCaptiveGetLedger_ErrClosingAfterLastLedger(t *testing.T) {
 	mockRunner.AssertExpectations(t)
 }
 
+func TestCaptiveAfterClose(t *testing.T) {
+	metaChan := make(chan metaResult, 100)
+
+	for i := 64; i <= 66; i++ {
+		meta := buildLedgerCloseMeta(testLedgerHeader{sequence: uint32(i)})
+		metaChan <- metaResult{
+			LedgerCloseMeta: &meta,
+		}
+	}
+
+	mockRunner := &stellarCoreRunnerMock{}
+	ctx, cancel := context.WithCancel(context.Background())
+	mockRunner.On("catchup", uint32(65), uint32(66)).Return(nil)
+	mockRunner.On("getMetaPipe").Return((<-chan metaResult)(metaChan))
+	mockRunner.On("context").Return(ctx)
+	mockRunner.On("close").Return(nil).Once()
+
+	mockArchive := &historyarchive.MockArchive{}
+	mockArchive.
+		On("GetRootHAS").
+		Return(historyarchive.HistoryArchiveState{
+			CurrentLedger: uint32(200),
+		}, nil)
+
+	captiveBackend := CaptiveStellarCore{
+		archive: mockArchive,
+		stellarCoreRunnerFactory: func(_ stellarCoreRunnerMode) (stellarCoreRunnerInterface, error) {
+			return mockRunner, nil
+		},
+		checkpointManager: historyarchive.NewCheckpointManager(64),
+		cancel:            cancel,
+	}
+
+	boundedRange := BoundedRange(65, 66)
+	err := captiveBackend.PrepareRange(boundedRange)
+	assert.NoError(t, err)
+
+	assert.NoError(t, captiveBackend.Close())
+
+	_, _, err = captiveBackend.GetLedger(boundedRange.to)
+	assert.EqualError(t, err, "session is closed, call PrepareRange first")
+
+	var prepared bool
+	prepared, err = captiveBackend.IsPrepared(boundedRange)
+	assert.False(t, prepared)
+	assert.NoError(t, err)
+
+	_, err = captiveBackend.GetLatestLedgerSequence()
+	assert.EqualError(t, err, "stellar-core must be opened to return latest available sequence")
+
+	mockArchive.AssertExpectations(t)
+	mockRunner.AssertExpectations(t)
+}
+
 func TestGetLedgerBoundsCheck(t *testing.T) {
 	metaChan := make(chan metaResult, 100)
 

diff --git a/services/horizon/cmd/db.go b/services/horizon/cmd/db.go
@@ -249,6 +249,7 @@ func RunDBReingestRange(from, to uint32, reingestForce bool, parallelWorkers uin
 		EnableCaptiveCore:           config.EnableCaptiveCoreIngestion,
 		CaptiveCoreBinaryPath:       config.CaptiveCoreBinaryPath,
 		RemoteCaptiveCoreURL:        config.RemoteCaptiveCoreURL,
+		CaptiveCoreConfigAppendPath: config.CaptiveCoreConfigAppendPath,
 	}
 
 	if !ingestConfig.EnableCaptiveCore {

diff --git a/services/horizon/docker/captive-core-integration-tests.cfg b/services/horizon/docker/captive-core-integration-tests.cfg
@@ -0,0 +1,13 @@
+PEER_PORT=11725
+ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true
+
+UNSAFE_QUORUM=true
+FAILURE_SAFETY=0
+
+[[VALIDATORS]]
+NAME="local_core"
+HOME_DOMAIN="core.local"
+# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK"
+PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS"
+ADDRESS="localhost"
+QUALITY="MEDIUM"
diff --git a/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg b/services/horizon/docker/captive-core-reingest-range-integration-tests.cfg
@@ -0,0 +1,9 @@
+ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true
+
+[[VALIDATORS]]
+NAME="local_core"
+HOME_DOMAIN="core.local"
+# From "SACJC372QBSSKJYTV5A7LWT4NXWHTQO6GHG4QDAVC2XDPX6CNNXFZ4JK"
+PUBLIC_KEY="GD5KD2KEZJIGTC63IGW6UMUSMVUVG5IHG64HUTFWCHVZH2N2IBOQN7PS"
+ADDRESS="localhost"
+QUALITY="MEDIUM"
diff --git a/services/horizon/docker/stellar-core-integration-tests.cfg b/services/horizon/docker/stellar-core-integration-tests.cfg
@@ -1,6 +1,3 @@
-# simple configuration for a standalone test "network"
-# see stellar-core_example.cfg for a description of the configuration parameters
-
 RUN_STANDALONE=false
 ARTIFICIALLY_ACCELERATE_TIME_FOR_TESTING=true