services/horizon: Check state rebuild in state verification integration test

.circleci/config.yml

@@ -408,7 +408,7 @@ jobs:
           command: |
             cd ~/go/src/github.com/stellar/go
             docker pull stellar/quickstart:testing
-            go test -timeout 20m -v ./services/horizon/internal/integration/...
+            go test -timeout 25m -v ./services/horizon/internal/integration/...
 #-------------------------------------------------------------------------#
 # Workflows orchestrate jobs and make sure they run in the right sequence #
 #-------------------------------------------------------------------------#

services/horizon/internal/integration/protocol14_state_verifier_test.go

@@ -100,34 +100,62 @@ func TestProtocol14StateVerifier(t *testing.T) {
 
 	// Wait for the first checkpoint ledger
 	for !itest.LedgerIngested(63) {
-		t.Log("First checkpoint ledger (63) not closed yet...")
+		t.Log("First checkpoint ledger (63) not ingested yet...")
 		time.Sleep(5 * time.Second)
 	}
 
-	var metrics string
+	verified := waitForStateVerifications(t, itest, 1)
+	if !verified {
+		t.Fatal("State verification not run...")
+	}
+
+	// Trigger state rebuild to check if ingesting from history archive works
+	itest.RunHorizonCLICommand([]string{"expingest", "trigger-state-rebuild"})
+
+	// Wait for the second checkpoint ledger and state rebuild
+	for !itest.LedgerClosed(127) {
+		t.Log("First checkpoint ledger (127) not closed yet...")
+		time.Sleep(5 * time.Second)
+	}
+
+	// Wait for the third checkpoint ledger and state verification trigger
+	for !itest.LedgerIngested(191) {
+		t.Log("First checkpoint ledger (191) not ingested yet...")
+		time.Sleep(5 * time.Second)
+	}
+
+	verified = waitForStateVerifications(t, itest, 2)
+	if !verified {
+		t.Fatal("State verification not run...")
+	}
+}
 
+func waitForStateVerifications(t *testing.T, itest *test.IntegrationTest, count int) bool {
 	// Check metrics until state verification run
-	for i := 0; i < 60; i++ {
+	for i := 0; i < 120; i++ {
 		t.Logf("Checking metrics (%d attempt)\n", i)
 		res, err := http.Get(fmt.Sprintf("http://localhost:%d/metrics", itest.AdminPort()))
 		assert.NoError(t, err)
 
 		metricsBytes, err := ioutil.ReadAll(res.Body)
 		res.Body.Close()
 		assert.NoError(t, err)
-		metrics = string(metricsBytes)
+		metrics := string(metricsBytes)
 
 		stateInvalid := strings.Contains(metrics, "horizon_ingest_state_invalid 1")
 		assert.False(t, stateInvalid, "State is invalid!")
 
-		notVerifiedYet := strings.Contains(metrics, "horizon_ingest_state_verify_duration_seconds_count 0")
+		notVerifiedYet := strings.Contains(
+			metrics,
+			fmt.Sprintf("horizon_ingest_state_verify_duration_seconds_count %d", count-1),
+		)
 		if notVerifiedYet {
 			time.Sleep(time.Second)
 			continue
 		}
 
-		return
+		return true
 	}
 
-	t.Fatal("State verification not run...")
+	return false
 }

services/horizon/internal/test/integration.go

@@ -208,6 +208,20 @@ func (i *IntegrationTest) LedgerIngested(sequence uint32) bool {
 	return root.IngestSequence >= sequence
 }
 
+// LedgerClosed returns true if the ledger with a given sequence has been
+// closed by Stellar-Core. Panics in case of errors. Note it's different
+// than LedgerIngested because it checks if the ledger was closed, not
+// necessarily ingested (ex. when rebuilding state Horizon does not ingest
+// recent ledgers).
+func (i *IntegrationTest) LedgerClosed(sequence uint32) bool {
+	root, err := i.Client().Root()
+	if err != nil {
+		panic(err)
+	}
+
+	return root.CoreSequence >= int32(sequence)
+}
+
 // AdminPort returns Horizon admin port.
 func (i *IntegrationTest) AdminPort() int {
 	return 6060
@@ -493,6 +507,20 @@ func (i *IntegrationTest) LogFailedTx(txResponse proto.Transaction, horizonResul
 		"Transaction doesn't have success code.")
 }
 
+func (i *IntegrationTest) RunHorizonCLICommand(cmd []string) {
+	fullCmd := append([]string{"/stellar/horizon/bin/horizon"}, cmd...)
+	id, err := i.cli.ContainerExecCreate(
+		context.Background(),
+		i.container.ID,
+		types.ExecConfig{
+			Cmd: fullCmd,
+		},
+	)
+	panicIf(err)
+	err = i.cli.ContainerExecStart(context.Background(), id.ID, types.ExecStartCheck{})
+	panicIf(err)
+}
+
 // Cluttering code with if err != nil is absolute nonsense.
 func panicIf(err error) {
 	if err != nil {