integration tests - split steps; run in parallel

.gitignore

@@ -6,3 +6,4 @@ docs/tla/states/
 *.out
 vendor/
 .vscode
+logs

tests/integration/actions.go

@@ -361,10 +361,9 @@ func (tr TestRun) voteGovProposal(
 }
 
 type startConsumerChainAction struct {
-	consumerChain  chainID
-	providerChain  chainID
-	genesisChanges string
-	validators     []StartChainValidator
+	consumerChain chainID
+	providerChain chainID
+	validators    []StartChainValidator
 }
 
 func (tr TestRun) startConsumerChain(
@@ -392,8 +391,9 @@ func (tr TestRun) startConsumerChain(
 	}
 
 	genesisChanges := ".app_state.ccvconsumer = " + string(bz)
-	if action.genesisChanges != "" {
-		genesisChanges = genesisChanges + " | " + action.genesisChanges
+	consumerGenesisChanges := tr.chainConfigs[action.consumerChain].genesisChanges
+	if consumerGenesisChanges != "" {
+		genesisChanges = genesisChanges + " | " + consumerGenesisChanges
 	}
 
 	tr.startChain(StartChainAction{

tests/integration/config.go

@@ -1,8 +1,8 @@
 package main
 
 import (
-	"flag"
 	"fmt"
+	"log"
 	"strconv"
 	"time"
 )
@@ -52,45 +52,53 @@ type TestRun struct {
 	validatorConfigs map[validatorID]ValidatorConfig
 	chainConfigs     map[chainID]ChainConfig
 	localSdkPath     string
+
+	name   string
+	logger *log.Logger
+}
+
+func getDefaultValidators() map[validatorID]ValidatorConfig {
+	return map[validatorID]ValidatorConfig{
+		validatorID("alice"): {
+			mnemonic:         "pave immune ethics wrap gain ceiling always holiday employ earth tumble real ice engage false unable carbon equal fresh sick tattoo nature pupil nuclear",
+			delAddress:       "cosmos19pe9pg5dv9k5fzgzmsrgnw9rl9asf7ddwhu7lm",
+			valoperAddress:   "cosmosvaloper19pe9pg5dv9k5fzgzmsrgnw9rl9asf7ddtrgtng",
+			valconsAddress:   "cosmosvalcons1qmq08eruchr5sf5s3rwz7djpr5a25f7xw4mceq",
+			privValidatorKey: `{"address":"06C0F3E47CC5C748269088DC2F36411D3AAA27C6","pub_key":{"type":"tendermint/PubKeyEd25519","value":"RrclQz9bIhkIy/gfL485g3PYMeiIku4qeo495787X10="},"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"uX+ZpDMg89a6gtqs/+MQpCTSqlkZ0nJQJOhLlCJvwvdGtyVDP1siGQjL+B8vjzmDc9gx6IiS7ip6jj3nvztfXQ=="}}`,
+			nodeKey:          `{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"fjw4/DAhyRPnwKgXns5SV7QfswRSXMWJpHS7TyULDmJ8ofUc5poQP8dgr8bZRbCV5RV8cPqDq3FPdqwpmUbmdA=="}}`,
+			ipSuffix:         "4",
+		},
+		validatorID("bob"): {
+			mnemonic:         "glass trip produce surprise diamond spin excess gaze wash drum human solve dress minor artefact canoe hard ivory orange dinner hybrid moral potato jewel",
+			delAddress:       "cosmos1dkas8mu4kyhl5jrh4nzvm65qz588hy9qcz08la",
+			valoperAddress:   "cosmosvaloper1dkas8mu4kyhl5jrh4nzvm65qz588hy9qakmjnw",
+			valconsAddress:   "cosmosvalcons1nx7n5uh0ztxsynn4sje6eyq2ud6rc6klc96w39",
+			privValidatorKey: `{"address":"99BD3A72EF12CD024E7584B3AC900AE3743C6ADF","pub_key":{"type":"tendermint/PubKeyEd25519","value":"mAN6RXYxSM4MNGSIriYiS7pHuwAcOHDQAy9/wnlSzOI="},"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"QePcwfWtOavNK7pBJrtoLMzarHKn6iBWfWPFeyV+IdmYA3pFdjFIzgw0ZIiuJiJLuke7ABw4cNADL3/CeVLM4g=="}}`,
+			nodeKey:          `{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"TQ4vHcO/vKdzGtWpelkX53WdMQd4kTsWGFrdcatdXFvWyO215Rewn5IRP0FszPLWr2DqPzmuH8WvxYGk5aeOXw=="}}`,
+			ipSuffix:         "5",
+		},
+		validatorID("carol"): {
+			mnemonic:         "sight similar better jar bitter laptop solve fashion father jelly scissors chest uniform play unhappy convince silly clump another conduct behave reunion marble animal",
+			delAddress:       "cosmos19hz4m226ztankqramvt4a7t0shejv4dc79gp9u",
+			valoperAddress:   "cosmosvaloper19hz4m226ztankqramvt4a7t0shejv4dcm3u5f0",
+			valconsAddress:   "cosmosvalcons1ezyrq65s3gshhx5585w6mpusq3xsj3ayzf4uv6",
+			privValidatorKey: `{"address":"C888306A908A217B9A943D1DAD8790044D0947A4","pub_key":{"type":"tendermint/PubKeyEd25519","value":"IHo4QEikWZfIKmM0X+N+BjKttz8HOzGs2npyjiba3Xk="},"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"z08bmSB91uFVpVmR3t2ewd/bDjZ/AzwQpe5rKjWiPG0gejhASKRZl8gqYzRf434GMq23Pwc7MazaenKOJtrdeQ=="}}`,
+			nodeKey:          `{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"WLTcHEjbwB24Wp3z5oBSYTvtGQonz/7IQabOFw85BN0UkkyY5HDf38o8oHlFxVI26f+DFVeICuLbe9aXKGnUeg=="}}`,
+			ipSuffix:         "6",
+		},
+	}
 }
 
 func DefaultTestRun() TestRun {
 	return TestRun{
+		name: "default",
 		containerConfig: ContainerConfig{
 			containerName: "interchain-security-container",
 			instanceName:  "interchain-security-instance",
 			ccvVersion:    "1",
 			now:           time.Now(),
 		},
-		validatorConfigs: map[validatorID]ValidatorConfig{
-			validatorID("alice"): {
-				mnemonic:         "pave immune ethics wrap gain ceiling always holiday employ earth tumble real ice engage false unable carbon equal fresh sick tattoo nature pupil nuclear",
-				delAddress:       "cosmos19pe9pg5dv9k5fzgzmsrgnw9rl9asf7ddwhu7lm",
-				valoperAddress:   "cosmosvaloper19pe9pg5dv9k5fzgzmsrgnw9rl9asf7ddtrgtng",
-				valconsAddress:   "cosmosvalcons1qmq08eruchr5sf5s3rwz7djpr5a25f7xw4mceq",
-				privValidatorKey: `{"address":"06C0F3E47CC5C748269088DC2F36411D3AAA27C6","pub_key":{"type":"tendermint/PubKeyEd25519","value":"RrclQz9bIhkIy/gfL485g3PYMeiIku4qeo495787X10="},"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"uX+ZpDMg89a6gtqs/+MQpCTSqlkZ0nJQJOhLlCJvwvdGtyVDP1siGQjL+B8vjzmDc9gx6IiS7ip6jj3nvztfXQ=="}}`,
-				nodeKey:          `{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"fjw4/DAhyRPnwKgXns5SV7QfswRSXMWJpHS7TyULDmJ8ofUc5poQP8dgr8bZRbCV5RV8cPqDq3FPdqwpmUbmdA=="}}`,
-				ipSuffix:         "4",
-			},
-			validatorID("bob"): {
-				mnemonic:         "glass trip produce surprise diamond spin excess gaze wash drum human solve dress minor artefact canoe hard ivory orange dinner hybrid moral potato jewel",
-				delAddress:       "cosmos1dkas8mu4kyhl5jrh4nzvm65qz588hy9qcz08la",
-				valoperAddress:   "cosmosvaloper1dkas8mu4kyhl5jrh4nzvm65qz588hy9qakmjnw",
-				valconsAddress:   "cosmosvalcons1nx7n5uh0ztxsynn4sje6eyq2ud6rc6klc96w39",
-				privValidatorKey: `{"address":"99BD3A72EF12CD024E7584B3AC900AE3743C6ADF","pub_key":{"type":"tendermint/PubKeyEd25519","value":"mAN6RXYxSM4MNGSIriYiS7pHuwAcOHDQAy9/wnlSzOI="},"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"QePcwfWtOavNK7pBJrtoLMzarHKn6iBWfWPFeyV+IdmYA3pFdjFIzgw0ZIiuJiJLuke7ABw4cNADL3/CeVLM4g=="}}`,
-				nodeKey:          `{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"TQ4vHcO/vKdzGtWpelkX53WdMQd4kTsWGFrdcatdXFvWyO215Rewn5IRP0FszPLWr2DqPzmuH8WvxYGk5aeOXw=="}}`,
-				ipSuffix:         "5",
-			},
-			validatorID("carol"): {
-				mnemonic:         "sight similar better jar bitter laptop solve fashion father jelly scissors chest uniform play unhappy convince silly clump another conduct behave reunion marble animal",
-				delAddress:       "cosmos19hz4m226ztankqramvt4a7t0shejv4dc79gp9u",
-				valoperAddress:   "cosmosvaloper19hz4m226ztankqramvt4a7t0shejv4dcm3u5f0",
-				valconsAddress:   "cosmosvalcons1ezyrq65s3gshhx5585w6mpusq3xsj3ayzf4uv6",
-				privValidatorKey: `{"address":"C888306A908A217B9A943D1DAD8790044D0947A4","pub_key":{"type":"tendermint/PubKeyEd25519","value":"IHo4QEikWZfIKmM0X+N+BjKttz8HOzGs2npyjiba3Xk="},"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"z08bmSB91uFVpVmR3t2ewd/bDjZ/AzwQpe5rKjWiPG0gejhASKRZl8gqYzRf434GMq23Pwc7MazaenKOJtrdeQ=="}}`,
-				nodeKey:          `{"priv_key":{"type":"tendermint/PrivKeyEd25519","value":"WLTcHEjbwB24Wp3z5oBSYTvtGQonz/7IQabOFw85BN0UkkyY5HDf38o8oHlFxVI26f+DFVeICuLbe9aXKGnUeg=="}}`,
-				ipSuffix:         "6",
-			},
-		},
+		validatorConfigs: getDefaultValidators(),
 		chainConfigs: map[chainID]ChainConfig{
 			chainID("provi"): {
 				chainId:        chainID("provi"),
@@ -116,12 +124,41 @@ func DefaultTestRun() TestRun {
 					".app_state.slashing.params.downtime_jail_duration = \"2s\" | " +
 					".app_state.slashing.params.slash_fraction_downtime = \"0.010000000000000000\"",
 			},
+		},
+	}
+}
+
+func DemocracyTestRun() TestRun {
+	return TestRun{
+		name: "democracy",
+		containerConfig: ContainerConfig{
+			containerName: "interchain-security-democ-container",
+			instanceName:  "interchain-security-democ-instance",
+			ccvVersion:    "1",
+			now:           time.Now(),
+		},
+		validatorConfigs: getDefaultValidators(),
+		chainConfigs: map[chainID]ChainConfig{
+			chainID("provi"): {
+				chainId:        chainID("provi"),
+				binaryName:     "interchain-security-pd",
+				ipPrefix:       "7.7.7",
+				votingWaitTime: 5,
+				genesisChanges: ".app_state.gov.voting_params.voting_period = \"5s\" | " +
+					// Custom slashing parameters for testing validator downtime functionality
+					// See https://docs.cosmos.network/main/modules/slashing/04_begin_block.html#uptime-tracking
+					".app_state.slashing.params.signed_blocks_window = \"2\" | " +
+					".app_state.slashing.params.min_signed_per_window = \"0.500000000000000000\" | " +
+					".app_state.slashing.params.downtime_jail_duration = \"2s\" | " +
+					".app_state.slashing.params.slash_fraction_downtime = \"0.010000000000000000\"",
+			},
 			chainID("democ"): {
 				chainId:        chainID("democ"),
 				binaryName:     "interchain-security-cdd",
 				ipPrefix:       "7.7.9",
 				votingWaitTime: 10,
-				genesisChanges: ".app_state.gov.voting_params.voting_period = \"10s\" | " +
+				genesisChanges: ".app_state.ccvconsumer.params.blocks_per_distribution_transmission = \"10\" | " +
+					".app_state.gov.voting_params.voting_period = \"10s\" | " +
 					".app_state.slashing.params.signed_blocks_window = \"2\" | " +
 					".app_state.slashing.params.min_signed_per_window = \"0.500000000000000000\" | " +
 					".app_state.slashing.params.downtime_jail_duration = \"2s\" | " +
@@ -131,12 +168,11 @@ func DefaultTestRun() TestRun {
 	}
 }
 
-func (s *TestRun) ParseCLIFlags() {
-	localSdkPath := flag.String("local-sdk-path", "",
-		"path of a local sdk version to build and reference in integration tests")
-	flag.Parse()
-	s.localSdkPath = *localSdkPath
-	fmt.Println(s.localSdkPath)
+func (s *TestRun) SetLocalSDKPath(path string) {
+	if path != "" {
+		fmt.Println("USING LOCAL SDK", path)
+	}
+	s.localSdkPath = path
 }
 
 // ValidateStringLiterals enforces that configs follow the constraints

tests/integration/main.go

@@ -2,44 +2,51 @@ package main
 
 import (
 	"bufio"
+	"encoding/json"
+	"errors"
+	"flag"
 	"fmt"
 	"log"
+	"os"
 	"os/exec"
 	"reflect"
+	"sync"
 	"time"
 
 	"github.com/kylelemons/godebug/pretty"
 )
 
 var verbose = false
+var localSdkPath = flag.String("local-sdk-path", "",
+	"path of a local sdk version to build and reference in integration tests")
 
 func main() {
-	fmt.Println("============================================ start happy path tests ============================================")
+	flag.Parse()
+
+	// wg waits for all runners to complete
+	var wg sync.WaitGroup
+
 	start := time.Now()
 	tr := DefaultTestRun()
-	tr.ParseCLIFlags()
+	tr.SetLocalSDKPath(*localSdkPath)
 	tr.ValidateStringLiterals()
-	tr.startDocker()
 
-	for _, step := range happyPathSteps {
-		tr.runStep(step, verbose)
-	}
+	dmc := DemocracyTestRun()
+	dmc.SetLocalSDKPath(*localSdkPath)
+	dmc.ValidateStringLiterals()
 
-	fmt.Printf("happy path tests successful - time elapsed %v\n", time.Since(start))
+	wg.Add(1)
+	go tr.ExecuteSteps(&wg, "default", happyPathSteps)
 
-	fmt.Println("============================================ start democracy tests ============================================")
-	start = time.Now()
-	tr.startDocker()
+	wg.Add(1)
+	go dmc.ExecuteSteps(&wg, "democracy", democracySteps)
 
-	for _, step := range democracySteps {
-		tr.runStep(step, verbose)
-	}
-
-	fmt.Printf("democracy tests successful - time elapsed %v\n", time.Since(start))
+	wg.Wait()
+	fmt.Printf("TOTAL TIME ELAPSED: %v\n", time.Since(start))
 }
 
 func (tr TestRun) runStep(step Step, verbose bool) {
-	fmt.Printf("%#v\n", step.action)
+	tr.logger.Printf("%#v\n", step.action)
 	switch action := step.action.(type) {
 	case StartChainAction:
 		tr.startChain(action, verbose)
@@ -80,20 +87,56 @@ func (tr TestRun) runStep(step Step, verbose bool) {
 	case registerRepresentativeAction:
 		tr.registerRepresentative(action, verbose)
 	default:
-		log.Fatalf(fmt.Sprintf(`unknown action: %#v`, action))
+		tr.logger.Fatalf(fmt.Sprintf(`unknown action: %#v`, action))
 	}
 
 	modelState := step.state
 	actualState := tr.getState(step.state)
 
 	// Check state
 	if !reflect.DeepEqual(actualState, modelState) {
-		pretty.Print("actual state", actualState)
-		pretty.Print("model state", modelState)
-		log.Fatal(`actual state (-) not equal to model state (+): ` + pretty.Compare(actualState, modelState))
+		pretty.Fprint(tr.logger.Writer(), "actual state", actualState)
+		pretty.Fprint(tr.logger.Writer(), "model state", modelState)
+		// pretty.Print("model state", modelState)
+		tr.logger.Fatalf(`actual state (-) not equal to model state (+): %s` + pretty.Compare(actualState, modelState))
+	}
+
+	// pretty.Print(actualState)
+	buf, _ := json.Marshal(actualState)
+	tr.logger.Println(string(buf))
+	// pretty.Fprint(tr.logger.Writer(), actualState)
+}
+
+// Starts docker container and sequentially runs steps
+func (tr TestRun) ExecuteSteps(wg *sync.WaitGroup, name string, steps []Step) {
+	defer wg.Done()
+
+	// save logs to file
+	if _, err := os.Stat("./logs"); errors.Is(err, os.ErrNotExist) {
+		err := os.Mkdir("./logs", os.ModePerm)
+		if !errors.Is(err, os.ErrExist) {
+			panic(fmt.Sprintf("could not create log dir: %s", err))
+		}
+	}
+	f, err := os.Create(fmt.Sprintf("./logs/%d_%s.log", time.Now().Unix(), tr.name))
+	if err != nil {
+		panic(fmt.Sprintf("could not create logfile: %s", err))
+	}
+	defer f.Close()
+
+	tr.logger = log.New(f, "", log.Ldate|log.Ltime)
+
+	fmt.Printf("============================================ start %s tests ============================================\n", name)
+
+	start := time.Now()
+	tr.startDocker()
+
+	for _, step := range steps {
+		tr.runStep(step, verbose)
 	}
 
-	pretty.Print(actualState)
+	fmt.Printf("\n\ntime elapsed for %s: %v\n", name, time.Since(start))
+	fmt.Printf("============================================ end %s tests ============================================\n", name)
 }
 
 func (tr TestRun) startDocker() {