Test suite partitioning

.changes/v3.11.0/1110-notes.md

@@ -0,0 +1 @@
+* Add ability to split the test suite across several VCDs [GH-1110]

scripts/gosec.sh

@@ -65,6 +65,7 @@ function get_gosec {
 function run_gosec {
     if [ -n "$gosec" ]
     then
+        go version
         $gosec -tests -tags ALL ./...
         exit_code=$?
         if [ "$exit_code" != "0" ]

scripts/start-multi-node-test.sh

@@ -0,0 +1,119 @@
+#!/usr/bin/env bash
+
+# Quality set for bash scripts:
+# [set -e] fails on error
+set -e
+
+# [set -u ] stops the script on unset variables
+set -u
+
+# [set -o pipefail] triggers a failure if one of the commands in the pipe fail
+set -o pipefail
+
+node=$1
+shift
+partitions=$#
+relative_configs=$@
+
+if [[  $partitions < 2 ]]
+then
+    echo "Syntax: ./scripts/start-multi-node-test.sh node-number config1 config2 [config3 [config4 ...]]"
+    exit 1
+fi
+
+if [ ! -d scripts ]
+then
+    echo "This script must be launched at the top of the repository, as ./scripts/start-multi-node-test.sh"
+    exit 1
+fi
+
+multi_dir="split$partitions"
+
+if [ -d "$multi_dir" ]
+then
+    echo "directory '$multi_dir' already exists"
+    exit 1
+fi
+
+function exists_in_path {
+    what=$1
+    for dir in $(echo $PATH | tr ':' ' ')
+    do
+        wanted=$dir/$what
+        if [ -x $wanted ]
+        then
+            echo $wanted
+            return
+        fi
+    done
+}
+
+for needed in tmux git realpath jq
+do
+    found_need=$(exists_in_path $needed)
+    if [ -z "$found_need" ]
+    then
+        echo "needed tool $needed not found"
+        exit 1
+    fi
+done
+
+count=0
+for tb in $relative_configs
+do
+    if [ ! -f $tb ]
+    then
+        echo "$tb not found"
+        exit 1
+    fi
+    configs[$count]=$(realpath $tb)
+    count=$((count+1))
+done
+
+current_repo_url=$(git config --get remote.origin.url)
+if [ -z "$current_repo_url" ]
+then
+    echo "error retrieving current repository URL"
+    exit 1
+fi
+
+current_commit=$(git rev-parse HEAD)
+if [ -z "$current_commit" ]
+then
+    echo "error retrieving current commit hash"
+    exit 1
+fi
+
+mkdir "${multi_dir}" || exit 1
+cd ${multi_dir} || exit 1
+
+echo "configs ${configs[*]}"
+echo "partitions ${partitions}"
+
+for n in $(seq 1 $partitions)
+do
+    git clone ${current_repo_url}
+    if [ "$?" != "0" ]
+    then
+        echo "error cloning current repository"
+        exit 1
+    fi
+    if [ ! -d terraform-provider-vcd ]
+    then
+        echo "terraform-provider-vcd not cloned in node $n"
+        exit 1
+    fi
+    mv terraform-provider-vcd "node$n"
+    cd "node$n/vcd"
+    pwd
+    index=$(($n-1))
+    cp ${configs[$index]} vcd_test_config.json
+
+    # TODO: this change is only needed while developing the current PR in branch
+    git checkout test-partitioning
+    # TODO: start tmux session with test
+    # go test -tags functional -v -vcd-partitions=$partitions -vcd-partition-node=$n -vcd-short 2>&1 > testacc-node$n.txt
+    cd -
+
+done
+

vcd/config_test.go

@@ -16,6 +16,7 @@ import (
 	"path/filepath"
 	"regexp"
 	"runtime"
+	"strconv"
 	"strings"
 	"testing"
 	"text/template"
@@ -55,7 +56,9 @@ func init() {
 	setStringFlag(&vcdSkipPattern, "vcd-skip-pattern", "VCD_SKIP_PATTERN", "Skip tests that match the pattern (implies vcd-pre-post-checks")
 	setBoolFlag(&skipLeftoversRemoval, "vcd-skip-leftovers-removal", "VCD_SKIP_LEFTOVERS_REMOVAL", "Do not attempt removal of leftovers at the end of the test suite")
 	setBoolFlag(&silentLeftoversRemoval, "vcd-silent-leftovers-removal", "VCD_SILENT_LEFTOVERS_REMOVAL", "Omit details during removal of leftovers")
-
+	setStringFlag(&testListFileName, "vcd-partition-tests-file", "VCD_PARTITION_TESTS_FILE", "Name of the file containing the tests to run in the current partition node")
+	setIntFlag(&numberOfPartitions, "vcd-partitions", "VCD_PARTITIONS", "")
+	setIntFlag(&partitionNode, "vcd-partition-node", "VCD_PARTITION_NODE", "")
 }
 
 // Structure to get info from a config json file that the user specifies
@@ -389,7 +392,17 @@ func GetVarsFromTemplate(tmpl string) []string {
 // configuration.
 // Returns the text of a ready-to-use Terraform directive. It also saves the filled
 // template to a file, for further troubleshooting.
-func templateFill(tmpl string, data StringMap) string {
+func templateFill(tmpl string, inputData StringMap) string {
+
+	// Copying the input data, to prevent side effects in the original string map:
+	// When we use the option -vcd-add-provider, the data will also contain the fields
+	// needed to populate the provider. Some of those fields are empty (e.g. "Token")
+	// If the data is evaluated (testParamsNotEmpty) after filling the template, the
+	// test gets skipped for what happen to be mysterious reasons.
+	data := make(StringMap)
+	for k, v := range inputData {
+		data[k] = v
+	}
 
 	// Gets the name of the function containing the template
 	caller := callFuncName()
@@ -887,6 +900,13 @@ func TestMain(m *testing.M) {
 	// Runs all test functions
 	exitCode := m.Run()
 
+	if numberOfPartitions != 0 {
+		entTestFileName := getTestFileName("end", testConfig.Provider.VcdVersion)
+		err := os.WriteFile(entTestFileName, []byte(fmt.Sprintf("%d", exitCode)), 0600)
+		if err != nil {
+			fmt.Printf("error writing to file '%s': %s\n", entTestFileName, err)
+		}
+	}
 	if vcdShowCount {
 		fmt.Printf("Pass: %5d - Skip: %5d - Fail: %5d\n", vcdPassCount, vcdSkipCount, vcdFailCount)
 	}
@@ -1127,6 +1147,18 @@ func setStringFlag(varPointer *string, name, envVar, help string) {
 	flag.StringVar(varPointer, name, *varPointer, help)
 }
 
+func setIntFlag(varPointer *int, name, envVar, help string) {
+	if envVar != "" && os.Getenv(envVar) != "" {
+		var err error
+		value := os.Getenv(envVar)
+		*varPointer, err = strconv.Atoi(value)
+		if err != nil {
+			panic(fmt.Sprintf("error converting value '%s' to integer: %s", value, err))
+		}
+	}
+	flag.IntVar(varPointer, name, *varPointer, help)
+}
+
 type envHelper struct {
 	vars map[string]string
 }
@@ -1392,6 +1424,7 @@ func timeStamp() string {
 //     contains a pattern that matches the test name.
 //  6. If the flag -vcd-re-run-failed is true, it will only run the tests that failed in the previous run
 func preTestChecks(t *testing.T) {
+	handlePartitioning(testConfig.Provider.VcdVersion, testConfig.Provider.Url, t)
 	// if the test runs without -vcd-pre-post-checks, all post-checks will be skipped
 	if !vcdPrePostChecks {
 		return

vcd/datasource_vcd_nsxt_edgegateway_qos_profile_test.go

@@ -9,6 +9,7 @@ import (
 )
 
 func TestAccVcdDatasourceNsxtGatewayQosProfile(t *testing.T) {
+	preTestChecks(t)
 	if vcdShortTest {
 		t.Skip(acceptanceTestsSkipped)
 		return
@@ -46,6 +47,7 @@ func TestAccVcdDatasourceNsxtGatewayQosProfile(t *testing.T) {
 			},
 		},
 	})
+	postTestChecks(t)
 }
 
 const testAccVcdDatasourceNsxtGatewayQosProfile = `