fix bugs in e2e testing and add kube feature gates

tests/e2e/e2e.mk

@@ -18,7 +18,6 @@ GOARCH ?= $(shell go env GOARCH)
 .PHONY: test-e2e-install
 test-e2e-install:
 	cd $(KOPS_ROOT)/tests/e2e && \
-		go install sigs.k8s.io/kubetest2 && \
 		go install ./kubetest2-tester-kops && \
 		go install ./kubetest2-kops
 

tests/e2e/go.mod

@@ -17,6 +17,7 @@ require (
 	k8s.io/client-go v11.0.1-0.20190805182717-6502b5e7b1b5+incompatible
 	k8s.io/klog/v2 v2.100.1
 	k8s.io/kops v1.24.1
+	k8s.io/utils v0.0.0-20230726121419-3b25d923346b
 	sigs.k8s.io/boskos v0.0.0-20220704141725-37bd9bb41b86
 	sigs.k8s.io/kubetest2 v0.0.0-20220801170629-1284e5ada592
 	sigs.k8s.io/yaml v1.3.0
@@ -139,7 +140,6 @@ require (
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	k8s.io/release v0.7.1-0.20210204090829-09fb5e3883b8 // indirect
 	k8s.io/test-infra v0.0.0-20210730160938-8ad9b8c53bd8 // indirect
-	k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect
 )

tests/e2e/kubetest2-kops/deployer/deployer.go

@@ -48,13 +48,14 @@ type deployer struct {
 	KopsBaseURL          string `flag:"-"`
 	PublishVersionMarker string `flag:"publish-version-marker" desc:"The GCS path to which the --kops-version-marker is uploaded if the tests pass"`
 
-	ClusterName    string   `flag:"cluster-name" desc:"The FQDN to use for the cluster name"`
-	CloudProvider  string   `flag:"cloud-provider" desc:"Which cloud provider to use"`
-	GCPProject     string   `flag:"gcp-project" desc:"Which GCP Project to use when --cloud-provider=gce"`
-	Env            []string `flag:"env" desc:"Additional env vars to set for kops commands in NAME=VALUE format"`
-	CreateArgs     string   `flag:"create-args" desc:"Extra space-separated arguments passed to 'kops create cluster'"`
-	KopsBinaryPath string   `flag:"kops-binary-path" desc:"The path to kops executable used for testing"`
-	createBucket   bool     `flag:"-"`
+	ClusterName            string   `flag:"cluster-name" desc:"The FQDN to use for the cluster name"`
+	CloudProvider          string   `flag:"cloud-provider" desc:"Which cloud provider to use"`
+	GCPProject             string   `flag:"gcp-project" desc:"Which GCP Project to use when --cloud-provider=gce"`
+	Env                    []string `flag:"env" desc:"Additional env vars to set for kops commands in NAME=VALUE format"`
+	CreateArgs             string   `flag:"create-args" desc:"Extra space-separated arguments passed to 'kops create cluster'"`
+	KopsBinaryPath         string   `flag:"kops-binary-path" desc:"The path to kops executable used for testing"`
+	KubernetesFeatureGates string   `flag:"kubernetes-feature-gates" desc:"Feature Gates to enable on Kubernetes components"`
+	createBucket           bool     `flag:"-"`
 
 	// ControlPlaneCount specifies the number of VMs in the control-plane.
 	ControlPlaneCount int `flag:"control-plane-count" desc:"Number of control-plane instances"`

tests/e2e/kubetest2-kops/deployer/dumplogs.go

@@ -281,8 +281,3 @@ func findControlPlaneIPUser(dump resources.Dump) (string, string, bool) {
 	klog.Warning("ControlPlane instance not found from kops toolbox dump")
 	return "", "", false
 }
-
-func runWithOutput(cmd exec.Cmd) error {
-	exec.InheritOutput(cmd)
-	return cmd.Run()
-}

tests/e2e/kubetest2-kops/deployer/up.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/kops/tests/e2e/pkg/kops"
 	"k8s.io/kops/tests/e2e/pkg/util"
 	"k8s.io/kops/tests/e2e/pkg/version"
+	"k8s.io/utils/strings/slices"
 	"sigs.k8s.io/kubetest2/pkg/exec"
 )
 
@@ -152,21 +153,31 @@ func (d *deployer) createCluster(zones []string, adminAccess string, yes bool) e
 	args = appendIfUnset(args, "--master-volume-size", "48")
 	args = appendIfUnset(args, "--node-count", "4")
 	args = appendIfUnset(args, "--node-volume-size", "48")
-	args = appendIfUnset(args, "--set", adminAccess)
 	args = appendIfUnset(args, "--zones", strings.Join(zones, ","))
 
 	switch d.CloudProvider {
 	case "aws":
 		if isArm {
 			args = appendIfUnset(args, "--master-size", "c6g.large")
+			args = appendIfUnset(args, "--node-size", "c6g.large")
 		} else {
 			args = appendIfUnset(args, "--master-size", "c5.large")
 		}
 	case "gce":
-		args = appendIfUnset(args, "--master-size", "e2-standard-2")
+		if isArm {
+			args = appendIfUnset(args, "--master-size", "t2a-standard-2")
+			args = appendIfUnset(args, "--node-size", "t2a-standard-2")
+		} else {
+			args = appendIfUnset(args, "--master-size", "e2-standard-2")
+			args = appendIfUnset(args, "--node-size", "e2-standard-2")
+		}
 		if d.GCPProject != "" {
 			args = appendIfUnset(args, "--project", d.GCPProject)
 		}
+		// set some sane default e2e testing behaviour on gce
+		args = appendIfUnset(args, "--gce-service-account", "default")
+		args = appendIfUnset(args, "--networking", "kubenet")
+
 		// We used to set the --vpc flag to split clusters into different networks, this is now the default.
 		// args = appendIfUnset(args, "--vpc", strings.Split(d.ClusterName, ".")[0])
 	case "digitalocean":
@@ -178,7 +189,10 @@ func (d *deployer) createCluster(zones []string, adminAccess string, yes bool) e
 		args = append(args, "--target", "terraform", "--out", d.terraform.Dir())
 	}
 
-	klog.Info(strings.Join(args, " "))
+	if d.KubernetesFeatureGates != "" {
+		args = appendIfUnset(args, "--kubernetes-feature-gates", d.KubernetesFeatureGates)
+	}
+
 	cmd := exec.Command(args[0], args[1:]...)
 	cmd.SetEnv(d.env()...)
 
@@ -308,11 +322,19 @@ func (d *deployer) zones() ([]string, error) {
 }
 
 // appendIfUnset will append an argument and its value to args if the arg is not already present
-// This shouldn't be used for arguments that can be specified multiple times like --set
+// This shouldn't be used for arguments that can be specified multiple times except --set
 func appendIfUnset(args []string, arg, value string) []string {
+	setFlags := []string{}
 	for _, existingArg := range args {
-		existingKey := strings.Split(existingArg, "=")
-		if existingKey[0] == arg {
+		existingKey := strings.SplitN(existingArg, "=", 2)
+		if existingKey[0] == "--set" {
+			if len(existingKey) == 3 {
+				setFlags = append(setFlags, existingKey[1])
+			}
+			if slices.Contains(setFlags, arg) {
+				return args
+			}
+		} else if existingKey[0] == arg {
 			return args
 		}
 	}

tests/e2e/kubetest2-kops/gce/gcs.go

@@ -17,8 +17,8 @@ limitations under the License.
 package gce
 
 import (
+	"crypto/rand"
 	"encoding/hex"
-	"math/rand"
 	"os"
 	"strings"
 

tests/e2e/pkg/util/publicip.go

@@ -25,37 +25,28 @@ import (
 	"time"
 )
 
-const externalIPMetadataURL = "http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/access-configs/0/external-ip"
-
 var externalIPServiceURLs = []string{
 	"https://ip.jsb.workers.dev",
-	"https://v4.ifconfig.co",
+	"https://ifconfig.co",
 }
 
 // ExternalIPRange returns the CIDR block for the public IP
 // in front of the kubetest2 client
 func ExternalIPRange() (string, error) {
 	var b bytes.Buffer
 
-	err := HTTPGETWithHeaders(externalIPMetadataURL, map[string]string{"Metadata-Flavor": "Google"}, &b)
-	if err != nil {
-		// This often fails due to workload identity
-		log.Printf("failed to get external ip from metadata service: %v", err)
-	} else if ip := net.ParseIP(strings.TrimSpace(b.String())); ip != nil {
-		return ip.String() + "/32", nil
-	} else {
-		log.Printf("metadata service returned invalid ip %q", b.String())
-	}
-
 	for attempt := 0; attempt < 5; attempt++ {
 		for _, u := range externalIPServiceURLs {
 			b.Reset()
-			err = HTTPGETWithHeaders(u, nil, &b)
+			err := HTTPGETWithHeaders(u, nil, &b)
 			if err != nil {
 				// The external service may well be down
 				log.Printf("failed to get external ip from %s: %v", u, err)
 			} else if ip := net.ParseIP(strings.TrimSpace(b.String())); ip != nil {
-				return ip.String() + "/32", nil
+				if ip.To4() != nil {
+					return ip.String() + "/32", nil
+				}
+				return ip.String() + "/128", nil
 			} else {
 				log.Printf("service %s returned invalid ip %q", u, b.String())
 			}