Ginkgo Test to exercise cross AZ testing, to be run on static canarie…

…s. (#2689)

* Test Pod Connectivity Across AZs, to exercise CNI and pod connectivity
  upon AZ failures.

* Test API Server Connectivity from various AZs to ensure that Service
  Discovery (kube-proxy) and Service Name look up (Coredns) remain
  functional during AZ failures.

scripts/run-static-canary.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# The script runs amazon-vpc-cni static canary tests
+# The tests in this suite are designed to exercise AZ failure scenarios.
+
+set -e
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+GINKGO_TEST_BUILD="$SCRIPT_DIR/../test/build"
+# TEST_IMAGE_REGISTRY is the registry in test-infra-* accounts where e2e test images are stored
+TEST_IMAGE_REGISTRY=${TEST_IMAGE_REGISTRY:-"617930562442.dkr.ecr.us-west-2.amazonaws.com"}
+
+source "$SCRIPT_DIR"/lib/cluster.sh
+source "$SCRIPT_DIR"/lib/canary.sh
+
+function run_ginkgo_test() {
+  local focus=$1
+  echo "Running ginkgo tests with focus: $focus"
+
+  (CGO_ENABLED=0 ginkgo $EXTRA_GINKGO_FLAGS --no-color --focus="$focus" -v --timeout 30m --fail-on-pending $GINKGO_TEST_BUILD/cni.test -- \
+      --cluster-kubeconfig="$KUBE_CONFIG_PATH" \
+      --cluster-name="$CLUSTER_NAME" \
+      --aws-region="$REGION" \
+      --aws-vpc-id="$VPC_ID" \
+      --ng-name-label-key="kubernetes.io/os" \
+      --ng-name-label-val="linux" \
+      --test-image-registry=$TEST_IMAGE_REGISTRY)
+}
+
+load_cluster_details
+
+run_ginkgo_test "STATIC_CANARY"
+
+echo "all tests ran successfully in $(($SECONDS / 60)) minutes and $(($SECONDS % 60)) seconds"

test/framework/resources/k8s/manifest/container.go

@@ -28,6 +28,7 @@ type Container struct {
 	probe           *v1.Probe
 	ports           []v1.ContainerPort
 	securityContext *v1.SecurityContext
+	Env             []v1.EnvVar
 }
 
 func NewBusyBoxContainerBuilder(testImageRegistry string) *Container {
@@ -101,6 +102,11 @@ func (w *Container) Command(cmd []string) *Container {
 	return w
 }
 
+func (w *Container) EnvVar(env []v1.EnvVar) *Container {
+	w.Env = env
+	return w
+}
+
 func (w *Container) Args(arg []string) *Container {
 	w.args = arg
 	return w
@@ -126,5 +132,6 @@ func (w *Container) Build() v1.Container {
 		LivenessProbe:   w.probe,
 		Ports:           w.ports,
 		SecurityContext: w.securityContext,
+		Env:             w.Env,
 	}
 }

test/framework/resources/k8s/manifest/daemonset.go

@@ -0,0 +1,123 @@
+// Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License"). You may
+// not use this file except in compliance with the License. A copy of the
+// License is located at
+//
+//     http://aws.amazon.com/apache2.0/
+//
+// or in the "license" file accompanying this file. This file is distributed
+// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+// express or implied. See the License for the specific language governing
+// permissions and limitations under the License.
+
+package manifest
+
+import (
+	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
+	"github.com/aws/aws-sdk-go/aws"
+	v1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+type DaemonsetBuilder struct {
+	namespace              string
+	name                   string
+	container              corev1.Container
+	labels                 map[string]string
+	nodeSelector           map[string]string
+	terminationGracePeriod int
+	hostNetwork            bool
+	volume                 []corev1.Volume
+	volumeMount            []corev1.VolumeMount
+}
+
+func NewDefaultDaemonsetBuilder() *DaemonsetBuilder {
+	return &DaemonsetBuilder{
+		namespace:              utils.DefaultTestNamespace,
+		terminationGracePeriod: 1,
+		labels:                 map[string]string{"role": "test"},
+		nodeSelector:           map[string]string{"kubernetes.io/os": "linux"},
+	}
+}
+
+func (d *DaemonsetBuilder) Labels(labels map[string]string) *DaemonsetBuilder {
+	d.labels = labels
+	return d
+}
+
+func (d *DaemonsetBuilder) NodeSelector(labelKey string, labelVal string) *DaemonsetBuilder {
+	if labelKey != "" {
+		d.nodeSelector[labelKey] = labelVal
+	}
+	return d
+}
+
+func (d *DaemonsetBuilder) Namespace(namespace string) *DaemonsetBuilder {
+	d.namespace = namespace
+	return d
+}
+
+func (d *DaemonsetBuilder) TerminationGracePeriod(tg int) *DaemonsetBuilder {
+	d.terminationGracePeriod = tg
+	return d
+}
+
+func (d *DaemonsetBuilder) Name(name string) *DaemonsetBuilder {
+	d.name = name
+	return d
+}
+
+func (d *DaemonsetBuilder) Container(container corev1.Container) *DaemonsetBuilder {
+	d.container = container
+	return d
+}
+
+func (d *DaemonsetBuilder) PodLabel(labelKey string, labelValue string) *DaemonsetBuilder {
+	d.labels[labelKey] = labelValue
+	return d
+}
+
+func (d *DaemonsetBuilder) HostNetwork(hostNetwork bool) *DaemonsetBuilder {
+	d.hostNetwork = hostNetwork
+	return d
+}
+
+func (d *DaemonsetBuilder) MountVolume(volume []corev1.Volume, volumeMount []corev1.VolumeMount) *DaemonsetBuilder {
+	d.volume = volume
+	d.volumeMount = volumeMount
+	return d
+}
+
+func (d *DaemonsetBuilder) Build() *v1.DaemonSet {
+	deploymentSpec := &v1.DaemonSet{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      d.name,
+			Namespace: d.namespace,
+			Labels:    d.labels,
+		},
+		Spec: v1.DaemonSetSpec{
+			Selector: &metav1.LabelSelector{
+				MatchLabels: d.labels,
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: d.labels,
+				},
+				Spec: corev1.PodSpec{
+					HostNetwork:                   d.hostNetwork,
+					NodeSelector:                  d.nodeSelector,
+					Containers:                    []corev1.Container{d.container},
+					TerminationGracePeriodSeconds: aws.Int64(int64(d.terminationGracePeriod)),
+				},
+			},
+		},
+	}
+
+	if len(d.volume) > 0 && len(d.volumeMount) > 0 {
+		deploymentSpec.Spec.Template.Spec.Volumes = d.volume
+		deploymentSpec.Spec.Template.Spec.Containers[0].VolumeMounts = d.volumeMount
+	}
+	return deploymentSpec
+}

test/framework/resources/k8s/resources/daemonset.go

@@ -16,18 +16,24 @@ package resources
 import (
 	"context"
 	"errors"
+	"time"
 
 	"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"
 	v1 "k8s.io/api/apps/v1"
+	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type DaemonSetManager interface {
 	GetDaemonSet(namespace string, name string) (*v1.DaemonSet, error)
+
+	CreateAndWaitTillDaemonSetIsReady(daemonSet *v1.DaemonSet, timeout time.Duration) (*v1.DaemonSet, error)
+
 	UpdateAndWaitTillDaemonSetReady(old *v1.DaemonSet, new *v1.DaemonSet) (*v1.DaemonSet, error)
 	CheckIfDaemonSetIsReady(namespace string, name string) error
+	DeleteAndWaitTillDaemonSetIsDeleted(daemonSet *v1.DaemonSet, timeout time.Duration) error
 }
 
 type defaultDaemonSetManager struct {
@@ -38,6 +44,24 @@ func NewDefaultDaemonSetManager(k8sClient client.Client) DaemonSetManager {
 	return &defaultDaemonSetManager{k8sClient: k8sClient}
 }
 
+func (d *defaultDaemonSetManager) CreateAndWaitTillDaemonSetIsReady(daemonSet *v1.DaemonSet, timeout time.Duration) (*v1.DaemonSet, error) {
+	ctx := context.Background()
+	err := d.k8sClient.Create(ctx, daemonSet)
+	if err != nil {
+		return nil, err
+	}
+
+	// Allow for the cache to sync
+	time.Sleep(utils.PollIntervalShort)
+
+	err = d.CheckIfDaemonSetIsReady(daemonSet.Namespace, daemonSet.Name)
+	if err != nil {
+		return nil, err
+	}
+
+	return daemonSet, nil
+}
+
 func (d *defaultDaemonSetManager) GetDaemonSet(namespace string, name string) (*v1.DaemonSet, error) {
 	ctx := context.Background()
 	daemonSet := &v1.DaemonSet{}
@@ -94,3 +118,28 @@ func (d *defaultDaemonSetManager) CheckIfDaemonSetIsReady(namespace string, name
 	}, ctx.Done())
 
 }
+
+func (d *defaultDaemonSetManager) DeleteAndWaitTillDaemonSetIsDeleted(daemonSet *v1.DaemonSet, timeout time.Duration) error {
+	ctx := context.Background()
+
+	err := d.k8sClient.Delete(ctx, daemonSet)
+
+	if k8sErrors.IsNotFound(err) {
+		return nil
+	}
+
+	if err != nil {
+		return err
+	}
+	observed := &v1.DaemonSet{}
+
+	return wait.PollImmediateUntil(utils.PollIntervalShort, func() (bool, error) {
+		if err := d.k8sClient.Get(ctx, utils.NamespacedName(daemonSet), observed); err != nil {
+			if k8sErrors.IsNotFound(err) {
+				return true, nil
+			}
+			return false, err
+		}
+		return false, nil
+	}, ctx.Done())
+}