From ee16652bc5f32f073ec638f34e7908dd256aabdf Mon Sep 17 00:00:00 2001 From: Jack Francis Date: Fri, 20 Sep 2019 11:42:01 -0700 Subject: [PATCH] test: address 4 E2E test flakes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. validate service URLs w/ retries 2. don’t enforce “shell out command” timeouts everywhere 3. standardize Linux stability tests timeout to 3 secs per test 4. standardize some Windows stability tests timeout to 1 minute per test --- test/e2e/kubernetes/deployment/deployment.go | 10 +- test/e2e/kubernetes/hpa/hpa.go | 4 +- test/e2e/kubernetes/job/job.go | 3 +- test/e2e/kubernetes/kubernetes_test.go | 36 ++++---- test/e2e/kubernetes/node/node.go | 4 +- .../persistentvolume/persistentvolume.go | 3 +- .../persistentvolumeclaims.go | 4 +- test/e2e/kubernetes/pod/pod.go | 4 +- test/e2e/kubernetes/service/service.go | 92 ++++++++++++------- .../kubernetes/storageclass/storageclass.go | 3 +- test/e2e/kubernetes/util/util.go | 7 +- 11 files changed, 104 insertions(+), 66 deletions(-) diff --git a/test/e2e/kubernetes/deployment/deployment.go b/test/e2e/kubernetes/deployment/deployment.go index 4e711ee48e..5cf16f7afc 100644 --- a/test/e2e/kubernetes/deployment/deployment.go +++ b/test/e2e/kubernetes/deployment/deployment.go @@ -21,7 +21,6 @@ import ( ) const ( - commandTimeout = 1 * time.Minute validateDeploymentNotExistRetries = 3 deploymentGetAfterCreateTimeout = 1 * time.Minute ) @@ -73,6 +72,7 @@ type Container struct { // CreateLinuxDeploy will create a deployment for a given image with a name in a namespace // --overrides='{ "spec":{"template":{"spec": {"nodeSelector":{"beta.kubernetes.io/os":"linux"}}}}}' func CreateLinuxDeploy(image, name, namespace, miscOpts string) (*Deployment, error) { + var commandTimeout time.Duration var cmd *exec.Cmd overrides := `{ "spec":{"template":{"spec": {"nodeSelector":{"beta.kubernetes.io/os":"linux"}}}}}` if miscOpts != "" { @@ -118,6 +118,7 @@ func CreateLinuxDeployDeleteIfExists(pattern, image, name, namespace, miscOpts s // RunLinuxDeploy will create a deployment that runs a bash command in a pod // --overrides=' "spec":{"template":{"spec": {"nodeSelector":{"beta.kubernetes.io/os":"linux"}}}}}' func RunLinuxDeploy(image, name, namespace, command string, replicas int) (*Deployment, error) { + var commandTimeout time.Duration overrides := `{ "spec":{"template":{"spec": {"nodeSelector":{"beta.kubernetes.io/os":"linux"}}}}}` cmd := exec.Command("k", "run", name, "-n", namespace, "--image", image, "--image-pull-policy=IfNotPresent", "--replicas", strconv.Itoa(replicas), "--overrides", overrides, "--command", "--", "/bin/sh", "-c", command) out, err := util.RunAndLogCommand(cmd, commandTimeout) @@ -148,6 +149,7 @@ func RunLinuxDeployDeleteIfExists(pattern, image, name, namespace, command strin // CreateWindowsDeploy will create a deployment for a given image with a name in a namespace and create a service mapping a hostPort func CreateWindowsDeploy(pattern, image, name, namespace, miscOpts string) (*Deployment, error) { + var commandTimeout time.Duration overrides := `{ "spec":{"template":{"spec": {"nodeSelector":{"beta.kubernetes.io/os":"windows"}}}}}` var args []string args = append(args, "run", name) @@ -173,6 +175,7 @@ func CreateWindowsDeploy(pattern, image, name, namespace, miscOpts string) (*Dep // CreateWindowsDeployWithHostport will create a deployment for a given image with a name in a namespace and create a service mapping a hostPort func CreateWindowsDeployWithHostport(image, name, namespace string, port int, hostport int) (*Deployment, error) { + var commandTimeout time.Duration overrides := `{ "spec":{"template":{"spec": {"nodeSelector":{"beta.kubernetes.io/os":"windows"}}}}}` cmd := exec.Command("k", "run", name, "-n", namespace, "--image", image, "--image-pull-policy=IfNotPresent", "--port", strconv.Itoa(port), "--hostport", strconv.Itoa(hostport), "--overrides", overrides) out, err := util.RunAndLogCommand(cmd, commandTimeout) @@ -287,6 +290,7 @@ func GetAllByPrefix(prefix, namespace string) ([]Deployment, error) { // Describe will describe a deployment resource func (d *Deployment) Describe() error { + var commandTimeout time.Duration cmd := exec.Command("k", "describe", "deployment", d.Metadata.Name, "-n", d.Metadata.Namespace) out, err := util.RunAndLogCommand(cmd, commandTimeout) log.Printf("\n%s\n", string(out)) @@ -295,6 +299,7 @@ func (d *Deployment) Describe() error { // Delete will delete a deployment in a given namespace func (d *Deployment) Delete(retries int) error { + var commandTimeout time.Duration var kubectlOutput []byte var kubectlError error for i := 0; i < retries; i++ { @@ -328,6 +333,7 @@ func (d *Deployment) Delete(retries int) error { // Expose will create a load balancer and expose the deployment on a given port func (d *Deployment) Expose(svcType string, targetPort, exposedPort int) error { + var commandTimeout time.Duration cmd := exec.Command("k", "expose", "deployment", d.Metadata.Name, "--type", svcType, "-n", d.Metadata.Namespace, "--target-port", strconv.Itoa(targetPort), "--port", strconv.Itoa(exposedPort)) out, err := util.RunAndLogCommand(cmd, commandTimeout) if err != nil { @@ -361,6 +367,7 @@ func (d *Deployment) ExposeDeleteIfExist(pattern, namespace, svcType string, tar // ScaleDeployment scales a deployment to n instancees func (d *Deployment) ScaleDeployment(n int) error { + var commandTimeout time.Duration cmd := exec.Command("k", "scale", fmt.Sprintf("--replicas=%d", n), "deployment", d.Metadata.Name) out, err := util.RunAndLogCommand(cmd, commandTimeout) if err != nil { @@ -372,6 +379,7 @@ func (d *Deployment) ScaleDeployment(n int) error { // CreateDeploymentHPA applies autoscale characteristics to deployment func (d *Deployment) CreateDeploymentHPA(cpuPercent, min, max int) error { + var commandTimeout time.Duration cmd := exec.Command("k", "autoscale", "deployment", d.Metadata.Name, fmt.Sprintf("--cpu-percent=%d", cpuPercent), fmt.Sprintf("--min=%d", min), fmt.Sprintf("--max=%d", max)) out, err := util.RunAndLogCommand(cmd, commandTimeout) diff --git a/test/e2e/kubernetes/hpa/hpa.go b/test/e2e/kubernetes/hpa/hpa.go index a1f8c23dd0..df3e924808 100644 --- a/test/e2e/kubernetes/hpa/hpa.go +++ b/test/e2e/kubernetes/hpa/hpa.go @@ -15,8 +15,6 @@ import ( "github.com/pkg/errors" ) -const commandTimeout = 1 * time.Minute - type List struct { HPAs []HPA `json:"items"` } @@ -132,6 +130,7 @@ func GetAllByPrefix(prefix, namespace string) ([]HPA, error) { // Describe will describe a HPA resource func (h *HPA) Describe() error { + var commandTimeout time.Duration cmd := exec.Command("k", "describe", "hpa", h.Metadata.Name, "-n", h.Metadata.Namespace) out, err := util.RunAndLogCommand(cmd, commandTimeout) log.Printf("\n%s\n", string(out)) @@ -140,6 +139,7 @@ func (h *HPA) Describe() error { // Delete will delete a HPA in a given namespace func (h *HPA) Delete(retries int) error { + var commandTimeout time.Duration var kubectlOutput []byte var kubectlError error for i := 0; i < retries; i++ { diff --git a/test/e2e/kubernetes/job/job.go b/test/e2e/kubernetes/job/job.go index b11720c300..f44db09fbf 100644 --- a/test/e2e/kubernetes/job/job.go +++ b/test/e2e/kubernetes/job/job.go @@ -20,8 +20,6 @@ import ( "github.com/pkg/errors" ) -const commandTimeout = 1 * time.Minute - // List is a container that holds all jobs returned from doing a kubectl get jobs type List struct { Jobs []Job `json:"items"` @@ -311,6 +309,7 @@ func DescribeJobs(jobPrefix, namespace string) { // Describe will describe a Job resource func (j *Job) Describe() error { + var commandTimeout time.Duration cmd := exec.Command("k", "describe", "jobs/", j.Metadata.Name, "-n", j.Metadata.Namespace) out, err := util.RunAndLogCommand(cmd, commandTimeout) log.Printf("\n%s\n", string(out)) diff --git a/test/e2e/kubernetes/kubernetes_test.go b/test/e2e/kubernetes/kubernetes_test.go index cb2b2de744..873a40d914 100644 --- a/test/e2e/kubernetes/kubernetes_test.go +++ b/test/e2e/kubernetes/kubernetes_test.go @@ -48,7 +48,7 @@ const ( kubeSystemPodsReadinessChecks = 6 sleepBetweenRetriesWhenWaitingForPodReady = 1 * time.Second timeoutWhenWaitingForPodOutboundAccess = 1 * time.Minute - stabilityCommandTimeout = 1 * time.Second + stabilityCommandTimeout = 3 * time.Second windowsCommandTimeout = 1 * time.Minute validateNetworkPolicyTimeout = 3 * time.Minute validateDNSTimeout = 2 * time.Minute @@ -979,7 +979,7 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu By("Ensuring we can create an ILB service attachment") sILB, err := service.CreateServiceFromFileDeleteIfExist(filepath.Join(WorkloadDir, "ingress-nginx-ilb.yaml"), serviceName+"-ilb", "default") Expect(err).NotTo(HaveOccurred()) - svc, err := sILB.WaitForIngress(cfg.Timeout, 5*time.Second) + err = sILB.WaitForIngress(cfg.Timeout, 5*time.Second) Expect(err).NotTo(HaveOccurred()) By("Ensuring we can create a curl pod to connect to the service") @@ -995,12 +995,12 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu By("Ensuring we can connect to the ILB service from another pod") var success bool for _, curlPod := range curlPods { - pass, curlErr := curlPod.ValidateCurlConnection(svc.Status.LoadBalancer.Ingress[0]["ip"], 30*time.Second, 3*time.Minute) + pass, curlErr := curlPod.ValidateCurlConnection(sILB.Status.LoadBalancer.Ingress[0]["ip"], 30*time.Second, 3*time.Minute) if curlErr == nil && pass { success = true break } else { - e := svc.Describe() + e := sILB.Describe() if e != nil { log.Printf("Unable to describe service\n: %s", e) } @@ -1011,21 +1011,21 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu By("Ensuring we can create an ELB service attachment") sELB, err := service.CreateServiceFromFileDeleteIfExist(filepath.Join(WorkloadDir, "ingress-nginx-elb.yaml"), serviceName+"-elb", "default") Expect(err).NotTo(HaveOccurred()) - svc, err = sELB.WaitForIngress(cfg.Timeout, 5*time.Second) + err = sELB.WaitForIngress(cfg.Timeout, 5*time.Second) Expect(err).NotTo(HaveOccurred()) By("Ensuring we can connect to the ELB service on the service IP") - valid := sELB.Validate("(Welcome to nginx)", 5, 30*time.Second, cfg.Timeout) - Expect(valid).To(BeTrue()) + err = sELB.ValidateWithRetry("(Welcome to nginx)", 30*time.Second, cfg.Timeout) + Expect(err).NotTo(HaveOccurred()) By("Ensuring we can connect to the ELB service from another pod") success = false for _, curlPod := range curlPods { - pass, curlErr := curlPod.ValidateCurlConnection(svc.Status.LoadBalancer.Ingress[0]["ip"], 30*time.Second, 3*time.Minute) + pass, curlErr := curlPod.ValidateCurlConnection(sELB.Status.LoadBalancer.Ingress[0]["ip"], 30*time.Second, 3*time.Minute) if curlErr == nil && pass { success = true break } else { - e := svc.Describe() + e := sELB.Describe() if e != nil { log.Printf("Unable to describe service\n: %s", e) } @@ -1569,10 +1569,12 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu Expect(err).NotTo(HaveOccurred()) iisService, err := service.Get(deploymentName, "default") Expect(err).NotTo(HaveOccurred()) + err = iisService.WaitForIngress(cfg.Timeout, 5*time.Second) + Expect(err).NotTo(HaveOccurred()) By("Verifying that the service is reachable and returns the default IIS start page") - valid := iisService.Validate("(IIS Windows Server)", 10, sleepBetweenRetriesWhenWaitingForPodReady, cfg.Timeout) - Expect(valid).To(BeTrue()) + err = iisService.ValidateWithRetry("(IIS Windows Server)", sleepBetweenRetriesWhenWaitingForPodReady, cfg.Timeout) + Expect(err).NotTo(HaveOccurred()) By("Checking that each pod can reach the internet") var iisPods []pod.Pod @@ -1601,8 +1603,8 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu Expect(len(iisPods)).To(Equal(5)) By("Verifying that the service is reachable and returns the default IIS start page") - valid = iisService.Validate("(IIS Windows Server)", 10, sleepBetweenRetriesWhenWaitingForPodReady, cfg.Timeout) - Expect(valid).To(BeTrue()) + err = iisService.ValidateWithRetry("(IIS Windows Server)", sleepBetweenRetriesWhenWaitingForPodReady, cfg.Timeout) + Expect(err).NotTo(HaveOccurred()) By("Checking that each pod can reach the internet") iisPods, err = iisDeploy.Pods() @@ -1632,8 +1634,8 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu Expect(len(iisPods)).To(Equal(2)) By("Verifying that the service is reachable and returns the default IIS start page") - valid = iisService.Validate("(IIS Windows Server)", 10, sleepBetweenRetriesWhenWaitingForPodReady, cfg.Timeout) - Expect(valid).To(BeTrue()) + err = iisService.ValidateWithRetry("(IIS Windows Server)", sleepBetweenRetriesWhenWaitingForPodReady, cfg.Timeout) + Expect(err).NotTo(HaveOccurred()) By("Checking that each pod can reach the internet") iisPods, err = iisDeploy.Pods() @@ -1698,14 +1700,14 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu By("Connecting to Windows from another Windows deployment") name := fmt.Sprintf("windows-2-windows-%s", cfg.Name) command := fmt.Sprintf("iwr -UseBasicParsing -TimeoutSec 60 %s", windowsService.Metadata.Name) - successes, err := pod.RunCommandMultipleTimes(pod.RunWindowsPod, windowsImages.ServerCore, name, command, cfg.StabilityIterations, 1*time.Second, retryCommandsTimeout, windowsCommandTimeout) + successes, err := pod.RunCommandMultipleTimes(pod.RunWindowsPod, windowsImages.ServerCore, name, command, cfg.StabilityIterations, 1*time.Second, windowsCommandTimeout, retryCommandsTimeout) Expect(err).NotTo(HaveOccurred()) Expect(successes).To(Equal(cfg.StabilityIterations)) By("Connecting to Linux from Windows deployment") name = fmt.Sprintf("windows-2-linux-%s", cfg.Name) command = fmt.Sprintf("iwr -UseBasicParsing -TimeoutSec 60 %s", linuxService.Metadata.Name) - successes, err = pod.RunCommandMultipleTimes(pod.RunWindowsPod, windowsImages.ServerCore, name, command, cfg.StabilityIterations, 1*time.Second, retryCommandsTimeout, windowsCommandTimeout) + successes, err = pod.RunCommandMultipleTimes(pod.RunWindowsPod, windowsImages.ServerCore, name, command, cfg.StabilityIterations, 1*time.Second, windowsCommandTimeout, retryCommandsTimeout) Expect(err).NotTo(HaveOccurred()) Expect(successes).To(Equal(cfg.StabilityIterations)) diff --git a/test/e2e/kubernetes/node/node.go b/test/e2e/kubernetes/node/node.go index 5cb78e5bf2..a0d4c1379b 100644 --- a/test/e2e/kubernetes/node/node.go +++ b/test/e2e/kubernetes/node/node.go @@ -18,8 +18,7 @@ import ( const ( //ServerVersion is used to parse out the version of the API running - ServerVersion = `(Server Version:\s)+(.*)` - commandTimeout = 1 * time.Minute + ServerVersion = `(Server Version:\s)+(.*)` ) // Node represents the kubernetes Node Resource @@ -162,6 +161,7 @@ func DescribeNodes() { // Describe will describe a node resource func (n *Node) Describe() error { + var commandTimeout time.Duration cmd := exec.Command("k", "describe", "node", n.Metadata.Name) out, err := util.RunAndLogCommand(cmd, commandTimeout) log.Printf("\n%s\n", string(out)) diff --git a/test/e2e/kubernetes/persistentvolume/persistentvolume.go b/test/e2e/kubernetes/persistentvolume/persistentvolume.go index 89f89cabf6..633433c279 100644 --- a/test/e2e/kubernetes/persistentvolume/persistentvolume.go +++ b/test/e2e/kubernetes/persistentvolume/persistentvolume.go @@ -13,8 +13,6 @@ import ( "github.com/Azure/aks-engine/test/e2e/kubernetes/util" ) -const commandTimeout = 1 * time.Minute - // PersistentVolume is used to parse data from kubectl get pv type PersistentVolume struct { Metadata Metadata `json:"metadata"` @@ -86,6 +84,7 @@ func DescribePVs() { // Describe will describe a pv resource func (pv *PersistentVolume) Describe() error { + var commandTimeout time.Duration cmd := exec.Command("k", "describe", "pv", pv.Metadata.Name) out, err := util.RunAndLogCommand(cmd, commandTimeout) log.Printf("\n%s\n", string(out)) diff --git a/test/e2e/kubernetes/persistentvolumeclaims/persistentvolumeclaims.go b/test/e2e/kubernetes/persistentvolumeclaims/persistentvolumeclaims.go index 5896f13f14..6fb85f2b66 100644 --- a/test/e2e/kubernetes/persistentvolumeclaims/persistentvolumeclaims.go +++ b/test/e2e/kubernetes/persistentvolumeclaims/persistentvolumeclaims.go @@ -15,8 +15,6 @@ import ( "github.com/pkg/errors" ) -const commandTimeout = 1 * time.Minute - type List struct { PersistentVolumeClaims []PersistentVolumeClaim `json:"items"` } @@ -179,6 +177,7 @@ func DescribePVCs(pvcPrefix, namespace string) { // Describe will describe a pv resource func (pvc *PersistentVolumeClaim) Describe() error { + var commandTimeout time.Duration cmd := exec.Command("k", "describe", "pvc", pvc.Metadata.Name, "-n", pvc.Metadata.Namespace) out, err := util.RunAndLogCommand(cmd, commandTimeout) log.Printf("\n%s\n", string(out)) @@ -187,6 +186,7 @@ func (pvc *PersistentVolumeClaim) Describe() error { // Delete will delete a PersistentVolumeClaim in a given namespace func (pvc *PersistentVolumeClaim) Delete(retries int) error { + var commandTimeout time.Duration var kubectlOutput []byte var kubectlError error for i := 0; i < retries; i++ { diff --git a/test/e2e/kubernetes/pod/pod.go b/test/e2e/kubernetes/pod/pod.go index e11c0630eb..369c27fa2c 100644 --- a/test/e2e/kubernetes/pod/pod.go +++ b/test/e2e/kubernetes/pod/pod.go @@ -25,7 +25,6 @@ import ( const ( testDir string = "testdirectory" - commandTimeout = 1 * time.Minute deleteTimeout = 5 * time.Minute validatePodNotExistRetries = 3 ) @@ -1112,6 +1111,7 @@ func (p *Pod) CheckWindowsOutboundConnection(sleep, timeout time.Duration) (bool // ValidateHostPort will attempt to run curl against the POD's hostIP and hostPort func (p *Pod) ValidateHostPort(check string, attempts int, sleep time.Duration, master, sshKeyPath string) bool { + var commandTimeout time.Duration hostIP := p.Status.HostIP if len(p.Spec.Containers) == 0 || len(p.Spec.Containers[0].Ports) == 0 { log.Printf("Unexpected POD container spec: %v. Should have hostPort.\n", p.Spec) @@ -1138,6 +1138,7 @@ func (p *Pod) ValidateHostPort(check string, attempts int, sleep time.Duration, // Logs will get logs from all containers in a pod func (p *Pod) Logs() error { + var commandTimeout time.Duration for _, container := range p.Spec.Containers { cmd := exec.Command("k", "logs", p.Metadata.Name, "-c", container.Name, "-n", p.Metadata.Namespace) out, err := util.RunAndLogCommand(cmd, commandTimeout) @@ -1151,6 +1152,7 @@ func (p *Pod) Logs() error { // Describe will describe a pod resource func (p *Pod) Describe() error { + var commandTimeout time.Duration cmd := exec.Command("k", "describe", "pod", p.Metadata.Name, "-n", p.Metadata.Namespace) out, err := util.RunAndLogCommand(cmd, commandTimeout) log.Printf("\n%s\n", string(out)) diff --git a/test/e2e/kubernetes/service/service.go b/test/e2e/kubernetes/service/service.go index dced775a14..e114edc34e 100644 --- a/test/e2e/kubernetes/service/service.go +++ b/test/e2e/kubernetes/service/service.go @@ -18,8 +18,6 @@ import ( "github.com/pkg/errors" ) -const commandTimeout = 3 * time.Minute - // List holds a list of services returned from kubectl get svc type List struct { Services []Service `json:"items"` @@ -153,6 +151,7 @@ func GetAllByPrefix(prefix, namespace string) ([]Service, error) { // Delete will delete a service in a given namespace func (s *Service) Delete(retries int) error { + var commandTimeout time.Duration var kubectlOutput []byte var kubectlError error for i := 0; i < retries; i++ { @@ -185,6 +184,7 @@ func DescribeServices(svcPrefix, namespace string) { // Describe will describe a service resource func (s *Service) Describe() error { + var commandTimeout time.Duration cmd := exec.Command("k", "describe", "svc", s.Metadata.Name, "-n", s.Metadata.Namespace) out, err := util.RunAndLogCommand(cmd, commandTimeout) log.Printf("\n%s\n", string(out)) @@ -202,7 +202,7 @@ func (s *Service) GetNodePort(port int) int { } // WaitForIngress waits for an Ingress to be provisioned -func (s *Service) WaitForIngress(timeout, sleep time.Duration) (*Service, error) { +func (s *Service) WaitForIngress(timeout, sleep time.Duration) error { ctx, cancel := context.WithTimeout(context.Background(), timeout) defer cancel() var mostRecentWaitForIngressError error @@ -224,7 +224,8 @@ func (s *Service) WaitForIngress(timeout, sleep time.Duration) (*Service, error) svc := result.svc if mostRecentWaitForIngressError == nil { if svc != nil && svc.Status.LoadBalancer.Ingress != nil { - return svc, nil + s.Status.LoadBalancer.Ingress = svc.Status.LoadBalancer.Ingress + return nil } } case <-ctx.Done(): @@ -232,7 +233,7 @@ func (s *Service) WaitForIngress(timeout, sleep time.Duration) (*Service, error) if err != nil { log.Printf("Unable to describe service\n: %s", err) } - return nil, errors.Errorf("WaitForIngress timed out: %s\n", mostRecentWaitForIngressError) + return errors.Errorf("WaitForIngress timed out: %s\n", mostRecentWaitForIngressError) } } } @@ -271,40 +272,65 @@ func WaitOnDeleted(servicePrefix, namespace string, sleep, timeout time.Duration } } -// Validate will attempt to run an http.Get against the root service url -func (s *Service) Validate(check string, attempts int, sleep, wait time.Duration) bool { - var err error - var url string - var i int - var resp *http.Response - svc, waitErr := s.WaitForIngress(wait, 5*time.Second) - if waitErr != nil { - log.Printf("Unable to verify external IP, cannot validate service:%s\n", waitErr) - return false - } - if svc.Status.LoadBalancer.Ingress == nil || len(svc.Status.LoadBalancer.Ingress) == 0 { - log.Printf("Service LB ingress is empty or nil: %#v\n", svc.Status.LoadBalancer.Ingress) - return false - } - for i = 1; i <= attempts; i++ { - url = fmt.Sprintf("http://%s", svc.Status.LoadBalancer.Ingress[0]["ip"]) - resp, err = http.Get(url) - if err == nil { - body, _ := ioutil.ReadAll(resp.Body) - matched, _ := regexp.MatchString(check, string(body)) - if matched { - defer resp.Body.Close() - return true +// ValidateWithRetry waits for an Ingress to be provisioned +func (s *Service) ValidateWithRetry(bodyResponseTextMatch string, sleep, timeout time.Duration) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + var mostRecentValidateWithRetryError error + ch := make(chan error) + go func() { + for { + select { + case <-ctx.Done(): + return + case ch <- s.Validate(bodyResponseTextMatch): + time.Sleep(sleep) + } + } + }() + for { + select { + case result := <-ch: + mostRecentValidateWithRetryError = result + if mostRecentValidateWithRetryError == nil { + return nil } - log.Printf("Got unexpected URL body, expected to find %s, got:\n%s\n", check, string(body)) + case <-ctx.Done(): + err := s.Describe() + if err != nil { + log.Printf("Unable to describe service\n: %s", err) + } + return errors.Errorf("ValidateWithRetry timed out: %s\n", mostRecentValidateWithRetryError) } - time.Sleep(sleep) } - log.Printf("Unable to validate URL %s after %s, err: %#v\n", url, time.Duration(i)*wait, err) +} + +// Validate will attempt to run an http.Get against the root service url +func (s *Service) Validate(bodyResponseTextMatch string) error { + if len(s.Status.LoadBalancer.Ingress) < 1 { + return errors.Errorf("No LB ingress IP for service %s", s.Metadata.Name) + } + var resp *http.Response + url := fmt.Sprintf("http://%s", s.Status.LoadBalancer.Ingress[0]["ip"]) + resp, err := http.Get(url) if resp != nil { defer resp.Body.Close() } - return false + if err != nil { + return errors.Errorf("Unable to call service at URL %s: %s", url, err) + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return errors.Errorf("Unable to parse response body: %s", err) + } + matched, err := regexp.MatchString(bodyResponseTextMatch, string(body)) + if err != nil { + return errors.Errorf("Unable to evalute response body against a regular expression match: %s", err) + } + if matched { + return nil + } + return errors.Errorf("Got unexpected URL body, expected to find %s, got:\n%s\n", bodyResponseTextMatch, string(body)) } // CreateServiceFromFile will create a Service from file with a name diff --git a/test/e2e/kubernetes/storageclass/storageclass.go b/test/e2e/kubernetes/storageclass/storageclass.go index f899c51443..b06f2a63d1 100644 --- a/test/e2e/kubernetes/storageclass/storageclass.go +++ b/test/e2e/kubernetes/storageclass/storageclass.go @@ -14,8 +14,6 @@ import ( "github.com/pkg/errors" ) -const commandTimeout = 1 * time.Minute - // StorageClass is used to parse data from kubectl get storageclass type StorageClass struct { Metadata Metadata `json:"metadata"` @@ -84,6 +82,7 @@ func Get(scName string) (*StorageClass, error) { // Describe will describe a storageclass resource func (sc *StorageClass) Describe() error { + var commandTimeout time.Duration cmd := exec.Command("k", "describe", "storageclass", sc.Metadata.Name) out, err := util.RunAndLogCommand(cmd, commandTimeout) log.Printf("\n%s\n", string(out)) diff --git a/test/e2e/kubernetes/util/util.go b/test/e2e/kubernetes/util/util.go index 794aa6af40..625a394074 100644 --- a/test/e2e/kubernetes/util/util.go +++ b/test/e2e/kubernetes/util/util.go @@ -23,6 +23,7 @@ func PrintCommand(cmd *exec.Cmd) { // RunAndLogCommand logs the command with a timestamp when it's run, and the duration at end func RunAndLogCommand(cmd *exec.Cmd, timeout time.Duration) ([]byte, error) { + var zeroValueDuration time.Duration var err error var out []byte cmdLine := fmt.Sprintf("$ %s", strings.Join(cmd.Args, " ")) @@ -32,8 +33,10 @@ func RunAndLogCommand(cmd *exec.Cmd, timeout time.Duration) ([]byte, error) { end := time.Now() total := time.Since(start) log.Printf("#### %s completed in %s", cmdLine, end.Sub(start).String()) - if total.Seconds() > timeout.Seconds() { - err = errors.Errorf("%s took too long!", cmdLine) + if zeroValueDuration != timeout { + if total.Seconds() > timeout.Seconds() { + err = errors.Errorf("%s took too long!", cmdLine) + } } return out, err }