From eebf5e801f020299038ebe48e1c17824df14d2b4 Mon Sep 17 00:00:00 2001 From: Jonathan Chauncey Date: Thu, 28 Sep 2017 13:49:43 -0400 Subject: [PATCH] feat(kubernetes_test): Add outbound internet checks for both agent types (#1502) --- test/e2e/kubernetes/deployment/deployment.go | 7 + test/e2e/kubernetes/kubernetes_test.go | 32 ++++- test/e2e/kubernetes/pod/pod.go | 128 +++++++++++++++++++ 3 files changed, 160 insertions(+), 7 deletions(-) diff --git a/test/e2e/kubernetes/deployment/deployment.go b/test/e2e/kubernetes/deployment/deployment.go index 206a8e4248..c68204732a 100644 --- a/test/e2e/kubernetes/deployment/deployment.go +++ b/test/e2e/kubernetes/deployment/deployment.go @@ -6,6 +6,8 @@ import ( "os/exec" "strconv" "time" + + "github.com/Azure/acs-engine/test/e2e/kubernetes/pod" ) // List holds a list of deployments returned from kubectl get deploy @@ -119,3 +121,8 @@ func (d *Deployment) Expose(targetPort, exposedPort int) error { } return nil } + +// Pods will return all pods related to a deployment +func (d *Deployment) Pods() ([]pod.Pod, error) { + return pod.GetAllByPrefix(d.Metadata.Name, d.Metadata.Namespace) +} diff --git a/test/e2e/kubernetes/kubernetes_test.go b/test/e2e/kubernetes/kubernetes_test.go index c54b669725..40eff8c467 100644 --- a/test/e2e/kubernetes/kubernetes_test.go +++ b/test/e2e/kubernetes/kubernetes_test.go @@ -45,7 +45,7 @@ var _ = BeforeSuite(func() { }) var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", func() { - Context("regardless of agent pool type", func() { + Describe("regardless of agent pool type", func() { It("should have have the appropriate node count", func() { nodeList, err := node.Get() @@ -155,19 +155,19 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu }) }) - Context("with a linux agent pool", func() { + Describe("with a linux agent pool", func() { It("should be able to deploy an nginx service", func() { if eng.HasLinuxAgents() { r := rand.New(rand.NewSource(time.Now().UnixNano())) deploymentName := fmt.Sprintf("nginx-%s-%v", cfg.Name, r.Intn(99999)) - d, err := deployment.CreateLinuxDeploy("library/nginx:latest", deploymentName, "default") + nginxDeploy, err := deployment.CreateLinuxDeploy("library/nginx:latest", deploymentName, "default") Expect(err).NotTo(HaveOccurred()) running, err := pod.WaitOnReady(deploymentName, "default", 5*time.Second, cfg.Timeout) Expect(err).NotTo(HaveOccurred()) Expect(running).To(Equal(true)) - err = d.Expose(80, 80) + err = nginxDeploy.Expose(80, 80) Expect(err).NotTo(HaveOccurred()) s, err := service.Get(deploymentName, "default") @@ -178,25 +178,34 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu valid := s.Validate("(Welcome to nginx)", 5, 5*time.Second) Expect(valid).To(BeTrue()) + + nginxPods, err := nginxDeploy.Pods() + Expect(err).NotTo(HaveOccurred()) + Expect(len(nginxPods)).ToNot(BeZero()) + for _, nginxPod := range nginxPods { + pass, err := nginxPod.CheckLinuxOutboundConnection(5*time.Second, cfg.Timeout) + Expect(err).NotTo(HaveOccurred()) + Expect(pass).To(BeTrue()) + } } else { Skip("No linux agent was provisioned for this Cluster Definition") } }) }) - Context("with a windows agent pool", func() { + Describe("with a windows agent pool", func() { It("should be able to deploy an iis webserver", func() { if eng.HasWindowsAgents() { r := rand.New(rand.NewSource(time.Now().UnixNano())) deploymentName := fmt.Sprintf("iis-%s-%v", cfg.Name, r.Intn(99999)) - d, err := deployment.CreateWindowsDeploy("microsoft/iis", deploymentName, "default", 80) + iisDeploy, err := deployment.CreateWindowsDeploy("microsoft/iis", deploymentName, "default", 80) Expect(err).NotTo(HaveOccurred()) running, err := pod.WaitOnReady(deploymentName, "default", 5*time.Second, cfg.Timeout) Expect(err).NotTo(HaveOccurred()) Expect(running).To(Equal(true)) - err = d.Expose(80, 80) + err = iisDeploy.Expose(80, 80) Expect(err).NotTo(HaveOccurred()) s, err := service.Get(deploymentName, "default") @@ -207,6 +216,15 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu valid := s.Validate("(IIS Windows Server)", 5, 5*time.Second) Expect(valid).To(BeTrue()) + + iisPods, err := iisDeploy.Pods() + Expect(err).NotTo(HaveOccurred()) + Expect(len(iisPods)).ToNot(BeZero()) + for _, iisPod := range iisPods { + pass, err := iisPod.CheckWindowsOutboundConnection(5*time.Second, cfg.Timeout) + Expect(err).NotTo(HaveOccurred()) + Expect(pass).To(BeTrue()) + } } else { Skip("No windows agent was provisioned for this Cluster Definition") } diff --git a/test/e2e/kubernetes/pod/pod.go b/test/e2e/kubernetes/pod/pod.go index 33496dfe00..322bb3e453 100644 --- a/test/e2e/kubernetes/pod/pod.go +++ b/test/e2e/kubernetes/pod/pod.go @@ -69,6 +69,27 @@ func Get(podName, namespace string) (*Pod, error) { return &p, nil } +// GetAllByPrefix will return all pods in a given namespace that match a prefix +func GetAllByPrefix(prefix, namespace string) ([]Pod, error) { + pl, err := GetAll(namespace) + if err != nil { + log.Printf("Error while trying to check if all pods are in running state:%s", err) + return nil, err + } + pods := []Pod{} + for _, p := range pl.Pods { + matched, err := regexp.MatchString(prefix+"-.*", p.Metadata.Name) + if err != nil { + log.Printf("Error trying to match pod name:%s\n", err) + return nil, err + } + if matched { + pods = append(pods, p) + } + } + return pods, nil +} + // AreAllPodsRunning will return true if all pods are in a Running State func AreAllPodsRunning(podPrefix, namespace string) (bool, error) { pl, err := GetAll(namespace) @@ -139,3 +160,110 @@ func WaitOnReady(podPrefix, namespace string, sleep, duration time.Duration) (bo } } } + +// Exec will execute the given command in the pod +func (p *Pod) Exec(cmd ...string) ([]byte, error) { + execCmd := []string{"exec", p.Metadata.Name, "-n", p.Metadata.Namespace} + for _, s := range cmd { + execCmd = append(execCmd, s) + } + out, err := exec.Command("kubectl", execCmd...).CombinedOutput() + if err != nil { + log.Printf("Error trying to run 'kubectl exec':%s\n", string(out)) + log.Printf("Command:kubectl exec %s -n %s %s \n", p.Metadata.Name, p.Metadata.Namespace, cmd) + return nil, err + } + return out, nil +} + +// CheckLinuxOutboundConnection will keep retrying the check if an error is received until the timeout occurs or it passes. This helps us when DNS may not be available for some time after a pod starts. +func (p *Pod) CheckLinuxOutboundConnection(sleep, duration time.Duration) (bool, error) { + exp, err := regexp.Compile("200 OK") + if err != nil { + log.Printf("Error while trying to create regex for linux outbound check:%s\n", err) + return false, err + } + readyCh := make(chan bool, 1) + errCh := make(chan error) + ctx, cancel := context.WithTimeout(context.Background(), duration) + defer cancel() + go func() { + for { + select { + case <-ctx.Done(): + errCh <- fmt.Errorf("Timeout exceeded (%s) while waiting for Pod (%s) to check outbound internet connection", duration.String(), p.Metadata.Name) + default: + _, err := p.Exec("--", "/usr/bin/apt", "update") + if err != nil { + break + } + _, err = p.Exec("--", "/usr/bin/apt", "install", "-y", "curl") + if err != nil { + break + } + out, err := p.Exec("--", "curl", "-I", "http://www.bing.com") + if err == nil { + matched := exp.MatchString(string(out)) + if matched { + readyCh <- true + } else { + readyCh <- false + } + } else { + log.Printf("Error:%s\n", err) + log.Printf("Out:%s\n", out) + } + } + } + }() + for { + select { + case err := <-errCh: + return false, err + case ready := <-readyCh: + return ready, nil + } + } +} + +// CheckWindowsOutboundConnection will keep retrying the check if an error is received until the timeout occurs or it passes. This helps us when DNS may not be available for some time after a pod starts. +func (p *Pod) CheckWindowsOutboundConnection(sleep, duration time.Duration) (bool, error) { + exp, err := regexp.Compile("(StatusCode\\s*:\\s*200)") + if err != nil { + log.Printf("Error while trying to create regex for windows outbound check:%s\n", err) + return false, err + } + readyCh := make(chan bool, 1) + errCh := make(chan error) + ctx, cancel := context.WithTimeout(context.Background(), duration) + defer cancel() + go func() { + for { + select { + case <-ctx.Done(): + errCh <- fmt.Errorf("Timeout exceeded (%s) while waiting for Pod (%s) to check outbound internet connection", duration.String(), p.Metadata.Name) + default: + out, err := p.Exec("--", "powershell", "iwr", "-UseBasicParsing", "-TimeoutSec", "60", "www.bing.com") + if err == nil { + matched := exp.MatchString(string(out)) + if matched { + readyCh <- true + } else { + readyCh <- false + } + } else { + log.Printf("Error:%s\n", err) + log.Printf("Out:%s\n", out) + } + } + } + }() + for { + select { + case err := <-errCh: + return false, err + case ready := <-readyCh: + return ready, nil + } + } +}