Skip to content
This repository has been archived by the owner on Jan 11, 2023. It is now read-only.

Commit

Permalink
feat(kubernetes_test): Add outbound internet checks for both agent ty…
Browse files Browse the repository at this point in the history 
…pes (#1502)
  • Loading branch information
@jchauncey @jackfrancis
jchauncey authored and jackfrancis committed Sep 28, 2017
1 parent 3760b36 commit eebf5e8
Show file tree
Hide file tree
Showing 3 changed files with 160 additions and 7 deletions.
7 changes: 7 additions & 0 deletions test/e2e/kubernetes/deployment/deployment.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@ import (
"os/exec"
"strconv"
"time"

"github.com/Azure/acs-engine/test/e2e/kubernetes/pod"
)

// List holds a list of deployments returned from kubectl get deploy
Expand Down Expand Up @@ -119,3 +121,8 @@ func (d *Deployment) Expose(targetPort, exposedPort int) error {
}
return nil
}

// Pods will return all pods related to a deployment
func (d *Deployment) Pods() ([]pod.Pod, error) {
return pod.GetAllByPrefix(d.Metadata.Name, d.Metadata.Namespace)
}
32 changes: 25 additions & 7 deletions test/e2e/kubernetes/kubernetes_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ var _ = BeforeSuite(func() {
})

var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", func() {
Context("regardless of agent pool type", func() {
Describe("regardless of agent pool type", func() {

It("should have have the appropriate node count", func() {
nodeList, err := node.Get()
Expand Down Expand Up @@ -155,19 +155,19 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
})
})

Context("with a linux agent pool", func() {
Describe("with a linux agent pool", func() {
It("should be able to deploy an nginx service", func() {
if eng.HasLinuxAgents() {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
deploymentName := fmt.Sprintf("nginx-%s-%v", cfg.Name, r.Intn(99999))
d, err := deployment.CreateLinuxDeploy("library/nginx:latest", deploymentName, "default")
nginxDeploy, err := deployment.CreateLinuxDeploy("library/nginx:latest", deploymentName, "default")
Expect(err).NotTo(HaveOccurred())

running, err := pod.WaitOnReady(deploymentName, "default", 5*time.Second, cfg.Timeout)
Expect(err).NotTo(HaveOccurred())
Expect(running).To(Equal(true))

err = d.Expose(80, 80)
err = nginxDeploy.Expose(80, 80)
Expect(err).NotTo(HaveOccurred())

s, err := service.Get(deploymentName, "default")
Expand All @@ -178,25 +178,34 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu

valid := s.Validate("(Welcome to nginx)", 5, 5*time.Second)
Expect(valid).To(BeTrue())

nginxPods, err := nginxDeploy.Pods()
Expect(err).NotTo(HaveOccurred())
Expect(len(nginxPods)).ToNot(BeZero())
for _, nginxPod := range nginxPods {
pass, err := nginxPod.CheckLinuxOutboundConnection(5*time.Second, cfg.Timeout)
Expect(err).NotTo(HaveOccurred())
Expect(pass).To(BeTrue())
}
} else {
Skip("No linux agent was provisioned for this Cluster Definition")
}
})
})

Context("with a windows agent pool", func() {
Describe("with a windows agent pool", func() {
It("should be able to deploy an iis webserver", func() {
if eng.HasWindowsAgents() {
r := rand.New(rand.NewSource(time.Now().UnixNano()))
deploymentName := fmt.Sprintf("iis-%s-%v", cfg.Name, r.Intn(99999))
d, err := deployment.CreateWindowsDeploy("microsoft/iis", deploymentName, "default", 80)
iisDeploy, err := deployment.CreateWindowsDeploy("microsoft/iis", deploymentName, "default", 80)
Expect(err).NotTo(HaveOccurred())

running, err := pod.WaitOnReady(deploymentName, "default", 5*time.Second, cfg.Timeout)
Expect(err).NotTo(HaveOccurred())
Expect(running).To(Equal(true))

err = d.Expose(80, 80)
err = iisDeploy.Expose(80, 80)
Expect(err).NotTo(HaveOccurred())

s, err := service.Get(deploymentName, "default")
Expand All @@ -207,6 +216,15 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu

valid := s.Validate("(IIS Windows Server)", 5, 5*time.Second)
Expect(valid).To(BeTrue())

iisPods, err := iisDeploy.Pods()
Expect(err).NotTo(HaveOccurred())
Expect(len(iisPods)).ToNot(BeZero())
for _, iisPod := range iisPods {
pass, err := iisPod.CheckWindowsOutboundConnection(5*time.Second, cfg.Timeout)
Expect(err).NotTo(HaveOccurred())
Expect(pass).To(BeTrue())
}
} else {
Skip("No windows agent was provisioned for this Cluster Definition")
}
Expand Down
128 changes: 128 additions & 0 deletions test/e2e/kubernetes/pod/pod.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,27 @@ func Get(podName, namespace string) (*Pod, error) {
return &p, nil
}

// GetAllByPrefix will return all pods in a given namespace that match a prefix
func GetAllByPrefix(prefix, namespace string) ([]Pod, error) {
pl, err := GetAll(namespace)
if err != nil {
log.Printf("Error while trying to check if all pods are in running state:%s", err)
return nil, err
}
pods := []Pod{}
for _, p := range pl.Pods {
matched, err := regexp.MatchString(prefix+"-.*", p.Metadata.Name)
if err != nil {
log.Printf("Error trying to match pod name:%s\n", err)
return nil, err
}
if matched {
pods = append(pods, p)
}
}
return pods, nil
}

// AreAllPodsRunning will return true if all pods are in a Running State
func AreAllPodsRunning(podPrefix, namespace string) (bool, error) {
pl, err := GetAll(namespace)
Expand Down Expand Up @@ -139,3 +160,110 @@ func WaitOnReady(podPrefix, namespace string, sleep, duration time.Duration) (bo
}
}
}

// Exec will execute the given command in the pod
func (p *Pod) Exec(cmd ...string) ([]byte, error) {
execCmd := []string{"exec", p.Metadata.Name, "-n", p.Metadata.Namespace}
for _, s := range cmd {
execCmd = append(execCmd, s)
}
out, err := exec.Command("kubectl", execCmd...).CombinedOutput()
if err != nil {
log.Printf("Error trying to run 'kubectl exec':%s\n", string(out))
log.Printf("Command:kubectl exec %s -n %s %s \n", p.Metadata.Name, p.Metadata.Namespace, cmd)
return nil, err
}
return out, nil
}

// CheckLinuxOutboundConnection will keep retrying the check if an error is received until the timeout occurs or it passes. This helps us when DNS may not be available for some time after a pod starts.
func (p *Pod) CheckLinuxOutboundConnection(sleep, duration time.Duration) (bool, error) {
exp, err := regexp.Compile("200 OK")
if err != nil {
log.Printf("Error while trying to create regex for linux outbound check:%s\n", err)
return false, err
}
readyCh := make(chan bool, 1)
errCh := make(chan error)
ctx, cancel := context.WithTimeout(context.Background(), duration)
defer cancel()
go func() {
for {
select {
case <-ctx.Done():
errCh <- fmt.Errorf("Timeout exceeded (%s) while waiting for Pod (%s) to check outbound internet connection", duration.String(), p.Metadata.Name)
default:
_, err := p.Exec("--", "/usr/bin/apt", "update")
if err != nil {
break
}
_, err = p.Exec("--", "/usr/bin/apt", "install", "-y", "curl")
if err != nil {
break
}
out, err := p.Exec("--", "curl", "-I", "http://www.bing.com")
if err == nil {
matched := exp.MatchString(string(out))
if matched {
readyCh <- true
} else {
readyCh <- false
}
} else {
log.Printf("Error:%s\n", err)
log.Printf("Out:%s\n", out)
}
}
}
}()
for {
select {
case err := <-errCh:
return false, err
case ready := <-readyCh:
return ready, nil
}
}
}

// CheckWindowsOutboundConnection will keep retrying the check if an error is received until the timeout occurs or it passes. This helps us when DNS may not be available for some time after a pod starts.
func (p *Pod) CheckWindowsOutboundConnection(sleep, duration time.Duration) (bool, error) {
exp, err := regexp.Compile("(StatusCode\\s*:\\s*200)")
if err != nil {
log.Printf("Error while trying to create regex for windows outbound check:%s\n", err)
return false, err
}
readyCh := make(chan bool, 1)
errCh := make(chan error)
ctx, cancel := context.WithTimeout(context.Background(), duration)
defer cancel()
go func() {
for {
select {
case <-ctx.Done():
errCh <- fmt.Errorf("Timeout exceeded (%s) while waiting for Pod (%s) to check outbound internet connection", duration.String(), p.Metadata.Name)
default:
out, err := p.Exec("--", "powershell", "iwr", "-UseBasicParsing", "-TimeoutSec", "60", "www.bing.com")
if err == nil {
matched := exp.MatchString(string(out))
if matched {
readyCh <- true
} else {
readyCh <- false
}
} else {
log.Printf("Error:%s\n", err)
log.Printf("Out:%s\n", out)
}
}
}
}()
for {
select {
case err := <-errCh:
return false, err
case ready := <-readyCh:
return ready, nil
}
}
}

0 comments on commit eebf5e8

Please sign in to comment.