more catching of build errors, dump of logs, etc.; add registry pod d…

…isk usage analysis

test/extended/builds/docker_pullsecret.go

@@ -6,8 +6,6 @@ import (
 	g "github.com/onsi/ginkgo"
 	o "github.com/onsi/gomega"
 
-	"k8s.io/kubernetes/test/e2e"
-
 	exutil "github.com/openshift/origin/test/extended/util"
 )
 
@@ -44,9 +42,9 @@ var _ = g.Describe("[builds][pullsecret][Conformance] docker build using a pull
 			g.By("expecting the build succeeds")
 			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "docker-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
 			if err != nil {
-				logs, _ := oc.Run("build-logs").Args("docker-build-1").Output()
-				e2e.Failf("build failed: %s", logs)
+				exutil.DumpBuildLogs("docker-build", oc)
 			}
+			o.Expect(err).NotTo(o.HaveOccurred())
 
 			g.By("starting a second build that pulls the image from the first build")
 			_, err = oc.Run("start-build").Args("docker-build-pull").Output()
@@ -55,9 +53,9 @@ var _ = g.Describe("[builds][pullsecret][Conformance] docker build using a pull
 			g.By("expecting the build succeeds")
 			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "docker-build-pull-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
 			if err != nil {
-				logs, _ := oc.Run("build-logs").Args("docker-build-pull-1").Output()
-				e2e.Failf("build failed: %s", logs)
+				exutil.DumpBuildLogs("docker-build-pull", oc)
 			}
+			o.Expect(err).NotTo(o.HaveOccurred())
 		})
 	})
 })

test/extended/builds/image_source.go

@@ -66,6 +66,9 @@ var _ = g.Describe("[builds][Slow] build can have Docker image source", func() {
 			o.Expect(err).NotTo(o.HaveOccurred())
 			g.By("expect the builds to complete successfully")
 			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "imagedockerbuild-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
+			if err != nil {
+				exutil.DumpBuildLogs("imagedockerbuild", oc)
+			}
 			o.Expect(err).NotTo(o.HaveOccurred())
 
 			g.By("expect the pod to deploy successfully")

test/extended/builds/labels.go

@@ -3,8 +3,6 @@ package builds
 import (
 	"fmt"
 
-	"k8s.io/kubernetes/test/e2e"
-
 	g "github.com/onsi/ginkgo"
 	o "github.com/onsi/gomega"
 
@@ -46,9 +44,9 @@ var _ = g.Describe("[builds][Slow] result image should have proper labels set",
 			g.By("o.Expecting the S2I build is in Complete phase")
 			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), buildName, exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
 			if err != nil {
-				logs, _ := oc.Run("build-logs").Args(buildName).Output()
-				e2e.Failf("build failed: %s", logs)
+				exutil.DumpBuildLogs("test", oc)
 			}
+			o.Expect(err).NotTo(o.HaveOccurred())
 
 			g.By("getting the Docker image reference from ImageStream")
 			imageRef, err := exutil.GetDockerImageReference(oc.REST().ImageStreams(oc.Namespace()), "test", "latest")
@@ -82,9 +80,9 @@ var _ = g.Describe("[builds][Slow] result image should have proper labels set",
 			g.By("o.Expecting the Docker build is in Complete phase")
 			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), buildName, exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
 			if err != nil {
-				logs, _ := oc.Run("build-logs").Args(buildName).Output()
-				e2e.Failf("build failed: %s", logs)
+				exutil.DumpBuildLogs("test", oc)
 			}
+			o.Expect(err).NotTo(o.HaveOccurred())
 
 			g.By("getting the Docker image reference from ImageStream")
 			imageRef, err := exutil.GetDockerImageReference(oc.REST().ImageStreams(oc.Namespace()), "test", "latest")

test/extended/builds/secrets.go

@@ -6,7 +6,6 @@ import (
 	g "github.com/onsi/ginkgo"
 	o "github.com/onsi/gomega"
 
-	buildapi "github.com/openshift/origin/pkg/build/api"
 	exutil "github.com/openshift/origin/test/extended/util"
 	kapi "k8s.io/kubernetes/pkg/api"
 )
@@ -49,9 +48,11 @@ var _ = g.Describe("[builds][Slow] can use build secrets", func() {
 			o.Expect(out).To(o.ContainSubstring("relative-secret3=secret3"))
 
 			g.By("checking the status of the build")
-			build, err := oc.REST().Builds(oc.Namespace()).Get("test-1")
+			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "test-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
+			if err != nil {
+				exutil.DumpBuildLogs("test", oc)
+			}
 			o.Expect(err).NotTo(o.HaveOccurred())
-			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
 
 			g.By("getting the image name")
 			image, err := exutil.GetDockerImageReference(oc.REST().ImageStreams(oc.Namespace()), "test", "latest")
@@ -86,9 +87,11 @@ var _ = g.Describe("[builds][Slow] can use build secrets", func() {
 			o.Expect(out).To(o.ContainSubstring("relative-secret2=secret2"))
 
 			g.By("checking the status of the build")
-			build, err := oc.REST().Builds(oc.Namespace()).Get("test-1")
+			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "test-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
+			if err != nil {
+				exutil.DumpBuildLogs("test", oc)
+			}
 			o.Expect(err).NotTo(o.HaveOccurred())
-			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
 		})
 
 	})

test/extended/builds/start.go

@@ -10,7 +10,6 @@ import (
 	g "github.com/onsi/ginkgo"
 	o "github.com/onsi/gomega"
 
-	buildapi "github.com/openshift/origin/pkg/build/api"
 	exutil "github.com/openshift/origin/test/extended/util"
 )
 
@@ -37,9 +36,11 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
 			o.Expect(err).NotTo(o.HaveOccurred())
 
 			g.By(fmt.Sprintf("verifying the build %q status", out))
-			build, err := oc.REST().Builds(oc.Namespace()).Get(out)
+			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
+			if err != nil {
+				exutil.DumpBuildLogs("sample-build", oc)
+			}
 			o.Expect(err).NotTo(o.HaveOccurred())
-			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
 		})
 
 		g.It("should start a build and wait for the build to fail", func() {
@@ -64,9 +65,11 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
 			o.Expect(out).To(o.ContainSubstring("BAR=test"))
 			o.Expect(out).To(o.ContainSubstring("VAR=test"))
 			g.By(fmt.Sprintf("verifying the build %q status", out))
-			build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
+			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
+			if err != nil {
+				exutil.DumpBuildLogs("sample-build", oc)
+			}
 			o.Expect(err).NotTo(o.HaveOccurred())
-			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
 		})
 
 		g.It("should allow to change build log level", func() {
@@ -76,9 +79,11 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
 			g.By(fmt.Sprintf("verifying the build output is not verbose"))
 			o.Expect(out).NotTo(o.ContainSubstring("Creating a new S2I builder"))
 			g.By(fmt.Sprintf("verifying the build %q status", out))
-			build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
+			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
+			if err != nil {
+				exutil.DumpBuildLogs("sample-build", oc)
+			}
 			o.Expect(err).NotTo(o.HaveOccurred())
-			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
 		})
 	})
 
@@ -92,9 +97,10 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
 			o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
 			o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))
 
-			build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
-			o.Expect(err).NotTo(o.HaveOccurred())
-			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
+			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
+			if err != nil {
+				exutil.DumpBuildLogs("sample-build", oc)
+			}
 		})
 
 		g.It("should accept --from-dir as input", func() {
@@ -106,9 +112,11 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
 			o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
 			o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))
 
-			build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
+			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
+			if err != nil {
+				exutil.DumpBuildLogs("sample-build", oc)
+			}
 			o.Expect(err).NotTo(o.HaveOccurred())
-			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
 		})
 
 		g.It("should accept --from-repo as input", func() {
@@ -121,9 +129,11 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
 			o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
 			o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))
 
-			build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
+			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
+			if err != nil {
+				exutil.DumpBuildLogs("sample-build", oc)
+			}
 			o.Expect(err).NotTo(o.HaveOccurred())
-			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
 		})
 
 		g.It("should accept --from-repo with --commit as input", func() {
@@ -139,9 +149,11 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
 			o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
 			o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))
 
-			build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
+			err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
+			if err != nil {
+				exutil.DumpBuildLogs("sample-build", oc)
+			}
 			o.Expect(err).NotTo(o.HaveOccurred())
-			o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
 		})
 	})
 

test/extended/images/sample_repos.go

@@ -4,8 +4,6 @@ import (
 	"fmt"
 	"time"
 
-	"k8s.io/kubernetes/test/e2e"
-
 	g "github.com/onsi/ginkgo"
 	o "github.com/onsi/gomega"
 
@@ -50,9 +48,9 @@ func NewSampleRepoTest(c SampleRepoConfig) func() {
 				g.By("expecting the build is in the Complete phase")
 				err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), buildName, exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
 				if err != nil {
-					logs, _ := oc.Run("build-logs").Args(buildName).Output()
-					e2e.Failf("build failed: %s", logs)
+					exutil.DumpBuildLogs(c.buildConfigName, oc)
 				}
+				o.Expect(err).NotTo(o.HaveOccurred())
 
 				g.By("expecting the deployment to be complete")
 				err = exutil.WaitForADeploymentToComplete(oc.KubeREST().ReplicationControllers(oc.Namespace()), c.deploymentConfigName)

test/extended/setup.sh

@@ -97,7 +97,9 @@ function os::test::extended::setup {
     # Similar to above check, if the XFS volume dir mount point exists enable
     # local storage quota in node-config.yaml so these tests can pass:
     if [ -d "/mnt/openshift-xfs-vol-dir" ]; then
-      sed -i 's/perFSGroup: null/perFSGroup: 256Mi/' $NODE_CONFIG_DIR/node-config.yaml
+	# The ec2 images have have 1Gi of space defined; want to give /registry a good chunk of that
+	# to store the images created when the extended tests run
+      sed -i 's/perFSGroup: null/perFSGroup: 896Mi/' $NODE_CONFIG_DIR/node-config.yaml
     fi
     echo "[INFO] Using VOLUME_DIR=${VOLUME_DIR}"
 

test/extended/util/framework.go

@@ -1,6 +1,7 @@
 package util
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -48,25 +49,58 @@ func DumpBuildLogs(bc string, oc *CLI) {
 		fmt.Fprintf(g.GinkgoWriter, "\n\n  got error on bld logs %v\n\n", err)
 	}
 
-	ExamineDiskUsage()
+	// if we suspect that we are filling up the registry file syste, call ExamineDiskUsage / ExaminePodDiskUsage
+	// also see if manipulations of the quota around /mnt/openshift-xfs-vol-dir exist in the extended test set up scripts
+	//ExamineDiskUsage()
+	//ExaminePodDiskUsage(oc)
 }
 
-// ExamineDiskUsage will dump some df/du output; leveraging this as part of diagnosing
+// ExamineDiskUsage will dump df output on the testing system; leveraging this as part of diagnosing
 // the registry's disk filling up during external tests on jenkins
 func ExamineDiskUsage() {
-	out, err := exec.Command("/bin/df", "-k").Output()
+	out, err := exec.Command("/bin/df", "-m").Output()
 	if err == nil {
-		fmt.Fprintf(g.GinkgoWriter, "\n\n df -k output: %s\n\n", string(out))
+		fmt.Fprintf(g.GinkgoWriter, "\n\n df -m output: %s\n\n", string(out))
 	} else {
 		fmt.Fprintf(g.GinkgoWriter, "\n\n got error on df %v\n\n", err)
 	}
-	if _, err := os.Stat("/registry"); err == nil {
-		out, err = exec.Command("/bin/du", "-a", "/registry").Output()
+}
+
+// ExaminePodDiskUsage will dump df/du output on registry pod; leveraging this as part of diagnosing
+// the registry's disk filling up during external tests on jenkins
+func ExaminePodDiskUsage(oc *CLI) {
+	out, err := oc.Run("get").Args("pods", "-o", "json", "-n", "default", "--config", KubeConfigPath()).Output()
+	var podName string
+	if err == nil {
+		b := []byte(out)
+		var list kapi.PodList
+		err = json.Unmarshal(b, &list)
 		if err == nil {
-			fmt.Fprintf(g.GinkgoWriter, "\n\n du -a /registry output: %s\n\n", string(out))
+			for _, pod := range list.Items {
+				fmt.Fprintf(g.GinkgoWriter, "\n\n looking at pod %s \n\n", pod.ObjectMeta.Name)
+				if strings.Contains(pod.ObjectMeta.Name, "docker-registry-") && !strings.Contains(pod.ObjectMeta.Name, "deploy") {
+					podName = pod.ObjectMeta.Name
+					break
+				}
+			}
 		} else {
-			fmt.Fprintf(g.GinkgoWriter, "\n\n got error on df %v\n\n", err)
+			fmt.Fprintf(g.GinkgoWriter, "\n\n got json unmarshal err: %v\n\n", err)
 		}
+	} else {
+		fmt.Fprintf(g.GinkgoWriter, "\n\n  got error on get pods: %v\n\n", err)
+	}
+
+	out, err = oc.Run("exec").Args("-n", "default", podName, "df", "--config", KubeConfigPath()).Output()
+	if err == nil {
+		fmt.Fprintf(g.GinkgoWriter, "\n\n df from registry pod: \n%s\n\n", out)
+	} else {
+		fmt.Fprintf(g.GinkgoWriter, "\n\n got error on reg pod df: %v\n", err)
+	}
+	out, err = oc.Run("exec").Args("-n", "default", podName, "du", "/registry", "--config", KubeConfigPath()).Output()
+	if err == nil {
+		fmt.Fprintf(g.GinkgoWriter, "\n\n du from registry pod: \n%s\n\n", out)
+	} else {
+		fmt.Fprintf(g.GinkgoWriter, "\n\n got error on reg pod du: %v\n", err)
 	}
 }