Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

more catching of build errors, dump of logs, etc.; add registry pod d… #8591

Merged
merged 1 commit into from
May 7, 2016
Merged

more catching of build errors, dump of logs, etc.; add registry pod d… #8591

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 4 additions & 6 deletions test/extended/builds/docker_pullsecret.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,6 @@ import (
g "github.com/onsi/ginkgo"
o "github.com/onsi/gomega"

"k8s.io/kubernetes/test/e2e"

exutil "github.com/openshift/origin/test/extended/util"
)

Expand Down Expand Up @@ -44,9 +42,9 @@ var _ = g.Describe("[builds][pullsecret][Conformance] docker build using a pull
g.By("expecting the build succeeds")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "docker-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
logs, _ := oc.Run("build-logs").Args("docker-build-1").Output()
e2e.Failf("build failed: %s", logs)
exutil.DumpBuildLogs("docker-build", oc)
}
o.Expect(err).NotTo(o.HaveOccurred())

g.By("starting a second build that pulls the image from the first build")
_, err = oc.Run("start-build").Args("docker-build-pull").Output()
Expand All @@ -55,9 +53,9 @@ var _ = g.Describe("[builds][pullsecret][Conformance] docker build using a pull
g.By("expecting the build succeeds")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "docker-build-pull-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
logs, _ := oc.Run("build-logs").Args("docker-build-pull-1").Output()
e2e.Failf("build failed: %s", logs)
exutil.DumpBuildLogs("docker-build-pull", oc)
}
o.Expect(err).NotTo(o.HaveOccurred())
})
})
})
3 changes: 3 additions & 0 deletions test/extended/builds/image_source.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,9 @@ var _ = g.Describe("[builds][Slow] build can have Docker image source", func() {
o.Expect(err).NotTo(o.HaveOccurred())
g.By("expect the builds to complete successfully")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "imagedockerbuild-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
exutil.DumpBuildLogs("imagedockerbuild", oc)
}
o.Expect(err).NotTo(o.HaveOccurred())

g.By("expect the pod to deploy successfully")
Expand Down
10 changes: 4 additions & 6 deletions test/extended/builds/labels.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,6 @@ package builds
import (
"fmt"

"k8s.io/kubernetes/test/e2e"

g "github.com/onsi/ginkgo"
o "github.com/onsi/gomega"

Expand Down Expand Up @@ -46,9 +44,9 @@ var _ = g.Describe("[builds][Slow] result image should have proper labels set",
g.By("o.Expecting the S2I build is in Complete phase")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), buildName, exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
logs, _ := oc.Run("build-logs").Args(buildName).Output()
e2e.Failf("build failed: %s", logs)
exutil.DumpBuildLogs("test", oc)
}
o.Expect(err).NotTo(o.HaveOccurred())

g.By("getting the Docker image reference from ImageStream")
imageRef, err := exutil.GetDockerImageReference(oc.REST().ImageStreams(oc.Namespace()), "test", "latest")
Expand Down Expand Up @@ -82,9 +80,9 @@ var _ = g.Describe("[builds][Slow] result image should have proper labels set",
g.By("o.Expecting the Docker build is in Complete phase")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), buildName, exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
logs, _ := oc.Run("build-logs").Args(buildName).Output()
e2e.Failf("build failed: %s", logs)
exutil.DumpBuildLogs("test", oc)
}
o.Expect(err).NotTo(o.HaveOccurred())

g.By("getting the Docker image reference from ImageStream")
imageRef, err := exutil.GetDockerImageReference(oc.REST().ImageStreams(oc.Namespace()), "test", "latest")
Expand Down
13 changes: 8 additions & 5 deletions test/extended/builds/secrets.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import (
g "github.com/onsi/ginkgo"
o "github.com/onsi/gomega"

buildapi "github.com/openshift/origin/pkg/build/api"
exutil "github.com/openshift/origin/test/extended/util"
kapi "k8s.io/kubernetes/pkg/api"
)
Expand Down Expand Up @@ -49,9 +48,11 @@ var _ = g.Describe("[builds][Slow] can use build secrets", func() {
o.Expect(out).To(o.ContainSubstring("relative-secret3=secret3"))

g.By("checking the status of the build")
build, err := oc.REST().Builds(oc.Namespace()).Get("test-1")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "test-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
exutil.DumpBuildLogs("test", oc)
}
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))

g.By("getting the image name")
image, err := exutil.GetDockerImageReference(oc.REST().ImageStreams(oc.Namespace()), "test", "latest")
Expand Down Expand Up @@ -86,9 +87,11 @@ var _ = g.Describe("[builds][Slow] can use build secrets", func() {
o.Expect(out).To(o.ContainSubstring("relative-secret2=secret2"))

g.By("checking the status of the build")
build, err := oc.REST().Builds(oc.Namespace()).Get("test-1")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "test-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
exutil.DumpBuildLogs("test", oc)
}
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
})

})
Expand Down
44 changes: 28 additions & 16 deletions test/extended/builds/start.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
g "github.com/onsi/ginkgo"
o "github.com/onsi/gomega"

buildapi "github.com/openshift/origin/pkg/build/api"
exutil "github.com/openshift/origin/test/extended/util"
)

Expand All @@ -37,9 +36,11 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
o.Expect(err).NotTo(o.HaveOccurred())

g.By(fmt.Sprintf("verifying the build %q status", out))
build, err := oc.REST().Builds(oc.Namespace()).Get(out)
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
exutil.DumpBuildLogs("sample-build", oc)
}
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
})

g.It("should start a build and wait for the build to fail", func() {
Expand All @@ -64,9 +65,11 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
o.Expect(out).To(o.ContainSubstring("BAR=test"))
o.Expect(out).To(o.ContainSubstring("VAR=test"))
g.By(fmt.Sprintf("verifying the build %q status", out))
build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
exutil.DumpBuildLogs("sample-build", oc)
}
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
})

g.It("should allow to change build log level", func() {
Expand All @@ -76,9 +79,11 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
g.By(fmt.Sprintf("verifying the build output is not verbose"))
o.Expect(out).NotTo(o.ContainSubstring("Creating a new S2I builder"))
g.By(fmt.Sprintf("verifying the build %q status", out))
build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
exutil.DumpBuildLogs("sample-build", oc)
}
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
})
})

Expand All @@ -92,9 +97,10 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))

build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
exutil.DumpBuildLogs("sample-build", oc)
}
})

g.It("should accept --from-dir as input", func() {
Expand All @@ -106,9 +112,11 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))

build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
exutil.DumpBuildLogs("sample-build", oc)
}
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
})

g.It("should accept --from-repo as input", func() {
Expand All @@ -121,9 +129,11 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))

build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
exutil.DumpBuildLogs("sample-build", oc)
}
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
})

g.It("should accept --from-repo with --commit as input", func() {
Expand All @@ -139,9 +149,11 @@ var _ = g.Describe("[builds][Slow] starting a build using CLI", func() {
o.Expect(out).To(o.ContainSubstring("as binary input for the build ..."))
o.Expect(out).To(o.ContainSubstring("Your bundle is complete"))

build, err := oc.REST().Builds(oc.Namespace()).Get("sample-build-1")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), "sample-build-1", exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
exutil.DumpBuildLogs("sample-build", oc)
}
o.Expect(err).NotTo(o.HaveOccurred())
o.Expect(build.Status.Phase).Should(o.BeEquivalentTo(buildapi.BuildPhaseComplete))
})
})

Expand Down
6 changes: 2 additions & 4 deletions test/extended/images/sample_repos.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,6 @@ import (
"fmt"
"time"

"k8s.io/kubernetes/test/e2e"

g "github.com/onsi/ginkgo"
o "github.com/onsi/gomega"

Expand Down Expand Up @@ -50,9 +48,9 @@ func NewSampleRepoTest(c SampleRepoConfig) func() {
g.By("expecting the build is in the Complete phase")
err = exutil.WaitForABuild(oc.REST().Builds(oc.Namespace()), buildName, exutil.CheckBuildSuccessFn, exutil.CheckBuildFailedFn)
if err != nil {
logs, _ := oc.Run("build-logs").Args(buildName).Output()
e2e.Failf("build failed: %s", logs)
exutil.DumpBuildLogs(c.buildConfigName, oc)
}
o.Expect(err).NotTo(o.HaveOccurred())

g.By("expecting the deployment to be complete")
err = exutil.WaitForADeploymentToComplete(oc.KubeREST().ReplicationControllers(oc.Namespace()), c.deploymentConfigName)
Expand Down
4 changes: 3 additions & 1 deletion test/extended/setup.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,9 @@ function os::test::extended::setup {
# Similar to above check, if the XFS volume dir mount point exists enable
# local storage quota in node-config.yaml so these tests can pass:
if [ -d "/mnt/openshift-xfs-vol-dir" ]; then
sed -i 's/perFSGroup: null/perFSGroup: 256Mi/' $NODE_CONFIG_DIR/node-config.yaml
# The ec2 images have have 1Gi of space defined; want to give /registry a good chunk of that
# to store the images created when the extended tests run
sed -i 's/perFSGroup: null/perFSGroup: 896Mi/' $NODE_CONFIG_DIR/node-config.yaml
fi
echo "[INFO] Using VOLUME_DIR=${VOLUME_DIR}"

Expand Down
50 changes: 42 additions & 8 deletions test/extended/util/framework.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package util

import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
Expand Down Expand Up @@ -48,25 +49,58 @@ func DumpBuildLogs(bc string, oc *CLI) {
fmt.Fprintf(g.GinkgoWriter, "\n\n got error on bld logs %v\n\n", err)
}

ExamineDiskUsage()
// if we suspect that we are filling up the registry file syste, call ExamineDiskUsage / ExaminePodDiskUsage
// also see if manipulations of the quota around /mnt/openshift-xfs-vol-dir exist in the extended test set up scripts
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

this would be one case where i'd leave the call in place, commented out.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

add calls commented out

//ExamineDiskUsage()
//ExaminePodDiskUsage(oc)
}

// ExamineDiskUsage will dump some df/du output; leveraging this as part of diagnosing
// ExamineDiskUsage will dump df output on the testing system; leveraging this as part of diagnosing
// the registry's disk filling up during external tests on jenkins
func ExamineDiskUsage() {
out, err := exec.Command("/bin/df", "-k").Output()
out, err := exec.Command("/bin/df", "-m").Output()
if err == nil {
fmt.Fprintf(g.GinkgoWriter, "\n\n df -k output: %s\n\n", string(out))
fmt.Fprintf(g.GinkgoWriter, "\n\n df -m output: %s\n\n", string(out))
} else {
fmt.Fprintf(g.GinkgoWriter, "\n\n got error on df %v\n\n", err)
}
if _, err := os.Stat("/registry"); err == nil {
out, err = exec.Command("/bin/du", "-a", "/registry").Output()
}

// ExaminePodDiskUsage will dump df/du output on registry pod; leveraging this as part of diagnosing
// the registry's disk filling up during external tests on jenkins
func ExaminePodDiskUsage(oc *CLI) {
out, err := oc.Run("get").Args("pods", "-o", "json", "-n", "default", "--config", KubeConfigPath()).Output()
var podName string
if err == nil {
b := []byte(out)
var list kapi.PodList
err = json.Unmarshal(b, &list)
if err == nil {
fmt.Fprintf(g.GinkgoWriter, "\n\n du -a /registry output: %s\n\n", string(out))
for _, pod := range list.Items {
fmt.Fprintf(g.GinkgoWriter, "\n\n looking at pod %s \n\n", pod.ObjectMeta.Name)
if strings.Contains(pod.ObjectMeta.Name, "docker-registry-") && !strings.Contains(pod.ObjectMeta.Name, "deploy") {
podName = pod.ObjectMeta.Name
break
}
}
} else {
fmt.Fprintf(g.GinkgoWriter, "\n\n got error on df %v\n\n", err)
fmt.Fprintf(g.GinkgoWriter, "\n\n got json unmarshal err: %v\n\n", err)
}
} else {
fmt.Fprintf(g.GinkgoWriter, "\n\n got error on get pods: %v\n\n", err)
}

out, err = oc.Run("exec").Args("-n", "default", podName, "df", "--config", KubeConfigPath()).Output()
if err == nil {
fmt.Fprintf(g.GinkgoWriter, "\n\n df from registry pod: \n%s\n\n", out)
} else {
fmt.Fprintf(g.GinkgoWriter, "\n\n got error on reg pod df: %v\n", err)
}
out, err = oc.Run("exec").Args("-n", "default", podName, "du", "/registry", "--config", KubeConfigPath()).Output()
if err == nil {
fmt.Fprintf(g.GinkgoWriter, "\n\n du from registry pod: \n%s\n\n", out)
} else {
fmt.Fprintf(g.GinkgoWriter, "\n\n got error on reg pod du: %v\n", err)
}
}

Expand Down