Merge pull request #67 from mfojtik/analyze-e2e

analyze-e2e: add openshift-dev only tool to help with e2e-aws artifacts analysis

cmd/openshift-dev-helpers/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"os"
 
+	"github.com/openshift/must-gather/pkg/cmd/analyze-e2e"
 	"github.com/openshift/must-gather/pkg/cmd/audit"
 	"github.com/openshift/must-gather/pkg/cmd/certinspection"
 
@@ -52,6 +53,7 @@ func NewCmdDevHelpers(streams genericclioptions.IOStreams) *cobra.Command {
 	cmd.AddCommand(audit.NewCmdAudit("openshift-dev-helpers", streams))
 	cmd.AddCommand(mustgather.NewCmdRevisionStatus("openshift-dev-helpers", streams))
 	cmd.AddCommand(certinspection.NewCmdCertInspection(streams))
+	cmd.AddCommand(analyze_e2e.NewCmdAnalyze("openshift-dev-helpers", streams))
 
 	return cmd
 }

pkg/cmd/analyze-e2e/analyze-e2e.go

@@ -0,0 +1,96 @@
+package analyze_e2e
+
+import (
+	"fmt"
+	"sort"
+
+	"github.com/spf13/cobra"
+
+	"k8s.io/cli-runtime/pkg/genericclioptions"
+
+	"github.com/openshift/must-gather/pkg/cmd/analyze-e2e/analyzers"
+)
+
+var artifactsToAnalyzeList = map[string]Analyzer{
+	"clusteroperators.json": &analyzers.ClusterOperatorsAnalyzer{},
+	"pods.json":             &analyzers.PodsAnalyzer{},
+}
+
+var (
+	analyzeExample = `
+	# print out result of analysis for given artifacts directory in e2e run
+	openshift-dev-helpers analyze-e2e https://gcsweb-ci.svc.ci.openshift.org/gcs/origin-ci-test/pr-logs/pull/openshift_cluster-kube-apiserver-operator/310/pull-ci-openshift-cluster-kube-apiserver-operator-master-e2e-aws/1559/artifacts/e2e-aws
+`
+)
+
+type AnalyzeOptions struct {
+	artifactsBaseURL string
+
+	genericclioptions.IOStreams
+}
+
+func NewAnalyzeOptions(streams genericclioptions.IOStreams) *AnalyzeOptions {
+	return &AnalyzeOptions{
+		IOStreams: streams,
+	}
+}
+
+func NewCmdAnalyze(parentName string, streams genericclioptions.IOStreams) *cobra.Command {
+	o := NewAnalyzeOptions(streams)
+
+	cmd := &cobra.Command{
+		Use:          "analyze-e2e URL [flags]",
+		Short:        "Inspects the artifacts gathered during e2e-aws run and analyze them.",
+		Example:      fmt.Sprintf(analyzeExample, parentName),
+		SilenceUsage: true,
+		RunE: func(c *cobra.Command, args []string) error {
+			if err := o.Complete(c, args); err != nil {
+				return err
+			}
+			if err := o.Validate(); err != nil {
+				return err
+			}
+			if err := o.Run(); err != nil {
+				return err
+			}
+
+			return nil
+		},
+	}
+
+	return cmd
+}
+
+func (o *AnalyzeOptions) Complete(command *cobra.Command, args []string) error {
+	if len(args) >= 1 {
+		o.artifactsBaseURL = args[0]
+	}
+	return nil
+}
+
+func (o *AnalyzeOptions) Validate() error {
+	if len(o.artifactsBaseURL) == 0 {
+		return fmt.Errorf("the URL to e2e-aws artifacts must be specified")
+	}
+	return nil
+}
+
+func (o *AnalyzeOptions) Run() error {
+	results, err := GetArtifacts(o.artifactsBaseURL)
+	if err != nil {
+		return err
+	}
+
+	sort.Slice(results, func(i, j int) bool { return results[i].ArtifactName < results[j].ArtifactName })
+
+	for _, result := range results {
+		fmt.Fprintf(o.Out, "\n%s:\n\n", result.ArtifactName)
+		if result.Error != nil {
+			fmt.Fprintf(o.Out, "ERROR: %v\n", result.Error)
+			continue
+		}
+		fmt.Fprintf(o.Out, "%s", result.Output)
+	}
+	fmt.Fprintf(o.Out, "\n%d files analyzed\n", len(results))
+	return nil
+}

pkg/cmd/analyze-e2e/analyzers/clusteroperators_analyzer.go

@@ -0,0 +1,52 @@
+package analyzers
+
+import (
+	"bytes"
+	"fmt"
+	"sort"
+	"strings"
+	"text/tabwriter"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+type ClusterOperatorsAnalyzer struct{}
+
+func (*ClusterOperatorsAnalyzer) Analyze(content []byte) (string, error) {
+	manifestObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, content)
+	if err != nil {
+		return "", err
+	}
+	manifestUnstructured := manifestObj.(*unstructured.UnstructuredList)
+
+	writer := &bytes.Buffer{}
+	w := tabwriter.NewWriter(writer, 60, 0, 0, ' ', tabwriter.DiscardEmptyColumns)
+
+	err = manifestUnstructured.EachListItem(func(object runtime.Object) error {
+		u := object.(*unstructured.Unstructured)
+		conditions, _, err := unstructured.NestedSlice(u.Object, "status", "conditions")
+		if err != nil {
+			return err
+		}
+		resultConditions := []string{}
+		for _, condition := range conditions {
+			condType, _, err := unstructured.NestedString(condition.(map[string]interface{}), "type")
+			if err != nil {
+				return err
+			}
+			condStatus, _, err := unstructured.NestedString(condition.(map[string]interface{}), "status")
+			if err != nil {
+				return err
+			}
+			resultConditions = append(resultConditions, fmt.Sprintf("%s=%s", condType, condStatus))
+		}
+		sort.Strings(resultConditions)
+		fmt.Fprintf(w, "%s\t%s\n", u.GetName(), strings.Join(resultConditions, ", "))
+		return nil
+	})
+
+	w.Flush()
+
+	return writer.String(), err
+}

pkg/cmd/analyze-e2e/analyzers/pods_analyzer.go

@@ -0,0 +1,81 @@
+package analyzers
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+	"text/tabwriter"
+
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+type PodsAnalyzer struct{}
+
+func (*PodsAnalyzer) Analyze(content []byte) (string, error) {
+	manifestObj, err := runtime.Decode(unstructured.UnstructuredJSONScheme, content)
+	if err != nil {
+		return "", err
+	}
+	manifestUnstructured := manifestObj.(*unstructured.UnstructuredList)
+
+	writer := &bytes.Buffer{}
+	w := tabwriter.NewWriter(writer, 70, 0, 0, ' ', tabwriter.DiscardEmptyColumns)
+
+	err = manifestUnstructured.EachListItem(func(object runtime.Object) error {
+		u := object.(*unstructured.Unstructured)
+		conditions, _, err := unstructured.NestedSlice(u.Object, "status", "conditions")
+		if err != nil {
+			return err
+		}
+		resultConditions := []string{}
+		for _, condition := range conditions {
+			condType, _, err := unstructured.NestedString(condition.(map[string]interface{}), "type")
+			if err != nil {
+				return err
+			}
+			condStatus, _, err := unstructured.NestedString(condition.(map[string]interface{}), "status")
+			if err != nil {
+				return err
+			}
+			resultConditions = append(resultConditions, fmt.Sprintf("%s=%s", condType, condStatus))
+		}
+
+		resultContainers := []string{}
+		containerStatuses, _, err := unstructured.NestedSlice(u.Object, "status", "containerStatuses")
+		if err != nil {
+			return err
+		}
+		for _, status := range containerStatuses {
+			exitCode, exists, err := unstructured.NestedInt64(status.(map[string]interface{}), "lastState", "terminated", "exitCode")
+			if !exists {
+				continue
+			}
+			if err != nil {
+				return err
+			}
+			if exitCode != 0 {
+				message, _, err := unstructured.NestedString(status.(map[string]interface{}), "lastState", "terminated", "message")
+				if err != nil {
+					return err
+				}
+				containerName, _, err := unstructured.NestedString(status.(map[string]interface{}), "name")
+				if err != nil {
+					return err
+				}
+				resultContainers = append(resultContainers, fmt.Sprintf("  [!] Container %q exited with %d:\n   %s\n", containerName, exitCode, message))
+			}
+		}
+
+		fmt.Fprintf(w, "%s\t%s\n", u.GetName(), strings.Join(resultConditions, ", "))
+
+		if len(resultContainers) > 0 {
+			fmt.Fprintf(w, "%s\n", strings.Join(resultContainers, ", "))
+		}
+		return nil
+	})
+
+	w.Flush()
+
+	return writer.String(), err
+}

pkg/cmd/analyze-e2e/get_artifacts.go

@@ -0,0 +1,52 @@
+package analyze_e2e
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+type Analyzer interface {
+	Analyze(content []byte) (string, error)
+}
+
+type AnalyzeResult struct {
+	ArtifactName string
+	Output       string
+	Error        error
+}
+
+func GetArtifacts(baseURL string) ([]AnalyzeResult, error) {
+	out := []AnalyzeResult{}
+	for name, analyzer := range artifactsToAnalyzeList {
+		tr := &http.Transport{
+			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+		}
+		client := &http.Client{Transport: tr}
+		artifactFileName := getArtifactStorageURL(baseURL, name)
+		response, err := client.Get(artifactFileName)
+		if err != nil {
+			return nil, err
+		}
+		defer func() {
+			if err := response.Body.Close(); err != nil {
+			}
+		}()
+		if response.StatusCode != http.StatusOK {
+			return nil, fmt.Errorf("failed to get %q, HTTP code: %d", artifactFileName, response.StatusCode)
+		}
+		content, err := ioutil.ReadAll(response.Body)
+		if err != nil {
+			return nil, err
+		}
+		result, err := analyzer.Analyze(content)
+		out = append(out, AnalyzeResult{ArtifactName: name, Output: result, Error: err})
+	}
+	return out, nil
+}
+
+func getArtifactStorageURL(baseURL, artifactName string) string {
+	return strings.TrimSuffix(strings.Replace(baseURL, "gcsweb-ci.svc.ci.openshift.org/gcs", "storage.googleapis.com", 1), "/") + "/" + artifactName
+}