test(scorecard): add container logs to scorecard results

internal/test/scorecard/common_utils.go

@@ -30,8 +30,10 @@ import (
 	netv1 "k8s.io/api/networking/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
 )
 
@@ -464,3 +466,29 @@ func cleanupCryostat(r *scapiv1alpha3.TestResult, client *CryostatClientset, nam
 		r.Log += fmt.Sprintf("failed to delete Cryostat: %s\n", err.Error())
 	}
 }
+
+func getCryostatPodNameForCR(clientset *kubernetes.Clientset, cr *operatorv1beta1.Cryostat) (string, error) {
+	selector := metav1.LabelSelector{
+		MatchLabels: map[string]string{
+			"app":       cr.Name,
+			"component": "cryostat",
+		},
+	}
+	opts := metav1.ListOptions{
+		LabelSelector: labels.Set(selector.MatchLabels).String(),
+	}
+
+	ctx, cancel := context.WithTimeout(context.TODO(), testTimeout)
+	defer cancel()
+
+	pods, err := clientset.CoreV1().Pods(cr.Namespace).List(ctx, opts)
+	if err != nil {
+		return "", err
+	}
+
+	if len(pods.Items) == 0 {
+		return "", fmt.Errorf("no matching cryostat pods for cr: %s", cr.Name)
+	}
+
+	return pods.Items[0].ObjectMeta.Name, nil
+}

internal/test/scorecard/logger.go

@@ -0,0 +1,136 @@
+// Copyright The Cryostat Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package scorecard
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"strings"
+
+	operatorv1beta1 "github.com/cryostatio/cryostat-operator/api/v1beta1"
+	scapiv1alpha3 "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3"
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/client-go/kubernetes"
+)
+
+type ContainerLog struct {
+	ContainerName string
+	Log           string
+}
+
+func LogCryostatContainer(clientset *kubernetes.Clientset, cr *operatorv1beta1.Cryostat, ch chan *ContainerLog) {
+	containerLog := &ContainerLog{
+		ContainerName: cr.Name,
+	}
+	buf := &strings.Builder{}
+	podName, err := getCryostatPodNameForCR(clientset, cr)
+	if err != nil {
+		buf.WriteString(fmt.Sprintf("failed to get pod name: %s", err.Error()))
+	} else {
+		err := LogContainer(clientset, cr.Namespace, podName, cr.Name, buf)
+		if err != nil {
+			buf.WriteString(err.Error())
+		}
+	}
+
+	containerLog.Log = buf.String()
+	ch <- containerLog
+}
+
+func LogGrafanaContainer(clientset *kubernetes.Clientset, cr *operatorv1beta1.Cryostat, ch chan *ContainerLog) {
+	containerLog := &ContainerLog{
+		ContainerName: cr.Name,
+	}
+	buf := &strings.Builder{}
+	podName, err := getCryostatPodNameForCR(clientset, cr)
+	if err != nil {
+		buf.WriteString(fmt.Sprintf("failed to get pod name: %s", err.Error()))
+	} else {
+		err := LogContainer(clientset, cr.Namespace, podName, cr.Name+"-grafana", buf)
+		if err != nil {
+			buf.WriteString(err.Error())
+		}
+	}
+
+	containerLog.Log = buf.String()
+	ch <- containerLog
+}
+
+func LogDatasourceContainer(clientset *kubernetes.Clientset, cr *operatorv1beta1.Cryostat, ch chan *ContainerLog) {
+	containerLog := &ContainerLog{
+		ContainerName: cr.Name,
+	}
+	buf := &strings.Builder{}
+	podName, err := getCryostatPodNameForCR(clientset, cr)
+	if err != nil {
+		buf.WriteString(fmt.Sprintf("failed to get pod name: %s", err.Error()))
+	} else {
+		err := LogContainer(clientset, cr.Namespace, podName, cr.Name+"-jfr-datasource", buf)
+		if err != nil {
+			buf.WriteString(err.Error())
+		}
+	}
+
+	containerLog.Log = buf.String()
+	ch <- containerLog
+}
+
+func LogContainer(clientset *kubernetes.Clientset, namespace, podName, containerName string, dest io.Writer) error {
+	ctx, cancel := context.WithTimeout(context.TODO(), testTimeout)
+	defer cancel()
+
+	logOptions := &v1.PodLogOptions{
+		Follow:    true,
+		Container: containerName,
+	}
+	stream, err := clientset.CoreV1().Pods(namespace).GetLogs(podName, logOptions).Stream(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to get logs for container %s in pod %s: %s", containerName, podName, err.Error())
+	}
+	defer stream.Close()
+
+	_, err = io.Copy(dest, stream)
+	if err != nil {
+		return fmt.Errorf("failed to store logs for container %s in pod %s: %s", containerName, podName, err.Error())
+	}
+	return nil
+}
+
+func CollectLogs(ch chan *ContainerLog) []*ContainerLog {
+	logs := make([]*ContainerLog, 0)
+	for i := 0; i < cap(ch); i++ {
+		log := <-ch
+		logs = append(logs, log)
+	}
+	return logs
+}
+
+func CollectContainersLogsToResult(result *scapiv1alpha3.TestResult, ch chan *ContainerLog) {
+	logs := CollectLogs(ch)
+	for _, log := range logs {
+		if log != nil {
+			result.Log += fmt.Sprintf("%s\n\t%s\n", strings.ToUpper(log.ContainerName), log.Log)
+		}
+	}
+}
+
+func StartLogs(clientset *kubernetes.Clientset, cr *operatorv1beta1.Cryostat) chan *ContainerLog {
+	ch := make(chan *ContainerLog, 3)
+	go LogCryostatContainer(clientset, cr, ch)
+	go LogGrafanaContainer(clientset, cr, ch)
+	go LogDatasourceContainer(clientset, cr, ch)
+	return ch
+}

internal/test/scorecard/tests.go

@@ -126,7 +126,7 @@ func CryostatConfigChangeTest(bundle *apimanifests.Bundle, namespace string, ope
 	if err != nil {
 		return fail(*r, fmt.Sprintf("Cryostat redeployment did not become available: %s", err.Error()))
 	}
-	r.Log += "Cryostat deployment has successfully updated with new spec template"
+	r.Log += "Cryostat deployment has successfully updated with new spec template\n"
 
 	base, err := url.Parse(cr.Status.ApplicationURL)
 	if err != nil {
@@ -156,6 +156,9 @@ func CryostatRecordingTest(bundle *apimanifests.Bundle, namespace string, openSh
 	if err != nil {
 		return fail(*r, fmt.Sprintf("failed to determine application URL: %s", err.Error()))
 	}
+	ch := StartLogs(tr.Client.Clientset, cr)
+	defer CollectContainersLogsToResult(tr.TestResult, ch)
+
 	defer cleanupCryostat(r, tr.Client, CryostatRecordingTestName, namespace)
 
 	base, err := url.Parse(cr.Status.ApplicationURL)