From 8a2c819aba258ef84d1a8e64d96959458e53db70 Mon Sep 17 00:00:00 2001 From: Thuan Vo Date: Fri, 8 Mar 2024 15:09:16 -0800 Subject: [PATCH] test(scorecard): add container logs to scorecard results --- .../rbac/scorecard_role.yaml | 7 + internal/test/scorecard/common_utils.go | 28 ++++ internal/test/scorecard/logger.go | 135 ++++++++++++++++++ internal/test/scorecard/tests.go | 7 +- 4 files changed, 175 insertions(+), 2 deletions(-) create mode 100644 internal/test/scorecard/logger.go diff --git a/internal/images/custom-scorecard-tests/rbac/scorecard_role.yaml b/internal/images/custom-scorecard-tests/rbac/scorecard_role.yaml index d350e6464..7eaedd854 100644 --- a/internal/images/custom-scorecard-tests/rbac/scorecard_role.yaml +++ b/internal/images/custom-scorecard-tests/rbac/scorecard_role.yaml @@ -102,6 +102,13 @@ rules: - statefulsets verbs: - get +# Permissions to retrieve container logs +- apiGroups: + - "" + resources: + - pods/log + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRole diff --git a/internal/test/scorecard/common_utils.go b/internal/test/scorecard/common_utils.go index c36dc720a..20afcc612 100644 --- a/internal/test/scorecard/common_utils.go +++ b/internal/test/scorecard/common_utils.go @@ -30,8 +30,10 @@ import ( netv1 "k8s.io/api/networking/v1" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/scheme" ) @@ -464,3 +466,29 @@ func cleanupCryostat(r *scapiv1alpha3.TestResult, client *CryostatClientset, nam r.Log += fmt.Sprintf("failed to delete Cryostat: %s\n", err.Error()) } } + +func getCryostatPodNameForCR(clientset *kubernetes.Clientset, cr *operatorv1beta1.Cryostat) (string, error) { + selector := metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": cr.Name, + "component": "cryostat", + }, + } + opts := metav1.ListOptions{ + LabelSelector: labels.Set(selector.MatchLabels).String(), + } + + ctx, cancel := context.WithTimeout(context.TODO(), testTimeout) + defer cancel() + + pods, err := clientset.CoreV1().Pods(cr.Namespace).List(ctx, opts) + if err != nil { + return "", err + } + + if len(pods.Items) == 0 { + return "", fmt.Errorf("no matching cryostat pods for cr: %s", cr.Name) + } + + return pods.Items[0].ObjectMeta.Name, nil +} diff --git a/internal/test/scorecard/logger.go b/internal/test/scorecard/logger.go new file mode 100644 index 000000000..105da33b5 --- /dev/null +++ b/internal/test/scorecard/logger.go @@ -0,0 +1,135 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorecard + +import ( + "context" + "fmt" + "io" + "strings" + + operatorv1beta1 "github.com/cryostatio/cryostat-operator/api/v1beta1" + scapiv1alpha3 "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3" + v1 "k8s.io/api/core/v1" + "k8s.io/client-go/kubernetes" +) + +type ContainerLog struct { + Container string + Log string +} + +func LogCryostatContainer(clientset *kubernetes.Clientset, cr *operatorv1beta1.Cryostat, ch chan *ContainerLog) { + containerLog := &ContainerLog{ + Container: "cryostat", + } + buf := &strings.Builder{} + podName, err := getCryostatPodNameForCR(clientset, cr) + if err != nil { + buf.WriteString(fmt.Sprintf("failed to get pod name: %s", err.Error())) + } else { + err := LogContainer(clientset, cr.Namespace, podName, cr.Name, buf) + if err != nil { + buf.WriteString(err.Error()) + } + } + + containerLog.Log = buf.String() + ch <- containerLog +} + +func LogGrafanaContainer(clientset *kubernetes.Clientset, cr *operatorv1beta1.Cryostat, ch chan *ContainerLog) { + containerLog := &ContainerLog{ + Container: "grafana", + } + buf := &strings.Builder{} + podName, err := getCryostatPodNameForCR(clientset, cr) + if err != nil { + buf.WriteString(fmt.Sprintf("failed to get pod name: %s", err.Error())) + } else { + err := LogContainer(clientset, cr.Namespace, podName, cr.Name+"-grafana", buf) + if err != nil { + buf.WriteString(err.Error()) + } + } + + containerLog.Log = buf.String() + ch <- containerLog +} + +func LogDatasourceContainer(clientset *kubernetes.Clientset, cr *operatorv1beta1.Cryostat, ch chan *ContainerLog) { + containerLog := &ContainerLog{ + Container: "jfr-datasource", + } + buf := &strings.Builder{} + podName, err := getCryostatPodNameForCR(clientset, cr) + if err != nil { + buf.WriteString(fmt.Sprintf("failed to get pod name: %s", err.Error())) + } else { + err := LogContainer(clientset, cr.Namespace, podName, cr.Name+"-jfr-datasource", buf) + if err != nil { + buf.WriteString(err.Error()) + } + } + + containerLog.Log = buf.String() + ch <- containerLog +} + +func LogContainer(clientset *kubernetes.Clientset, namespace, podName, containerName string, dest io.Writer) error { + ctx, cancel := context.WithTimeout(context.TODO(), testTimeout) + defer cancel() + + logOptions := &v1.PodLogOptions{ + Follow: true, + Container: containerName, + } + stream, err := clientset.CoreV1().Pods(namespace).GetLogs(podName, logOptions).Stream(ctx) + if err != nil { + return fmt.Errorf("failed to get logs for container %s in pod %s: %s", containerName, podName, err.Error()) + } + defer stream.Close() + + _, err = io.Copy(dest, stream) + if err != nil { + return fmt.Errorf("failed to store logs for container %s in pod %s: %s", containerName, podName, err.Error()) + } + return nil +} + +func CollectLogs(ch chan *ContainerLog) []*ContainerLog { + logs := make([]*ContainerLog, 0) + for i := 0; i < cap(ch); i++ { + logs = append(logs, <-ch) + } + return logs +} + +func CollectContainersLogsToResult(result *scapiv1alpha3.TestResult, ch chan *ContainerLog) { + logs := CollectLogs(ch) + for _, log := range logs { + if log != nil { + result.Log += fmt.Sprintf("%s CONTAINER LOG:\n\n\t%s\n", strings.ToUpper(log.Container), log.Log) + } + } +} + +func StartLogs(clientset *kubernetes.Clientset, cr *operatorv1beta1.Cryostat) chan *ContainerLog { + ch := make(chan *ContainerLog, 3) + go LogCryostatContainer(clientset, cr, ch) + go LogGrafanaContainer(clientset, cr, ch) + go LogDatasourceContainer(clientset, cr, ch) + return ch +} diff --git a/internal/test/scorecard/tests.go b/internal/test/scorecard/tests.go index 72bb48f1a..92b783b3a 100644 --- a/internal/test/scorecard/tests.go +++ b/internal/test/scorecard/tests.go @@ -126,7 +126,7 @@ func CryostatConfigChangeTest(bundle *apimanifests.Bundle, namespace string, ope if err != nil { return fail(*r, fmt.Sprintf("Cryostat redeployment did not become available: %s", err.Error())) } - r.Log += "Cryostat deployment has successfully updated with new spec template" + r.Log += "Cryostat deployment has successfully updated with new spec template\n" base, err := url.Parse(cr.Status.ApplicationURL) if err != nil { @@ -142,7 +142,7 @@ func CryostatConfigChangeTest(bundle *apimanifests.Bundle, namespace string, ope } // TODO add a built in discovery test too -func CryostatRecordingTest(bundle *apimanifests.Bundle, namespace string, openShiftCertManager bool) scapiv1alpha3.TestResult { +func CryostatRecordingTest(bundle *apimanifests.Bundle, namespace string, openShiftCertManager bool) (result scapiv1alpha3.TestResult) { tr := newTestResources(CryostatRecordingTestName) r := tr.TestResult @@ -156,6 +156,9 @@ func CryostatRecordingTest(bundle *apimanifests.Bundle, namespace string, openSh if err != nil { return fail(*r, fmt.Sprintf("failed to determine application URL: %s", err.Error())) } + ch := StartLogs(tr.Client.Clientset, cr) + defer CollectContainersLogsToResult(&result, ch) + defer cleanupCryostat(r, tr.Client, CryostatRecordingTestName, namespace) base, err := url.Parse(cr.Status.ApplicationURL)