From a4df4b9e917b44454c9caaabc2cd327dea34966d Mon Sep 17 00:00:00 2001 From: Ming Wang Date: Thu, 29 Feb 2024 15:53:38 -0500 Subject: [PATCH] reformat --- internal/test/scorecard/clients.go | 42 +++ internal/test/scorecard/common_utils.go | 451 +++++++++++++++++++++++ internal/test/scorecard/openshift.go | 7 + internal/test/scorecard/tests.go | 453 ++---------------------- internal/test/scorecard/types.go | 44 +++ 5 files changed, 569 insertions(+), 428 deletions(-) create mode 100644 internal/test/scorecard/common_utils.go create mode 100644 internal/test/scorecard/types.go diff --git a/internal/test/scorecard/clients.go b/internal/test/scorecard/clients.go index a93fbbc66..8343661d7 100644 --- a/internal/test/scorecard/clients.go +++ b/internal/test/scorecard/clients.go @@ -16,7 +16,9 @@ package scorecard import ( "context" + "encoding/base64" "encoding/json" + "fmt" "io" "net/http" @@ -178,3 +180,43 @@ func ReadJSON(resp *http.Response, result interface{}) error { } return nil } + +func ReadString(resp *http.Response) (string, error) { + body, err := io.ReadAll(resp.Body) + if err != nil { + return "", err + } + return string(body), nil +} + +func ReadError(resp *http.Response) string { + body, _ := ReadString(resp) + return body +} + +func NewHttpClient() *http.Client { + client := &http.Client{ + Timeout: testTimeout, + } + + transport := http.DefaultTransport.(*http.Transport).Clone() + // Ignore verifying certs + transport.TLSClientConfig.InsecureSkipVerify = true + + client.Transport = transport + return client +} + +func NewHttpRequest(ctx context.Context, method string, url string, body io.Reader) (*http.Request, error) { + req, err := http.NewRequestWithContext(ctx, method, url, body) + if err != nil { + return nil, err + } + // Authentication is only enabled on OCP. Ignored on k8s. + config, err := rest.InClusterConfig() + if err != nil { + return nil, fmt.Errorf("failed to get in-cluster configurations: %s", err.Error()) + } + req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", base64.StdEncoding.EncodeToString([]byte(config.BearerToken)))) + return req, nil +} diff --git a/internal/test/scorecard/common_utils.go b/internal/test/scorecard/common_utils.go new file mode 100644 index 000000000..32661d7af --- /dev/null +++ b/internal/test/scorecard/common_utils.go @@ -0,0 +1,451 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorecard + +import ( + "context" + "fmt" + "time" + + operatorv1beta1 "github.com/cryostatio/cryostat-operator/api/v1beta1" + scapiv1alpha3 "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + netv1 "k8s.io/api/networking/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes/scheme" +) + +const ( + operatorDeploymentName string = "cryostat-operator-controller-manager" + testTimeout time.Duration = time.Minute * 10 +) + +type TestResources struct { + OpenShift bool + Client *CryostatClientset + *scapiv1alpha3.TestResult +} + +func waitForDeploymentAvailability(ctx context.Context, client *CryostatClientset, namespace string, + name string, r *scapiv1alpha3.TestResult) error { + err := wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (done bool, err error) { + deploy, err := client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + if kerrors.IsNotFound(err) { + r.Log += fmt.Sprintf("deployment %s is not yet found\n", name) + return false, nil // Retry + } + return false, fmt.Errorf("failed to get deployment: %s", err.Error()) + } + // Check for Available condition + for _, condition := range deploy.Status.Conditions { + if condition.Type == appsv1.DeploymentAvailable && + condition.Status == corev1.ConditionTrue { + r.Log += fmt.Sprintf("deployment %s is available\n", deploy.Name) + return true, nil + } + if condition.Type == appsv1.DeploymentReplicaFailure && + condition.Status == corev1.ConditionTrue { + r.Log += fmt.Sprintf("deployment %s is failing, %s: %s\n", deploy.Name, + condition.Reason, condition.Message) + } + } + r.Log += fmt.Sprintf("deployment %s is not yet available\n", deploy.Name) + return false, nil + }) + if err != nil { + logErr := logWorkloadEvents(r, client, namespace, name) + if logErr != nil { + r.Log += fmt.Sprintf("failed to look up deployment errors: %s\n", logErr.Error()) + } + } + return err +} + +func logError(r *scapiv1alpha3.TestResult, message string) { + r.State = scapiv1alpha3.FailState + r.Errors = append(r.Errors, message) +} + +func fail(r scapiv1alpha3.TestResult, message string) scapiv1alpha3.TestResult { + r.State = scapiv1alpha3.FailState + r.Errors = append(r.Errors, message) + return r +} + +func logWorkloadEvents(r *scapiv1alpha3.TestResult, client *CryostatClientset, namespace string, name string) error { + ctx := context.Background() + deploy, err := client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return err + } + // Log deployment conditions and events + r.Log += fmt.Sprintf("deployment %s conditions:\n", deploy.Name) + for _, condition := range deploy.Status.Conditions { + r.Log += fmt.Sprintf("\t%s == %s, %s: %s\n", condition.Type, + condition.Status, condition.Reason, condition.Message) + } + + r.Log += fmt.Sprintf("deployment %s warning events:\n", deploy.Name) + err = logEvents(r, client, namespace, scheme.Scheme, deploy) + if err != nil { + return err + } + + // Look up replica sets for deployment and log conditions and events + selector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) + if err != nil { + return err + } + replicaSets, err := client.AppsV1().ReplicaSets(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: selector.String(), + }) + if err != nil { + return err + } + for _, rs := range replicaSets.Items { + r.Log += fmt.Sprintf("replica set %s conditions:\n", rs.Name) + for _, condition := range rs.Status.Conditions { + r.Log += fmt.Sprintf("\t%s == %s, %s: %s\n", condition.Type, condition.Status, + condition.Reason, condition.Message) + } + r.Log += fmt.Sprintf("replica set %s warning events:\n", rs.Name) + err = logEvents(r, client, namespace, scheme.Scheme, &rs) + if err != nil { + return err + } + } + + // Look up pods for deployment and log conditions and events + pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ + LabelSelector: selector.String(), + }) + if err != nil { + return err + } + for _, pod := range pods.Items { + r.Log += fmt.Sprintf("pod %s phase: %s\n", pod.Name, pod.Status.Phase) + r.Log += fmt.Sprintf("pod %s conditions:\n", pod.Name) + for _, condition := range pod.Status.Conditions { + r.Log += fmt.Sprintf("\t%s == %s, %s: %s\n", condition.Type, condition.Status, + condition.Reason, condition.Message) + } + r.Log += fmt.Sprintf("pod %s warning events:\n", pod.Name) + err = logEvents(r, client, namespace, scheme.Scheme, &pod) + if err != nil { + return err + } + } + return nil +} + +func logEvents(r *scapiv1alpha3.TestResult, client *CryostatClientset, namespace string, + scheme *runtime.Scheme, obj runtime.Object) error { + events, err := client.CoreV1().Events(namespace).Search(scheme, obj) + if err != nil { + return err + } + for _, event := range events.Items { + if event.Type == corev1.EventTypeWarning { + r.Log += fmt.Sprintf("\t%s: %s\n", event.Reason, event.Message) + } + } + return nil +} + +func newEmptyTestResult(testName string) *scapiv1alpha3.TestResult { + return &scapiv1alpha3.TestResult{ + Name: testName, + State: scapiv1alpha3.PassState, + Errors: make([]string, 0), + Suggestions: make([]string, 0), + } +} + +func newTestResources(testName string) *TestResources { + return &TestResources{ + TestResult: newEmptyTestResult(testName), + } +} + +func setupCRTestResources(tr *TestResources, openShiftCertManager bool) error { + r := tr.TestResult + + // Create a new Kubernetes REST client for this test + client, err := NewClientset() + if err != nil { + logError(r, fmt.Sprintf("failed to create client: %s", err.Error())) + return err + } + tr.Client = client + + openshift, err := isOpenShift(client) + if err != nil { + logError(r, fmt.Sprintf("could not determine whether platform is OpenShift: %s", err.Error())) + return err + } + tr.OpenShift = openshift + + if openshift && openShiftCertManager { + err := installOpenShiftCertManager(r) + if err != nil { + logError(r, fmt.Sprintf("failed to install cert-manager Operator for Red Hat OpenShift: %s", err.Error())) + return err + } + } + return nil +} + +func newCryostatCR(name string, namespace string, withIngress bool) *operatorv1beta1.Cryostat { + cr := &operatorv1beta1.Cryostat{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + Spec: operatorv1beta1.CryostatSpec{ + Minimal: false, + EnableCertManager: &[]bool{true}[0], + }, + } + + if withIngress { + pathType := netv1.PathTypePrefix + cr.Spec.NetworkOptions = &operatorv1beta1.NetworkConfigurationList{ + CoreConfig: &operatorv1beta1.NetworkConfiguration{ + Annotations: map[string]string{ + "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS", + }, + IngressSpec: &netv1.IngressSpec{ + TLS: []netv1.IngressTLS{{}}, + Rules: []netv1.IngressRule{ + { + Host: "testing.cryostat", + IngressRuleValue: netv1.IngressRuleValue{ + HTTP: &netv1.HTTPIngressRuleValue{ + Paths: []netv1.HTTPIngressPath{ + { + Path: "/", + PathType: &pathType, + Backend: netv1.IngressBackend{ + Service: &netv1.IngressServiceBackend{ + Name: name, + Port: netv1.ServiceBackendPort{ + Number: 8181, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + GrafanaConfig: &operatorv1beta1.NetworkConfiguration{ + Annotations: map[string]string{ + "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS", + }, + IngressSpec: &netv1.IngressSpec{ + TLS: []netv1.IngressTLS{{}}, + Rules: []netv1.IngressRule{ + { + Host: "testing.cryostat-grafana", + IngressRuleValue: netv1.IngressRuleValue{ + HTTP: &netv1.HTTPIngressRuleValue{ + Paths: []netv1.HTTPIngressPath{ + { + Path: "/", + PathType: &pathType, + Backend: netv1.IngressBackend{ + Service: &netv1.IngressServiceBackend{ + Name: fmt.Sprintf("%s-grafana", name), + Port: netv1.ServiceBackendPort{ + Number: 3000, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + } + return cr +} + +func createAndWaitTillCryostatAvailable(cr *operatorv1beta1.Cryostat, resources *TestResources) (*operatorv1beta1.Cryostat, error) { + client := resources.Client + r := resources.TestResult + + cr, err := client.OperatorCRDs().Cryostats(cr.Namespace).Create(context.Background(), cr) + if err != nil { + logError(r, fmt.Sprintf("failed to create Cryostat CR: %s", err.Error())) + return nil, err + } + + // Poll the deployment until it becomes available or we timeout + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + err = waitForDeploymentAvailability(ctx, client, cr.Namespace, cr.Name, r) + if err != nil { + logError(r, fmt.Sprintf("Cryostat main deployment did not become available: %s", err.Error())) + return nil, err + } + + err = wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (done bool, err error) { + cr, err = client.OperatorCRDs().Cryostats(cr.Namespace).Get(ctx, cr.Name) + if err != nil { + return false, fmt.Errorf("failed to get Cryostat CR: %s", err.Error()) + } + if len(cr.Status.ApplicationURL) > 0 { + return true, nil + } + r.Log += "application URL is not yet available\n" + return false, nil + }) + if err != nil { + logError(r, fmt.Sprintf("application URL not found in CR: %s", err.Error())) + return nil, err + } + r.Log += fmt.Sprintf("application is available at %s\n", cr.Status.ApplicationURL) + + return cr, nil +} + +func updateAndWaitTillCryostatAvailable(cr *operatorv1beta1.Cryostat, resources *TestResources) error { + client := resources.Client + r := resources.TestResult + + cr, err := client.OperatorCRDs().Cryostats(cr.Namespace).Update(context.Background(), cr) + if err != nil { + r.Log += fmt.Sprintf("failed to update Cryostat CR: %s", err.Error()) + return err + } + + // Poll the deployment until it becomes available or we timeout + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + err = wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (done bool, err error) { + deploy, err := client.AppsV1().Deployments(cr.Namespace).Get(ctx, cr.Name, metav1.GetOptions{}) + if err != nil { + if kerrors.IsNotFound(err) { + r.Log += fmt.Sprintf("deployment %s is not yet found\n", cr.Name) + return false, nil // Retry + } + return false, fmt.Errorf("failed to get deployment: %s", err.Error()) + } + + // Wait for deployment to update by verifying Cryostat has PVC configured + for _, volume := range deploy.Spec.Template.Spec.Volumes { + if volume.VolumeSource.EmptyDir != nil { + r.Log += fmt.Sprintf("Cryostat deployment is still updating. Storage: %s\n", volume.VolumeSource.EmptyDir) + return false, nil // Retry + } + if volume.VolumeSource.PersistentVolumeClaim != nil { + r.Log += fmt.Sprintf("Cryostat deployment has successfully updated to pvc: %s\n", volume.PersistentVolumeClaim.ClaimName) + return true, nil + } + } + + // Check for deployment condition + if deploy.Generation <= deploy.Status.ObservedGeneration { + for _, condition := range deploy.Status.Conditions { + if condition.Type == appsv1.DeploymentProgressing && condition.Status == corev1.ConditionFalse && condition.Reason == "ProgressDeadlineExceeded" { + return false, fmt.Errorf("deployment %s exceeded its progress deadline", deploy.Name) // Don't Retry + } + } + if deploy.Spec.Replicas != nil && deploy.Status.UpdatedReplicas < *deploy.Spec.Replicas { + r.Log += fmt.Sprintf("Waiting for deployment %s rollout to finish: %d out of %d new replicas have been updated... \n", deploy.Name, deploy.Status.UpdatedReplicas, *deploy.Spec.Replicas) + return false, nil + } + if deploy.Status.Replicas > deploy.Status.UpdatedReplicas { + r.Log += fmt.Sprintf("Waiting for deployment %s rollout to finish: %d old replicas are pending termination.. \n", deploy.Name, deploy.Status.Replicas-deploy.Status.UpdatedReplicas) + return false, nil + } + if deploy.Status.AvailableReplicas < deploy.Status.UpdatedReplicas { + r.Log += fmt.Sprintf("Waiting for deployment %s rollout to finish: %d out of %d updated replicas are available... \n", deploy.Name, deploy.Status.AvailableReplicas, deploy.Status.UpdatedReplicas) + return false, nil + } + r.Log += fmt.Sprintf("deployment %s successfully rolled out\n", deploy.Name) + return true, nil + } + r.Log += "Waiting for deployment spec update to be observed...\n" + return false, nil + }) + if err != nil { + return fmt.Errorf("failed to look up deployment errors: %s", err.Error()) + } + + err = wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (done bool, err error) { + cr, err = client.OperatorCRDs().Cryostats(cr.Namespace).Get(ctx, cr.Name) + if err != nil { + return false, fmt.Errorf("failed to get Cryostat CR: %s", err.Error()) + } + if len(cr.Status.ApplicationURL) > 0 { + return true, nil + } + r.Log += "application URL is not yet available\n" + return false, nil + }) + if err != nil { + return fmt.Errorf("application URL not found in CR: %s", err.Error()) + } + r.Log += fmt.Sprintf("application is available at %s\n", cr.Status.ApplicationURL) + + return err +} + +func cleanupCryostat(r *scapiv1alpha3.TestResult, client *CryostatClientset, name string, namespace string) { + cr := &operatorv1beta1.Cryostat{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + }, + } + ctx := context.Background() + err := client.OperatorCRDs().Cryostats(cr.Namespace).Delete(ctx, + cr.Name, &metav1.DeleteOptions{}) + if err != nil { + r.Log += fmt.Sprintf("failed to delete Cryostat: %s\n", err.Error()) + } +} + +func newPVCSpec(storageClass string, storageRequest string, + accessModes ...corev1.PersistentVolumeAccessMode) *corev1.PersistentVolumeClaimSpec { + return &corev1.PersistentVolumeClaimSpec{ + StorageClassName: &storageClass, + AccessModes: accessModes, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse(storageRequest), + }, + }, + } +} + +func StatusOK(statusCode int) bool { + return statusCode >= 200 && statusCode < 300 +} diff --git a/internal/test/scorecard/openshift.go b/internal/test/scorecard/openshift.go index dee12c4d3..247068a48 100644 --- a/internal/test/scorecard/openshift.go +++ b/internal/test/scorecard/openshift.go @@ -27,12 +27,15 @@ import ( ctrl "sigs.k8s.io/controller-runtime" configv1 "github.com/openshift/api/config/v1" + routev1 "github.com/openshift/api/route/v1" corev1 "k8s.io/api/core/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/discovery" corev1client "k8s.io/client-go/kubernetes/typed/core/v1" ) @@ -202,3 +205,7 @@ func installOpenShiftCertManager(r *scapiv1alpha3.TestResult) error { return false, nil }) } + +func isOpenShift(client discovery.DiscoveryInterface) (bool, error) { + return discovery.IsResourceEnabled(client, routev1.GroupVersion.WithResource("routes")) +} diff --git a/internal/test/scorecard/tests.go b/internal/test/scorecard/tests.go index 92588a701..545628f83 100644 --- a/internal/test/scorecard/tests.go +++ b/internal/test/scorecard/tests.go @@ -17,30 +17,15 @@ package scorecard import ( "context" "fmt" - "time" operatorv1beta1 "github.com/cryostatio/cryostat-operator/api/v1beta1" scapiv1alpha3 "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3" apimanifests "github.com/operator-framework/api/pkg/manifests" - - routev1 "github.com/openshift/api/route/v1" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" - netv1 "k8s.io/api/networking/v1" - kerrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/api/resource" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/util/wait" - "k8s.io/client-go/discovery" - "k8s.io/client-go/kubernetes/scheme" ) const ( - OperatorInstallTestName string = "operator-install" - CryostatCRTestName string = "cryostat-cr" - operatorDeploymentName string = "cryostat-operator-controller-manager" - testTimeout time.Duration = time.Minute * 10 + OperatorInstallTestName string = "operator-install" + CryostatCRTestName string = "cryostat-cr" ) // OperatorInstallTest checks that the operator installed correctly @@ -70,124 +55,53 @@ func OperatorInstallTest(bundle *apimanifests.Bundle, namespace string) scapiv1a // CryostatCRTest checks that the operator installs Cryostat in response to a Cryostat CR func CryostatCRTest(bundle *apimanifests.Bundle, namespace string, openShiftCertManager bool) scapiv1alpha3.TestResult { - r := scapiv1alpha3.TestResult{} - r.Name = CryostatCRTestName - r.State = scapiv1alpha3.PassState - r.Errors = make([]string, 0) - r.Suggestions = make([]string, 0) - - // Create a new Kubernetes REST client for this test - client, err := NewClientset() - if err != nil { - return fail(r, fmt.Sprintf("failed to create client: %s", err.Error())) - } - defer cleanupCryostat(&r, client, namespace) - - openshift, err := isOpenShift(client.DiscoveryClient) - if err != nil { - return fail(r, fmt.Sprintf("could not determine whether platform is OpenShift: %s", err.Error())) - } + tr := newTestResources(CryostatCRTestName) + r := tr.TestResult - if openshift && openShiftCertManager { - err := installOpenShiftCertManager(&r) - if err != nil { - return fail(r, fmt.Sprintf("failed to install cert-manager Operator for Red Hat OpenShift: %s", err.Error())) - } - } - - // Create a default Cryostat CR with default empty dir - cr := newCryostatCR(CryostatCRTestName, namespace, !openshift) - cr.Spec.StorageOptions = &operatorv1beta1.StorageConfiguration{ - EmptyDir: &operatorv1beta1.EmptyDirConfig{ - Enabled: true, - }, - } - - ctx := context.Background() - cr, err = client.OperatorCRDs().Cryostats(namespace).Create(ctx, cr) + err := setupCRTestResources(tr, openShiftCertManager) if err != nil { - return fail(r, fmt.Sprintf("failed to create Cryostat CR: %s", err.Error())) + return fail(*r, fmt.Sprintf("failed to set up %s test: %s", CryostatCRTestName, err.Error())) } - // Wait for deployment to become available - ctx, cancel := context.WithTimeout(context.Background(), testTimeout) - defer cancel() - err = waitForDeploymentAvailability(ctx, client, cr.Namespace, cr.Name, &r) + // Create a default Cryostat CR + _, err = createAndWaitTillCryostatAvailable(newCryostatCR(CryostatCRTestName, namespace, !tr.OpenShift), tr) if err != nil { - return fail(r, fmt.Sprintf("Cryostat main deployment did not become available: %s", err.Error())) + return fail(*r, fmt.Sprintf("%s test failed: %s", CryostatCRTestName, err.Error())) } + defer cleanupCryostat(r, tr.Client, CryostatCRTestName, namespace) - // Poll the deployment until it becomes available or we timeout - err = wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (done bool, err error) { - cr, err = client.OperatorCRDs().Cryostats(namespace).Get(ctx, cr.Name) - if err != nil { - return false, fmt.Errorf("failed to get Cryostat CR: %s", err.Error()) - } - if len(cr.Status.ApplicationURL) > 0 { - return true, nil - } - r.Log += "Application URL is not yet available\n" - return false, nil - }) - if err != nil { - return fail(r, fmt.Sprintf("Application URL not found in CR: %s", err.Error())) - } - r.Log += fmt.Sprintf("Application is ready at %s\n", cr.Status.ApplicationURL) - return r + return *r } func CryostatCRConfigChangeTest(bundle *apimanifests.Bundle, namespace string, openShiftCertManager bool) scapiv1alpha3.TestResult { - r := scapiv1alpha3.TestResult{} - r.Name = CryostatCRTestName - r.State = scapiv1alpha3.PassState - r.Errors = make([]string, 0) - r.Suggestions = make([]string, 0) - - // Create a new Kubernetes REST client for this test - client, err := NewClientset() - if err != nil { - return fail(r, fmt.Sprintf("failed to create client: %s", err.Error())) - } - defer cleanupCryostat(&r, client, namespace) + tr := newTestResources(CryostatCRTestName) + r := tr.TestResult + client := tr.Client - openshift, err := isOpenShift(client.DiscoveryClient) + err := setupCRTestResources(tr, openShiftCertManager) if err != nil { - return fail(r, fmt.Sprintf("could not determine whether platform is OpenShift: %s", err.Error())) - } - - if openshift && openShiftCertManager { - err := installOpenShiftCertManager(&r) - if err != nil { - return fail(r, fmt.Sprintf("failed to install cert-manager Operator for Red Hat OpenShift: %s", err.Error())) - } + return fail(*r, fmt.Sprintf("failed to set up %s test: %s", CryostatCRTestName, err.Error())) } // Create a default Cryostat CR with default empty dir - cr := newCryostatCR(CryostatCRTestName, namespace, !openshift) + cr := newCryostatCR(CryostatCRTestName, namespace, !tr.OpenShift) cr.Spec.StorageOptions = &operatorv1beta1.StorageConfiguration{ EmptyDir: &operatorv1beta1.EmptyDirConfig{ Enabled: true, }, } - ctx := context.Background() - cr, err = client.OperatorCRDs().Cryostats(namespace).Create(ctx, cr) + _, err = createAndWaitTillCryostatAvailable(cr, tr) if err != nil { - return fail(r, fmt.Sprintf("failed to create Cryostat CR: %s", err.Error())) + return fail(*r, fmt.Sprintf("%s test failed: %s", CryostatCRTestName, err.Error())) } - // Wait for deployment to become available + // Switch Cryostat CR to PVC for redeployment ctx, cancel := context.WithTimeout(context.Background(), testTimeout) defer cancel() - err = waitForDeploymentAvailability(ctx, client, cr.Namespace, cr.Name, &r) - if err != nil { - return fail(r, fmt.Sprintf("Cryostat main deployment did not become available: %s", err.Error())) - } - - // Switch Cryostat CR to PVC for redeployment cr, err = client.OperatorCRDs().Cryostats(namespace).Get(ctx, CryostatCRTestName) if err != nil { - return fail(r, fmt.Sprintf("failed to get Cryostat CR: %s", err.Error())) + return fail(*r, fmt.Sprintf("failed to get Cryostat CR: %s", err.Error())) } cr.Spec.StorageOptions = &operatorv1beta1.StorageConfiguration{ PVC: &operatorv1beta1.PersistentVolumeClaimConfig{ @@ -196,328 +110,11 @@ func CryostatCRConfigChangeTest(bundle *apimanifests.Bundle, namespace string, o } // Wait for redeployment of Cryostat CR - err = updateAndWaitTillCryostatAvailable(cr, client, &r) - if err != nil { - return fail(r, fmt.Sprintf("Cryostat redeployment did not become available: %s", err.Error())) - } - - // Poll the deployment until it becomes available or we timeout - err = wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (done bool, err error) { - cr, err = client.OperatorCRDs().Cryostats(namespace).Get(ctx, cr.Name) - if err != nil { - return false, fmt.Errorf("failed to get Cryostat CR: %s", err.Error()) - } - if len(cr.Status.ApplicationURL) > 0 { - return true, nil - } - r.Log += "Application URL is not yet available\n" - return false, nil - }) + err = updateAndWaitTillCryostatAvailable(cr, tr) if err != nil { - return fail(r, fmt.Sprintf("Application URL not found in CR: %s", err.Error())) + return fail(*r, fmt.Sprintf("Cryostat redeployment did not become available: %s", err.Error())) } - r.Log += fmt.Sprintf("Application is ready at %s\n", cr.Status.ApplicationURL) - return r -} - -func waitForDeploymentAvailability(ctx context.Context, client *CryostatClientset, namespace string, - name string, r *scapiv1alpha3.TestResult) error { - err := wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (done bool, err error) { - deploy, err := client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) - if err != nil { - if kerrors.IsNotFound(err) { - r.Log += fmt.Sprintf("deployment %s is not yet found\n", name) - return false, nil // Retry - } - return false, fmt.Errorf("failed to get deployment: %s", err.Error()) - } - // Check for Available condition - for _, condition := range deploy.Status.Conditions { - if condition.Type == appsv1.DeploymentAvailable && - condition.Status == corev1.ConditionTrue { - r.Log += fmt.Sprintf("deployment %s is available\n", deploy.Name) - return true, nil - } - if condition.Type == appsv1.DeploymentReplicaFailure && - condition.Status == corev1.ConditionTrue { - r.Log += fmt.Sprintf("deployment %s is failing, %s: %s\n", deploy.Name, - condition.Reason, condition.Message) - } - } - r.Log += fmt.Sprintf("deployment %s is not yet available\n", deploy.Name) - return false, nil - }) - if err != nil { - logErr := logErrors(r, client, namespace, name) - if logErr != nil { - r.Log += fmt.Sprintf("failed to look up deployment errors: %s\n", logErr.Error()) - } - } - return err -} - -func updateAndWaitTillCryostatAvailable(cr *operatorv1beta1.Cryostat, client *CryostatClientset, r *scapiv1alpha3.TestResult) error { - cr, err := client.OperatorCRDs().Cryostats(cr.Namespace).Update(context.Background(), cr) - if err != nil { - r.Log += fmt.Sprintf("failed to update Cryostat CR: %s", err.Error()) - return err - } - - // Poll the deployment until it becomes available or we timeout - ctx, cancel := context.WithTimeout(context.Background(), testTimeout) - defer cancel() - err = wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (done bool, err error) { - deploy, err := client.AppsV1().Deployments(cr.Namespace).Get(ctx, cr.Name, metav1.GetOptions{}) - if err != nil { - if kerrors.IsNotFound(err) { - r.Log += fmt.Sprintf("deployment %s is not yet found\n", cr.Name) - return false, nil // Retry - } - return false, fmt.Errorf("failed to get deployment: %s", err.Error()) - } - - // Wait for deployment to update by verifying Cryostat has PVC configured - for _, volume := range deploy.Spec.Template.Spec.Volumes { - if volume.VolumeSource.EmptyDir != nil { - r.Log += fmt.Sprintf("Cryostat deployment is still updating. Storage: %s\n", volume.VolumeSource.EmptyDir) - return false, nil // Retry - } - if volume.VolumeSource.PersistentVolumeClaim != nil { - r.Log += fmt.Sprintf("Cryostat deployment has successfully updated to pvc: %s\n", volume.PersistentVolumeClaim.ClaimName) - return true, nil - } - } - - // Check for redeployment condition - if deploy.Generation <= deploy.Status.ObservedGeneration { - for _, condition := range deploy.Status.Conditions { - if condition.Type == appsv1.DeploymentProgressing && condition.Status == corev1.ConditionFalse && condition.Reason == "ProgressDeadlineExceeded" { - return false, fmt.Errorf("deployment %s exceeded its progress deadline", deploy.Name) // Don't Retry - } - } - if deploy.Spec.Replicas != nil && deploy.Status.UpdatedReplicas < *deploy.Spec.Replicas { - r.Log += fmt.Sprintf("Waiting for deployment %s rollout to finish: %d out of %d new replicas have been updated... \n", deploy.Name, deploy.Status.UpdatedReplicas, *deploy.Spec.Replicas) - return false, nil - } - if deploy.Status.Replicas > deploy.Status.UpdatedReplicas { - r.Log += fmt.Sprintf("Waiting for deployment %s rollout to finish: %d old replicas are pending termination.. \n", deploy.Name, deploy.Status.Replicas-deploy.Status.UpdatedReplicas) - return false, nil - } - if deploy.Status.AvailableReplicas < deploy.Status.UpdatedReplicas { - r.Log += fmt.Sprintf("Waiting for deployment %s rollout to finish: %d out of %d updated replicas are available... \n", deploy.Name, deploy.Status.AvailableReplicas, deploy.Status.UpdatedReplicas) - return false, nil - } - r.Log += fmt.Sprintf("deployment %s successfully rolled out\n", deploy.Name) - return true, nil - } - r.Log += fmt.Sprintf("Waiting for redeployment spec update to be observed...\n") - return false, nil - }) - if err != nil { - logErr := logErrors(r, client, cr.Namespace, cr.DeepCopy().Name) - if logErr != nil { - r.Log += fmt.Sprintf("failed to look up redeployment errors: %s\n", logErr.Error()) - } - } - return err -} - -func fail(r scapiv1alpha3.TestResult, message string) scapiv1alpha3.TestResult { - r.State = scapiv1alpha3.FailState - r.Errors = append(r.Errors, message) - return r -} - -func cleanupCryostat(r *scapiv1alpha3.TestResult, client *CryostatClientset, namespace string) { - cr := &operatorv1beta1.Cryostat{ - ObjectMeta: metav1.ObjectMeta{ - Name: "cryostat-cr-test", - Namespace: namespace, - }, - } - ctx := context.Background() - err := client.OperatorCRDs().Cryostats(cr.Namespace).Delete(ctx, - cr.Name, &metav1.DeleteOptions{}) - if err != nil { - r.Log += fmt.Sprintf("failed to delete Cryostat: %s\n", err.Error()) - } -} - -func newPVCSpec(storageClass string, storageRequest string, - accessModes ...corev1.PersistentVolumeAccessMode) *corev1.PersistentVolumeClaimSpec { - return &corev1.PersistentVolumeClaimSpec{ - StorageClassName: &storageClass, - AccessModes: accessModes, - Resources: corev1.ResourceRequirements{ - Requests: corev1.ResourceList{ - corev1.ResourceStorage: resource.MustParse(storageRequest), - }, - }, - } -} - -func logErrors(r *scapiv1alpha3.TestResult, client *CryostatClientset, namespace string, name string) error { - ctx := context.Background() - deploy, err := client.AppsV1().Deployments(namespace).Get(ctx, name, metav1.GetOptions{}) - if err != nil { - return err - } - // Log deployment conditions and events - r.Log += fmt.Sprintf("deployment %s conditions:\n", deploy.Name) - for _, condition := range deploy.Status.Conditions { - r.Log += fmt.Sprintf("\t%s == %s, %s: %s\n", condition.Type, - condition.Status, condition.Reason, condition.Message) - } - - r.Log += fmt.Sprintf("deployment %s warning events:\n", deploy.Name) - err = logEvents(r, client, namespace, scheme.Scheme, deploy) - if err != nil { - return err - } - - // Look up replica sets for deployment and log conditions and events - selector, err := metav1.LabelSelectorAsSelector(deploy.Spec.Selector) - if err != nil { - return err - } - replicaSets, err := client.AppsV1().ReplicaSets(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: selector.String(), - }) - if err != nil { - return err - } - for _, rs := range replicaSets.Items { - r.Log += fmt.Sprintf("replica set %s conditions:\n", rs.Name) - for _, condition := range rs.Status.Conditions { - r.Log += fmt.Sprintf("\t%s == %s, %s: %s\n", condition.Type, condition.Status, - condition.Reason, condition.Message) - } - r.Log += fmt.Sprintf("replica set %s warning events:\n", rs.Name) - err = logEvents(r, client, namespace, scheme.Scheme, &rs) - if err != nil { - return err - } - } - - // Look up pods for deployment and log conditions and events - pods, err := client.CoreV1().Pods(namespace).List(ctx, metav1.ListOptions{ - LabelSelector: selector.String(), - }) - if err != nil { - return err - } - for _, pod := range pods.Items { - r.Log += fmt.Sprintf("pod %s phase: %s\n", pod.Name, pod.Status.Phase) - r.Log += fmt.Sprintf("pod %s conditions:\n", pod.Name) - for _, condition := range pod.Status.Conditions { - r.Log += fmt.Sprintf("\t%s == %s, %s: %s\n", condition.Type, condition.Status, - condition.Reason, condition.Message) - } - r.Log += fmt.Sprintf("pod %s warning events:\n", pod.Name) - err = logEvents(r, client, namespace, scheme.Scheme, &pod) - if err != nil { - return err - } - } - return nil -} - -func logEvents(r *scapiv1alpha3.TestResult, client *CryostatClientset, namespace string, - scheme *runtime.Scheme, obj runtime.Object) error { - events, err := client.CoreV1().Events(namespace).Search(scheme, obj) - if err != nil { - return err - } - for _, event := range events.Items { - if event.Type == corev1.EventTypeWarning { - r.Log += fmt.Sprintf("\t%s: %s\n", event.Reason, event.Message) - } - } - return nil -} - -func newCryostatCR(name string, namespace string, withIngress bool) *operatorv1beta1.Cryostat { - cr := &operatorv1beta1.Cryostat{ - ObjectMeta: metav1.ObjectMeta{ - Name: name, - Namespace: namespace, - }, - Spec: operatorv1beta1.CryostatSpec{ - Minimal: false, - EnableCertManager: &[]bool{true}[0], - }, - } - - if withIngress { - pathType := netv1.PathTypePrefix - cr.Spec.NetworkOptions = &operatorv1beta1.NetworkConfigurationList{ - CoreConfig: &operatorv1beta1.NetworkConfiguration{ - Annotations: map[string]string{ - "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS", - }, - IngressSpec: &netv1.IngressSpec{ - TLS: []netv1.IngressTLS{{}}, - Rules: []netv1.IngressRule{ - { - Host: "testing.cryostat", - IngressRuleValue: netv1.IngressRuleValue{ - HTTP: &netv1.HTTPIngressRuleValue{ - Paths: []netv1.HTTPIngressPath{ - { - Path: "/", - PathType: &pathType, - Backend: netv1.IngressBackend{ - Service: &netv1.IngressServiceBackend{ - Name: name, - Port: netv1.ServiceBackendPort{ - Number: 8181, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - GrafanaConfig: &operatorv1beta1.NetworkConfiguration{ - Annotations: map[string]string{ - "nginx.ingress.kubernetes.io/backend-protocol": "HTTPS", - }, - IngressSpec: &netv1.IngressSpec{ - TLS: []netv1.IngressTLS{{}}, - Rules: []netv1.IngressRule{ - { - Host: "testing.cryostat-grafana", - IngressRuleValue: netv1.IngressRuleValue{ - HTTP: &netv1.HTTPIngressRuleValue{ - Paths: []netv1.HTTPIngressPath{ - { - Path: "/", - PathType: &pathType, - Backend: netv1.IngressBackend{ - Service: &netv1.IngressServiceBackend{ - Name: fmt.Sprintf("%s-grafana", name), - Port: netv1.ServiceBackendPort{ - Number: 3000, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - }, - } - } - return cr -} + defer cleanupCryostat(r, tr.Client, CryostatCRTestName, namespace) -func isOpenShift(client discovery.DiscoveryInterface) (bool, error) { - return discovery.IsResourceEnabled(client, routev1.GroupVersion.WithResource("routes")) + return *r } diff --git a/internal/test/scorecard/types.go b/internal/test/scorecard/types.go new file mode 100644 index 000000000..bca0d6b48 --- /dev/null +++ b/internal/test/scorecard/types.go @@ -0,0 +1,44 @@ +// Copyright The Cryostat Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package scorecard + +import ( + "errors" +) + +type HealthResponse struct { + CryostatVersion string `json:"cryostatVersion"` + DashboardAvailable bool `json:"dashboardAvailable"` + DashboardConfigured bool `json:"dashboardConfigured"` + DataSourceAvailable bool `json:"datasourceAvailable"` + DataSourceConfigured bool `json:"datasourceConfigured"` + ReportsAvailable bool `json:"reportsAvailable"` + ReportsConfigured bool `json:"reportsConfigured"` +} + +func (health *HealthResponse) Ready() error { + if !health.DashboardAvailable { + return errors.New("dashboard is not available") + } + + if !health.DataSourceAvailable { + return errors.New("datasource is not available") + } + + if !health.ReportsAvailable { + return errors.New("report is not available") + } + return nil +}