test(scorecard): scorecard test for Cryostat CR configuration changes (…

…#739)

* CR config scorecard

* reformat

* reviews

* add kubectl license

(cherry picked from commit bf8df15)

# Conflicts:
#	bundle/manifests/cryostat-operator.clusterserviceversion.yaml

bundle/manifests/cryostat-operator.clusterserviceversion.yaml

@@ -54,7 +54,11 @@ metadata:
     capabilities: Seamless Upgrades
     categories: Monitoring, Developer Tools
     containerImage: quay.io/cryostat/cryostat-operator:2.5.0-dev
+<<<<<<< HEAD
     createdAt: "2024-03-06T21:13:39Z"
+=======
+    createdAt: "2024-03-07T15:43:22Z"
+>>>>>>> bf8df15 (test(scorecard): scorecard test for Cryostat CR configuration changes (#739))
     description: JVM monitoring and profiling tool
     operatorframework.io/initialization-resource: |-
       {

bundle/tests/scorecard/config.yaml

@@ -70,7 +70,7 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - operator-install
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240305020416
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307153901
     labels:
       suite: cryostat
       test: operator-install
@@ -80,7 +80,7 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - cryostat-cr
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240305020416
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307153901
     labels:
       suite: cryostat
       test: cryostat-cr
@@ -90,13 +90,23 @@ stages:
   - entrypoint:
     - cryostat-scorecard-tests
     - cryostat-recording
-    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240305020416
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307153901
     labels:
       suite: cryostat
       test: cryostat-recording
     storage:
       spec:
         mountPath: {}
+  - entrypoint:
+    - cryostat-scorecard-tests
+    - cryostat-config-change
+    image: quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307153901
+    labels:
+      suite: cryostat
+      test: cryostat-config-change
+    storage:
+      spec:
+        mountPath: {}
 storage:
   spec:
     mountPath: {}

config/scorecard/patches/custom.config.yaml

@@ -8,7 +8,7 @@
     entrypoint:
     - cryostat-scorecard-tests
     - operator-install
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240305020416"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322"
     labels:
       suite: cryostat
       test: operator-install
@@ -18,7 +18,7 @@
     entrypoint:
     - cryostat-scorecard-tests
     - cryostat-cr
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240305020416"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322"
     labels:
       suite: cryostat
       test: cryostat-cr
@@ -28,7 +28,17 @@
     entrypoint:
     - cryostat-scorecard-tests
     - cryostat-recording
-    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240305020416"
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322"
     labels:
       suite: cryostat
       test: cryostat-recording
+- op: add
+  path: /stages/1/tests/-
+  value:
+    entrypoint:
+    - cryostat-scorecard-tests
+    - cryostat-config-change
+    image: "quay.io/cryostat/cryostat-operator-scorecard:2.5.0-20240307154322"
+    labels:
+      suite: cryostat
+      test: cryostat-config-change

go-license.yml

@@ -13,6 +13,26 @@ header: |
   // See the License for the specific language governing permissions and
   // limitations under the License.
 
+custom-headers:
+  - name: kubectl
+    header: |
+      // Copyright The Cryostat Authors.
+      // Copyright 2016 The Kubernetes Authors.
+      //
+      // Licensed under the Apache License, Version 2.0 (the "License");
+      // you may not use this file except in compliance with the License.
+      // You may obtain a copy of the License at
+      //
+      // http://www.apache.org/licenses/LICENSE-2.0
+      //
+      // Unless required by applicable law or agreed to in writing, software
+      // distributed under the License is distributed on an "AS IS" BASIS,
+      // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+      // See the License for the specific language governing permissions and
+      // limitations under the License.
+    paths:
+      - internal/test/scorecard/common_utils.go
+
 exclude:
   names:
     - '.*generated.*'

hack/custom.config.yaml.in

@@ -31,3 +31,13 @@
     labels:
       suite: cryostat
       test: cryostat-recording
+- op: add
+  path: /stages/1/tests/-
+  value:
+    entrypoint:
+    - cryostat-scorecard-tests
+    - cryostat-config-change
+    image: "${CUSTOM_SCORECARD_IMG}"
+    labels:
+      suite: cryostat
+      test: cryostat-config-change

internal/images/custom-scorecard-tests/main.go

@@ -80,6 +80,7 @@ func printValidTests() []scapiv1alpha3.TestResult {
 		tests.OperatorInstallTestName,
 		tests.CryostatCRTestName,
 		tests.CryostatRecordingTestName,
+		tests.CryostatConfigChangeTestName,
 	}, ","))
 	result.Errors = append(result.Errors, str)
 
@@ -92,6 +93,7 @@ func validateTests(testNames []string) bool {
 		case tests.OperatorInstallTestName:
 		case tests.CryostatCRTestName:
 		case tests.CryostatRecordingTestName:
+		case tests.CryostatConfigChangeTestName:
 		default:
 			return false
 		}
@@ -112,6 +114,8 @@ func runTests(testNames []string, bundle *apimanifests.Bundle, namespace string,
 			results = append(results, tests.CryostatCRTest(bundle, namespace, openShiftCertManager))
 		case tests.CryostatRecordingTestName:
 			results = append(results, tests.CryostatRecordingTest(bundle, namespace, openShiftCertManager))
+		case tests.CryostatConfigChangeTestName:
+			results = append(results, tests.CryostatConfigChangeTest(bundle, namespace, openShiftCertManager))
 		default:
 			log.Fatalf("unknown test found: %s", testName)
 		}

internal/test/scorecard/clients.go

@@ -136,7 +136,7 @@ func (c *CryostatClient) Create(ctx context.Context, obj *operatorv1beta1.Cryost
 
 // Update updates the provided Cryostat CR
 func (c *CryostatClient) Update(ctx context.Context, obj *operatorv1beta1.Cryostat) (*operatorv1beta1.Cryostat, error) {
-	return update(ctx, c.restClient, c.resource, c.namespace, obj, &operatorv1beta1.Cryostat{})
+	return update(ctx, c.restClient, c.resource, c.namespace, obj, &operatorv1beta1.Cryostat{}, obj.Name)
 }
 
 // Delete deletes the Cryostat CR with the given name
@@ -158,9 +158,9 @@ func create[r runtime.Object](ctx context.Context, c rest.Interface, res string,
 	return result, err
 }
 
-func update[r runtime.Object](ctx context.Context, c rest.Interface, res string, ns string, obj r, result r) (r, error) {
+func update[r runtime.Object](ctx context.Context, c rest.Interface, res string, ns string, obj r, result r, name string) (r, error) {
 	err := c.Put().
-		Namespace(ns).Resource(res).
+		Namespace(ns).Resource(res).Name(name).
 		Body(obj).Do(ctx).Into(result)
 	return result, err
 }

internal/test/scorecard/common_utils.go

@@ -1,4 +1,5 @@
 // Copyright The Cryostat Authors.
+// Copyright 2016 The Kubernetes Authors.
 //
 // Licensed under the Apache License, Version 2.0 (the "License");
 // you may not use this file except in compliance with the License.
@@ -383,6 +384,72 @@ func waitTillCryostatReady(base *url.URL, resources *TestResources) error {
 	return err
 }
 
+func updateAndWaitTillCryostatAvailable(cr *operatorv1beta1.Cryostat, resources *TestResources) error {
+	client := resources.Client
+	r := resources.TestResult
+
+	cr, err := client.OperatorCRDs().Cryostats(cr.Namespace).Update(context.Background(), cr)
+	if err != nil {
+		r.Log += fmt.Sprintf("failed to update Cryostat CR: %s", err.Error())
+		return err
+	}
+
+	// Poll the deployment until it becomes available or we timeout
+	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+	defer cancel()
+	err = wait.PollImmediateUntilWithContext(ctx, time.Second, func(ctx context.Context) (done bool, err error) {
+		deploy, err := client.AppsV1().Deployments(cr.Namespace).Get(ctx, cr.Name, metav1.GetOptions{})
+		if err != nil {
+			if kerrors.IsNotFound(err) {
+				r.Log += fmt.Sprintf("deployment %s is not yet found\n", cr.Name)
+				return false, nil // Retry
+			}
+			return false, fmt.Errorf("failed to get deployment: %s", err.Error())
+		}
+
+		// Wait for deployment to update by verifying Cryostat has PVC configured
+		for _, volume := range deploy.Spec.Template.Spec.Volumes {
+			if volume.VolumeSource.EmptyDir != nil {
+				r.Log += fmt.Sprintf("Cryostat deployment is still updating. Storage: %s\n", volume.VolumeSource.EmptyDir)
+				return false, nil // Retry
+			}
+			if volume.VolumeSource.PersistentVolumeClaim != nil {
+				break
+			}
+		}
+
+		// Derived from kubectl: https://github.com/kubernetes/kubectl/blob/24d21a0/pkg/polymorphichelpers/rollout_status.go#L75-L91
+		// Check for deployment condition
+		if deploy.Generation <= deploy.Status.ObservedGeneration {
+			for _, condition := range deploy.Status.Conditions {
+				if condition.Type == appsv1.DeploymentProgressing && condition.Status == corev1.ConditionFalse && condition.Reason == "ProgressDeadlineExceeded" {
+					return false, fmt.Errorf("deployment %s exceeded its progress deadline", deploy.Name) // Don't Retry
+				}
+			}
+			if deploy.Spec.Replicas != nil && deploy.Status.UpdatedReplicas < *deploy.Spec.Replicas {
+				r.Log += fmt.Sprintf("Waiting for deployment %s rollout to finish: %d out of %d new replicas have been updated... \n", deploy.Name, deploy.Status.UpdatedReplicas, *deploy.Spec.Replicas)
+				return false, nil
+			}
+			if deploy.Status.Replicas > deploy.Status.UpdatedReplicas {
+				r.Log += fmt.Sprintf("Waiting for deployment %s rollout to finish: %d old replicas are pending termination... \n", deploy.Name, deploy.Status.Replicas-deploy.Status.UpdatedReplicas)
+				return false, nil
+			}
+			if deploy.Status.AvailableReplicas < deploy.Status.UpdatedReplicas {
+				r.Log += fmt.Sprintf("Waiting for deployment %s rollout to finish: %d out of %d updated replicas are available... \n", deploy.Name, deploy.Status.AvailableReplicas, deploy.Status.UpdatedReplicas)
+				return false, nil
+			}
+			r.Log += fmt.Sprintf("deployment %s successfully rolled out\n", deploy.Name)
+			return true, nil
+		}
+		r.Log += "Waiting for deployment spec update to be observed...\n"
+		return false, nil
+	})
+	if err != nil {
+		return fmt.Errorf("failed to look up deployment errors: %s", err.Error())
+	}
+	return err
+}
+
 func cleanupCryostat(r *scapiv1alpha3.TestResult, client *CryostatClientset, name string, namespace string) {
 	cr := &operatorv1beta1.Cryostat{
 		ObjectMeta: metav1.ObjectMeta{

internal/test/scorecard/tests.go

@@ -20,15 +20,19 @@ import (
 	"net/url"
 	"time"
 
+	operatorv1beta1 "github.com/cryostatio/cryostat-operator/api/v1beta1"
 	scapiv1alpha3 "github.com/operator-framework/api/pkg/apis/scorecard/v1alpha3"
 	apimanifests "github.com/operator-framework/api/pkg/manifests"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 const (
-	OperatorInstallTestName   string = "operator-install"
-	CryostatCRTestName        string = "cryostat-cr"
-	CryostatRecordingTestName string = "cryostat-recording"
+	OperatorInstallTestName      string = "operator-install"
+	CryostatCRTestName           string = "cryostat-cr"
+	CryostatRecordingTestName    string = "cryostat-recording"
+	CryostatConfigChangeTestName string = "cryostat-config-change"
 )
 
 // OperatorInstallTest checks that the operator installed correctly
@@ -72,6 +76,71 @@ func CryostatCRTest(bundle *apimanifests.Bundle, namespace string, openShiftCert
 	return *r
 }
 
+func CryostatConfigChangeTest(bundle *apimanifests.Bundle, namespace string, openShiftCertManager bool) scapiv1alpha3.TestResult {
+	tr := newTestResources(CryostatConfigChangeTestName)
+	r := tr.TestResult
+
+	err := setupCRTestResources(tr, openShiftCertManager)
+	if err != nil {
+		return fail(*r, fmt.Sprintf("failed to set up %s test: %s", CryostatConfigChangeTestName, err.Error()))
+	}
+
+	// Create a default Cryostat CR with default empty dir
+	cr := newCryostatCR(CryostatConfigChangeTestName, namespace, !tr.OpenShift)
+	cr.Spec.StorageOptions = &operatorv1beta1.StorageConfiguration{
+		EmptyDir: &operatorv1beta1.EmptyDirConfig{
+			Enabled: true,
+		},
+	}
+
+	_, err = createAndWaitTillCryostatAvailable(cr, tr)
+	if err != nil {
+		return fail(*r, fmt.Sprintf("failed to determine application URL: %s", err.Error()))
+	}
+	defer cleanupCryostat(r, tr.Client, CryostatRecordingTestName, namespace)
+
+	// Switch Cryostat CR to PVC for redeployment
+	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
+	defer cancel()
+	client := tr.Client
+
+	cr, err = client.OperatorCRDs().Cryostats(namespace).Get(ctx, CryostatConfigChangeTestName)
+	if err != nil {
+		return fail(*r, fmt.Sprintf("failed to get Cryostat CR: %s", err.Error()))
+	}
+	cr.Spec.StorageOptions = &operatorv1beta1.StorageConfiguration{
+		PVC: &operatorv1beta1.PersistentVolumeClaimConfig{
+			Spec: &corev1.PersistentVolumeClaimSpec{
+				StorageClassName: nil,
+				Resources: corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						corev1.ResourceStorage: resource.MustParse("1Gi"),
+					},
+				},
+			},
+		},
+	}
+
+	// Wait for redeployment of Cryostat CR
+	err = updateAndWaitTillCryostatAvailable(cr, tr)
+	if err != nil {
+		return fail(*r, fmt.Sprintf("Cryostat redeployment did not become available: %s", err.Error()))
+	}
+	r.Log += "Cryostat deployment has successfully updated with new spec template"
+
+	base, err := url.Parse(cr.Status.ApplicationURL)
+	if err != nil {
+		return fail(*r, fmt.Sprintf("application URL is invalid: %s", err.Error()))
+	}
+
+	err = waitTillCryostatReady(base, tr)
+	if err != nil {
+		return fail(*r, fmt.Sprintf("failed to reach the application: %s", err.Error()))
+	}
+
+	return *r
+}
+
 // TODO add a built in discovery test too
 func CryostatRecordingTest(bundle *apimanifests.Bundle, namespace string, openShiftCertManager bool) scapiv1alpha3.TestResult {
 	tr := newTestResources(CryostatRecordingTestName)