fix: Debug logging the cache (#21)

* fix: Debug logging the cache

* debug mode

config/crd/bases/apps.wandb.com_weightsandbiases.yaml

@@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.12.0
+    controller-gen.kubebuilder.io/version: v0.14.0
   name: weightsandbiases.apps.wandb.com
 spec:
   group: apps.wandb.com
@@ -22,14 +22,19 @@ spec:
         description: WeightsAndBiases is the Schema for the weightsandbiases API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object

controllers/weightsandbiases_controller.go

@@ -18,10 +18,11 @@ package controllers
 
 import (
 	"context"
+	"reflect"
+
 	"github.com/wandb/operator/pkg/wandb/spec/state"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/record"
-	"reflect"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -54,6 +55,7 @@ type WeightsAndBiasesReconciler struct {
 	Scheme         *runtime.Scheme
 	Recorder       record.EventRecorder
 	DryRun         bool
+	Debug          bool
 }
 
 //+kubebuilder:rbac:groups=apps.wandb.com,resources=weightsandbiases,verbs=get;list;watch;create;update;patch;delete
@@ -113,7 +115,7 @@ func (r *WeightsAndBiasesReconciler) Reconcile(ctx context.Context, req ctrl.Req
 
 	currentActiveSpec, err := specManager.GetActive()
 	if err != nil {
-		// This scenario can happen if we have not successfully deploy in the
+		// This scenario can happen if we have not successfully deployed in the
 		// past.
 		log.Info("No active spec found.")
 	}
@@ -130,12 +132,16 @@ func (r *WeightsAndBiasesReconciler) Reconcile(ctx context.Context, req ctrl.Req
 			// is not operational, and a version has been deployed successfully. Rather than
 			// reverting to the container defaults, we've stored the most recent successful
 			// deployer release in the cache
+			// Attempt to retrieve the cached release
 			if deployerSpec, err = specManager.Get("latest-cached-release"); err != nil {
 				log.Error(err, "No cached release found for deployer spec", err, "error")
+			} else {
+				log.Info("Using cached deployer spec", "cachedSpec", deployerSpec)
 			}
 		}
 
 		if deployerSpec != nil {
+			log.Info("Writing deployer spec to cache", "cachedSpec", deployerSpec)
 			if err := specManager.Set("latest-cached-release", deployerSpec); err != nil {
 				r.Recorder.Event(wandb, corev1.EventTypeNormal, "SecretWriteFailed", "Unable to write secret to kubernetes")
 				log.Error(err, "Unable to save latest release.")
@@ -145,11 +151,31 @@ func (r *WeightsAndBiasesReconciler) Reconcile(ctx context.Context, req ctrl.Req
 	}
 
 	desiredSpec := new(spec.Spec)
+
+	if r.Debug {
+		log.Info("Initial desired spec", "spec", desiredSpec.SensitiveValuesMasked())
+	}
+
 	// First takes precedence
 	desiredSpec.Merge(crdSpec)
+	if r.Debug {
+		log.Info("Desired spec after merging crdSpec", "spec", desiredSpec.SensitiveValuesMasked())
+	}
+
 	desiredSpec.Merge(userInputSpec)
+	if r.Debug {
+		log.Info("Desired spec after merging userInputSpec", "spec", desiredSpec.SensitiveValuesMasked())
+	}
+
 	desiredSpec.Merge(deployerSpec)
+	if r.Debug {
+		log.Info("Desired spec after merging deployerSpec", "spec", desiredSpec.SensitiveValuesMasked())
+	}
+
 	desiredSpec.Merge(operator.Defaults(wandb, r.Scheme))
+	if r.Debug {
+		log.Info("Desired spec after merging operator defaults", "spec", desiredSpec.SensitiveValuesMasked())
+	}
 
 	log.Info("Desired spec", "spec", desiredSpec.SensitiveValuesMasked())
 

main.go

@@ -19,12 +19,14 @@ package main
 import (
 	"flag"
 	"fmt"
-	"github.com/wandb/operator/pkg/wandb/spec/channel/deployer"
 	"os"
 
-	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"github.com/wandb/operator/pkg/wandb/spec/channel/deployer"
+
 	"strings"
 
+	"sigs.k8s.io/controller-runtime/pkg/cache"
+
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 
@@ -56,13 +58,14 @@ func init() {
 
 func main() {
 	var metricsAddr, deployerAPI, probeAddr, isolationNamespaces string
-	var enableLeaderElection, airgapped bool
+	var enableLeaderElection, airgapped, debug bool
 
 	flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.")
 	flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
 	flag.StringVar(&deployerAPI, "deployer-channel-url", "", "URL of the deployer channel")
 
 	flag.BoolVar(&airgapped, "airgapped", false, "Enable airgapped mode")
+	flag.BoolVar(&debug, "debug", false, "Enable debug mode")
 	flag.BoolVar(&enableLeaderElection, "leader-elect", false,
 		"Enable leader election for controller manager. "+
 			"Enabling this will ensure there is only one active controller manager.")
@@ -117,6 +120,7 @@ func main() {
 		Client:         mgr.GetClient(),
 		Scheme:         mgr.GetScheme(),
 		DeployerClient: &deployer.DeployerClient{DeployerChannelUrl: deployerAPI},
+		Debug:          debug,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "WeightsAndBiases")
 		os.Exit(1)