diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 0aed5c3..b61fc2b 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -1,5 +1,4 @@ //go:build !ignore_autogenerated -// +build !ignore_autogenerated /* Copyright 2023. diff --git a/config/crd/bases/apps.wandb.com_weightsandbiases.yaml b/config/crd/bases/apps.wandb.com_weightsandbiases.yaml index f22633c..1c5fbed 100644 --- a/config/crd/bases/apps.wandb.com_weightsandbiases.yaml +++ b/config/crd/bases/apps.wandb.com_weightsandbiases.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.12.0 + controller-gen.kubebuilder.io/version: v0.14.0 name: weightsandbiases.apps.wandb.com spec: group: apps.wandb.com @@ -22,14 +22,19 @@ spec: description: WeightsAndBiases is the Schema for the weightsandbiases API properties: apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources type: string kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds type: string metadata: type: object diff --git a/controllers/weightsandbiases_controller.go b/controllers/weightsandbiases_controller.go index 3cbcf1f..eea1b57 100644 --- a/controllers/weightsandbiases_controller.go +++ b/controllers/weightsandbiases_controller.go @@ -18,10 +18,11 @@ package controllers import ( "context" + "reflect" + "github.com/wandb/operator/pkg/wandb/spec/state" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/record" - "reflect" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/builder" "sigs.k8s.io/controller-runtime/pkg/client" @@ -54,6 +55,7 @@ type WeightsAndBiasesReconciler struct { Scheme *runtime.Scheme Recorder record.EventRecorder DryRun bool + Debug bool } //+kubebuilder:rbac:groups=apps.wandb.com,resources=weightsandbiases,verbs=get;list;watch;create;update;patch;delete @@ -113,7 +115,7 @@ func (r *WeightsAndBiasesReconciler) Reconcile(ctx context.Context, req ctrl.Req currentActiveSpec, err := specManager.GetActive() if err != nil { - // This scenario can happen if we have not successfully deploy in the + // This scenario can happen if we have not successfully deployed in the // past. log.Info("No active spec found.") } @@ -130,12 +132,16 @@ func (r *WeightsAndBiasesReconciler) Reconcile(ctx context.Context, req ctrl.Req // is not operational, and a version has been deployed successfully. Rather than // reverting to the container defaults, we've stored the most recent successful // deployer release in the cache + // Attempt to retrieve the cached release if deployerSpec, err = specManager.Get("latest-cached-release"); err != nil { log.Error(err, "No cached release found for deployer spec", err, "error") + } else { + log.Info("Using cached deployer spec", "cachedSpec", deployerSpec) } } if deployerSpec != nil { + log.Info("Writing deployer spec to cache", "cachedSpec", deployerSpec) if err := specManager.Set("latest-cached-release", deployerSpec); err != nil { r.Recorder.Event(wandb, corev1.EventTypeNormal, "SecretWriteFailed", "Unable to write secret to kubernetes") log.Error(err, "Unable to save latest release.") @@ -145,11 +151,31 @@ func (r *WeightsAndBiasesReconciler) Reconcile(ctx context.Context, req ctrl.Req } desiredSpec := new(spec.Spec) + + if r.Debug { + log.Info("Initial desired spec", "spec", desiredSpec.SensitiveValuesMasked()) + } + // First takes precedence desiredSpec.Merge(crdSpec) + if r.Debug { + log.Info("Desired spec after merging crdSpec", "spec", desiredSpec.SensitiveValuesMasked()) + } + desiredSpec.Merge(userInputSpec) + if r.Debug { + log.Info("Desired spec after merging userInputSpec", "spec", desiredSpec.SensitiveValuesMasked()) + } + desiredSpec.Merge(deployerSpec) + if r.Debug { + log.Info("Desired spec after merging deployerSpec", "spec", desiredSpec.SensitiveValuesMasked()) + } + desiredSpec.Merge(operator.Defaults(wandb, r.Scheme)) + if r.Debug { + log.Info("Desired spec after merging operator defaults", "spec", desiredSpec.SensitiveValuesMasked()) + } log.Info("Desired spec", "spec", desiredSpec.SensitiveValuesMasked()) diff --git a/main.go b/main.go index 67bd760..0dadd35 100644 --- a/main.go +++ b/main.go @@ -19,12 +19,14 @@ package main import ( "flag" "fmt" - "github.com/wandb/operator/pkg/wandb/spec/channel/deployer" "os" - "sigs.k8s.io/controller-runtime/pkg/cache" + "github.com/wandb/operator/pkg/wandb/spec/channel/deployer" + "strings" + "sigs.k8s.io/controller-runtime/pkg/cache" + // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -56,13 +58,14 @@ func init() { func main() { var metricsAddr, deployerAPI, probeAddr, isolationNamespaces string - var enableLeaderElection, airgapped bool + var enableLeaderElection, airgapped, debug bool flag.StringVar(&metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.") flag.StringVar(&deployerAPI, "deployer-channel-url", "", "URL of the deployer channel") flag.BoolVar(&airgapped, "airgapped", false, "Enable airgapped mode") + flag.BoolVar(&debug, "debug", false, "Enable debug mode") flag.BoolVar(&enableLeaderElection, "leader-elect", false, "Enable leader election for controller manager. "+ "Enabling this will ensure there is only one active controller manager.") @@ -117,6 +120,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), DeployerClient: &deployer.DeployerClient{DeployerChannelUrl: deployerAPI}, + Debug: debug, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "WeightsAndBiases") os.Exit(1)