From dea0b63bf8aab606d98a749121aa4c124a5872ff Mon Sep 17 00:00:00 2001 From: Ahmed Mezghani Date: Fri, 6 Aug 2021 14:22:11 +0200 Subject: [PATCH] Generate a PodTemplate for each EDS --- api/v1alpha1/test/new.go | 16 +- config/rbac/role.yaml | 12 + controllers/podtemplate/controller.go | 160 +++++++++++++ controllers/podtemplate/controller_test.go | 261 +++++++++++++++++++++ controllers/podtemplate/doc.go | 7 + controllers/podtemplate_controller.go | 53 +++++ controllers/setup.go | 11 + 7 files changed, 514 insertions(+), 6 deletions(-) create mode 100644 controllers/podtemplate/controller.go create mode 100644 controllers/podtemplate/controller_test.go create mode 100644 controllers/podtemplate/doc.go create mode 100644 controllers/podtemplate_controller.go diff --git a/api/v1alpha1/test/new.go b/api/v1alpha1/test/new.go index fc07095f..3025bb7f 100644 --- a/api/v1alpha1/test/new.go +++ b/api/v1alpha1/test/new.go @@ -22,12 +22,13 @@ var apiVersion = fmt.Sprintf("%s/%s", datadoghqv1alpha1.GroupVersion.Group, data // NewExtendedDaemonSetOptions set of option for the ExtendedDaemonset creation. type NewExtendedDaemonSetOptions struct { - CreationTime *time.Time - Annotations map[string]string - Labels map[string]string - RollingUpdate *datadoghqv1alpha1.ExtendedDaemonSetSpecStrategyRollingUpdate - Canary *datadoghqv1alpha1.ExtendedDaemonSetSpecStrategyCanary - Status *datadoghqv1alpha1.ExtendedDaemonSetStatus + CreationTime *time.Time + Annotations map[string]string + Labels map[string]string + RollingUpdate *datadoghqv1alpha1.ExtendedDaemonSetSpecStrategyRollingUpdate + Canary *datadoghqv1alpha1.ExtendedDaemonSetSpecStrategyCanary + Status *datadoghqv1alpha1.ExtendedDaemonSetStatus + PodTemplateSpec *corev1.PodTemplateSpec } // NewExtendedDaemonSet return new ExtendedDDaemonset instance for test purpose. @@ -67,6 +68,9 @@ func NewExtendedDaemonSet(ns, name string, options *NewExtendedDaemonSetOptions) if options.Status != nil { dd.Status = *options.Status } + if options.PodTemplateSpec != nil { + dd.Spec.Template = *options.PodTemplateSpec + } } return dd diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 43306b88..1a22c99e 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -32,6 +32,18 @@ rules: - patch - update - watch +- apiGroups: + - "" + resources: + - podtemplates + verbs: + - create + - delete + - get + - list + - patch + - update + - watch - apiGroups: - "" resources: diff --git a/controllers/podtemplate/controller.go b/controllers/podtemplate/controller.go new file mode 100644 index 00000000..d86ecda9 --- /dev/null +++ b/controllers/podtemplate/controller.go @@ -0,0 +1,160 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package podtemplate + +import ( + "context" + "fmt" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + datadoghqv1alpha1 "github.com/DataDog/extendeddaemonset/api/v1alpha1" + "github.com/DataDog/extendeddaemonset/pkg/controller/utils/comparison" +) + +const ( + // podTemplateDaemonSetLabelKey declares to the cluster-autoscaler that the pod template corresponds to an extra daemonset. + podTemplateDaemonSetLabelKey = "cluster-autoscaler.kubernetes.io/daemonset-pod" + // podTemplateDaemonSetLabelValueTrue used as podTemplateDaemonSetLabelKey label value. + podTemplateDaemonSetLabelValueTrue = "true" +) + +// Reconciler is the internal reconciler for PodTemplate. +type Reconciler struct { + options ReconcilerOptions + client client.Client + scheme *runtime.Scheme + log logr.Logger + recorder record.EventRecorder +} + +// ReconcilerOptions provides options read from command line. +type ReconcilerOptions struct{} + +// NewReconciler returns a reconciler for PodTemplate. +func NewReconciler(options ReconcilerOptions, client client.Client, scheme *runtime.Scheme, log logr.Logger, recorder record.EventRecorder) (*Reconciler, error) { + return &Reconciler{ + options: options, + client: client, + scheme: scheme, + log: log, + recorder: recorder, + }, nil +} + +// Reconcile creates and updates PodTemplate objects corresponding to ExtendedDaemonSets. +func (r *Reconciler) Reconcile(ctx context.Context, request reconcile.Request) (reconcile.Result, error) { + reqLogger := r.log.WithValues("Req.NS", request.Namespace, "Req.Name", request.Name) + reqLogger.Info("Reconciling PodTemplate") + + eds := &datadoghqv1alpha1.ExtendedDaemonSet{} + err := r.client.Get(context.TODO(), request.NamespacedName, eds) + if err != nil { + return reconcile.Result{}, err + } + + podTpl := &corev1.PodTemplate{} + err = r.client.Get(context.TODO(), request.NamespacedName, podTpl) + if err != nil { + if errors.IsNotFound(err) { + return r.createPodTemplate(reqLogger, eds) + } + + return reconcile.Result{}, err + } + + return r.updatePodTemplateIfNeeded(reqLogger, eds, podTpl) +} + +// createPodTemplate creates a new PodTemplate object corresponding to a given ExtendedDaemonSet. +func (r *Reconciler) createPodTemplate(logger logr.Logger, eds *datadoghqv1alpha1.ExtendedDaemonSet) (reconcile.Result, error) { + podTpl, err := r.newPodTemplate(eds) + if err != nil { + return reconcile.Result{}, err + } + + logger.Info("Creating a new PodTemplate", "podTemplate.Namespace", podTpl.Namespace, "podTemplate.Name", podTpl.Name) + + err = r.client.Create(context.TODO(), podTpl) + if err != nil { + return reconcile.Result{}, err + } + + r.recorder.Event(eds, corev1.EventTypeNormal, "Create PodTemplate", fmt.Sprintf("%s/%s", podTpl.Namespace, podTpl.Name)) + + return reconcile.Result{}, nil +} + +// updatePodTemplateIfNeeded updates the PodTemplate object accordingly to the given ExtendedDaemonSet. +// It does nothing if the PodTemplate is up-to-date. +func (r *Reconciler) updatePodTemplateIfNeeded(logger logr.Logger, eds *datadoghqv1alpha1.ExtendedDaemonSet, podTpl *corev1.PodTemplate) (reconcile.Result, error) { + specHash, err := comparison.GenerateMD5PodTemplateSpec(&eds.Spec.Template) + if err != nil { + return reconcile.Result{}, fmt.Errorf("cannot generate pod template hash: %w", err) + } + + if podTpl.Annotations != nil && podTpl.Annotations[datadoghqv1alpha1.MD5ExtendedDaemonSetAnnotationKey] == specHash { + // Already up-to-date + return reconcile.Result{}, nil + } + + logger.Info("Updating PodTemplate", "podTemplate.Namespace", podTpl.Namespace, "podTemplate.Name", podTpl.Name) + + newPodTpl, err := r.newPodTemplate(eds) + if err != nil { + return reconcile.Result{}, err + } + + err = r.client.Update(context.TODO(), newPodTpl) + if err != nil { + return reconcile.Result{}, err + } + + r.recorder.Event(eds, corev1.EventTypeNormal, "Update PodTemplate", fmt.Sprintf("%s/%s", podTpl.Namespace, podTpl.Name)) + + return reconcile.Result{}, nil +} + +// newPodTemplate generates a PodTemplate object based on ExtendedDaemonSet. +func (r *Reconciler) newPodTemplate(eds *datadoghqv1alpha1.ExtendedDaemonSet) (*corev1.PodTemplate, error) { + podTpl := &corev1.PodTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: eds.Name, + Namespace: eds.Namespace, + Labels: eds.Labels, + Annotations: eds.Annotations, + }, + Template: *eds.Spec.Template.DeepCopy(), + } + + specHash, err := comparison.GenerateMD5PodTemplateSpec(&eds.Spec.Template) + if err != nil { + return nil, fmt.Errorf("cannot generate pod template hash: %w", err) + } + + if podTpl.Annotations == nil { + podTpl.Annotations = make(map[string]string) + } + + podTpl.Annotations[datadoghqv1alpha1.MD5ExtendedDaemonSetAnnotationKey] = specHash + + if podTpl.Labels == nil { + podTpl.Labels = make(map[string]string) + } + + podTpl.Labels[podTemplateDaemonSetLabelKey] = podTemplateDaemonSetLabelValueTrue + err = controllerutil.SetControllerReference(eds, podTpl, r.scheme) + + return podTpl, err +} diff --git a/controllers/podtemplate/controller_test.go b/controllers/podtemplate/controller_test.go new file mode 100644 index 00000000..82324187 --- /dev/null +++ b/controllers/podtemplate/controller_test.go @@ -0,0 +1,261 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +package podtemplate + +import ( + "context" + "errors" + "testing" + + datadoghqv1alpha1 "github.com/DataDog/extendeddaemonset/api/v1alpha1" + "github.com/DataDog/extendeddaemonset/api/v1alpha1/test" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/scheme" + "k8s.io/client-go/tools/record" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + logf "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +func podTemplateSpec(ctrName, ctrImage string) *corev1.PodTemplateSpec { + return &corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: ctrName, + Image: ctrImage, + }, + }, + }, + } +} + +func defaultOwnerRef() []metav1.OwnerReference { + return []metav1.OwnerReference{ + { + APIVersion: "datadoghq.com/v1alpha1", + Name: "eds-name", + Kind: "ExtendedDaemonSet", + Controller: datadoghqv1alpha1.NewBool(true), + BlockOwnerDeletion: datadoghqv1alpha1.NewBool(true), + }, + } +} + +func defaultRequest() reconcile.Request { + return reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: "eds-ns", + Name: "eds-name", + }, + } +} + +func TestReconciler_newPodTemplate(t *testing.T) { + s := scheme.Scheme + s.AddKnownTypes(datadoghqv1alpha1.GroupVersion, &datadoghqv1alpha1.ExtendedDaemonSet{}) + tests := []struct { + name string + eds *datadoghqv1alpha1.ExtendedDaemonSet + want *corev1.PodTemplate + wantErr bool + }{ + { + name: "nominal case", + eds: test.NewExtendedDaemonSet("eds-ns", "eds-name", &test.NewExtendedDaemonSetOptions{PodTemplateSpec: podTemplateSpec("name", "image")}), + want: &corev1.PodTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "eds-name", + Namespace: "eds-ns", + Labels: map[string]string{ + "cluster-autoscaler.kubernetes.io/daemonset-pod": "true", + }, + Annotations: map[string]string{ + "extendeddaemonset.datadoghq.com/templatehash": "c08e4c7b196a8a9ba3fd4723a4366c8b", + }, + OwnerReferences: defaultOwnerRef(), + }, + Template: *podTemplateSpec("name", "image"), + }, + wantErr: false, + }, + { + name: "with extra labels and annotations", + eds: test.NewExtendedDaemonSet("eds-ns", "eds-name", &test.NewExtendedDaemonSetOptions{Labels: map[string]string{"l-key": "l-val"}, Annotations: map[string]string{"a-key": "a-val"}, PodTemplateSpec: podTemplateSpec("name", "image")}), + want: &corev1.PodTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "eds-name", + Namespace: "eds-ns", + Labels: map[string]string{ + "cluster-autoscaler.kubernetes.io/daemonset-pod": "true", + "l-key": "l-val", + }, + Annotations: map[string]string{ + "extendeddaemonset.datadoghq.com/templatehash": "c08e4c7b196a8a9ba3fd4723a4366c8b", + "a-key": "a-val", + }, + OwnerReferences: defaultOwnerRef(), + }, + Template: *podTemplateSpec("name", "image"), + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &Reconciler{scheme: s} + got, err := r.newPodTemplate(tt.eds) + assert.True(t, (err == nil), tt.wantErr) + assert.EqualValues(t, tt.want, got) + }) + } +} + +func TestReconciler_Reconcile(t *testing.T) { + s := scheme.Scheme + s.AddKnownTypes(datadoghqv1alpha1.GroupVersion, &datadoghqv1alpha1.ExtendedDaemonSet{}) + tests := []struct { + name string + request reconcile.Request + loadFunc func(c client.Client) + want reconcile.Result + wantErr bool + wantFunc func(c client.Client) error + }{ + { + name: "EDS not found", + request: defaultRequest(), + want: reconcile.Result{}, + wantErr: true, + }, + { + name: "PodTemplate not found", + request: defaultRequest(), + loadFunc: func(c client.Client) { + eds := test.NewExtendedDaemonSet("eds-ns", "eds-name", &test.NewExtendedDaemonSetOptions{PodTemplateSpec: podTemplateSpec("name", "image")}) + eds = datadoghqv1alpha1.DefaultExtendedDaemonSet(eds, datadoghqv1alpha1.ExtendedDaemonSetSpecStrategyCanaryValidationModeAuto) + _ = c.Create(context.TODO(), eds) + }, + want: reconcile.Result{}, + wantErr: false, + wantFunc: func(c client.Client) error { + return c.Get(context.TODO(), defaultRequest().NamespacedName, &corev1.PodTemplate{}) + }, + }, + { + name: "PodTemplate found and up-to-date", + request: defaultRequest(), + loadFunc: func(c client.Client) { + eds := test.NewExtendedDaemonSet("eds-ns", "eds-name", &test.NewExtendedDaemonSetOptions{PodTemplateSpec: podTemplateSpec("name", "image")}) + eds = datadoghqv1alpha1.DefaultExtendedDaemonSet(eds, datadoghqv1alpha1.ExtendedDaemonSetSpecStrategyCanaryValidationModeAuto) + _ = c.Create(context.TODO(), eds) + podTemplate := &corev1.PodTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "eds-name", + Namespace: "eds-ns", + Labels: map[string]string{ + "cluster-autoscaler.kubernetes.io/daemonset-pod": "true", + }, + Annotations: map[string]string{ + "extendeddaemonset.datadoghq.com/templatehash": "c08e4c7b196a8a9ba3fd4723a4366c8b", + }, + OwnerReferences: defaultOwnerRef(), + }, + Template: *podTemplateSpec("name", "image"), + } + _ = c.Create(context.TODO(), podTemplate) + }, + want: reconcile.Result{}, + wantErr: false, + wantFunc: func(c client.Client) error { + podTemplate := &corev1.PodTemplate{} + + return c.Get(context.TODO(), defaultRequest().NamespacedName, podTemplate) + }, + }, + { + name: "PodTemplate outdated", + request: defaultRequest(), + loadFunc: func(c client.Client) { + eds := test.NewExtendedDaemonSet("eds-ns", "eds-name", &test.NewExtendedDaemonSetOptions{PodTemplateSpec: podTemplateSpec("new-name", "new-image")}) + eds = datadoghqv1alpha1.DefaultExtendedDaemonSet(eds, datadoghqv1alpha1.ExtendedDaemonSetSpecStrategyCanaryValidationModeAuto) + _ = c.Create(context.TODO(), eds) + podTemplate := &corev1.PodTemplate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "eds-name", + Namespace: "eds-ns", + Labels: map[string]string{ + "cluster-autoscaler.kubernetes.io/daemonset-pod": "true", + }, + Annotations: map[string]string{ + "extendeddaemonset.datadoghq.com/templatehash": "outdated-hash", + }, + OwnerReferences: defaultOwnerRef(), + }, + Template: *podTemplateSpec("name", "image"), + } + _ = c.Create(context.TODO(), podTemplate) + }, + want: reconcile.Result{}, + wantErr: false, + wantFunc: func(c client.Client) error { + podTemplate := &corev1.PodTemplate{} + err := c.Get(context.TODO(), defaultRequest().NamespacedName, podTemplate) + if err != nil { + return err + } + + err = errors.New("podTemplate not updated") + if podTemplate.Annotations["extendeddaemonset.datadoghq.com/templatehash"] == "outdated-hash" { + return err + } + + if podTemplate.Template.Spec.Containers[0].Name != "new-name" { + return err + } + + if podTemplate.Template.Spec.Containers[0].Image != "new-image" { + return err + } + + return nil + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &Reconciler{ + client: fake.NewClientBuilder().Build(), + scheme: s, + recorder: record.NewBroadcaster().NewRecorder(scheme.Scheme, corev1.EventSource{Component: "TestReconciler_Reconcile"}), + log: logf.Log.WithName("test"), + } + + if tt.loadFunc != nil { + tt.loadFunc(r.client) + } + + got, err := r.Reconcile(context.TODO(), tt.request) + if (err != nil) != tt.wantErr { + t.Errorf("Reconciler.Reconcile() error = %v, wantErr %v", err, tt.wantErr) + + return + } + + assert.Equal(t, tt.want, got) + if tt.wantFunc != nil { + if err := tt.wantFunc(r.client); err != nil { + t.Errorf("Reconciler.Reconcile() wantFunc validation error: %v", err) + } + } + }) + } +} diff --git a/controllers/podtemplate/doc.go b/controllers/podtemplate/doc.go new file mode 100644 index 00000000..329ddb6a --- /dev/null +++ b/controllers/podtemplate/doc.go @@ -0,0 +1,7 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-present Datadog, Inc. + +// Package podtemplate contains ExtendedDaemonset - PodTemplate controller logic. +package podtemplate diff --git a/controllers/podtemplate_controller.go b/controllers/podtemplate_controller.go new file mode 100644 index 00000000..9b911609 --- /dev/null +++ b/controllers/podtemplate_controller.go @@ -0,0 +1,53 @@ +// Unless explicitly stated otherwise all files in this repository are licensed +// under the Apache License Version 2.0. +// This product includes software developed at Datadog (https://www.datadoghq.com/). +// Copyright 2016-2019 Datadog, Inc. + +package controllers + +import ( + "context" + + "github.com/go-logr/logr" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/record" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + datadoghqv1alpha1 "github.com/DataDog/extendeddaemonset/api/v1alpha1" + "github.com/DataDog/extendeddaemonset/controllers/podtemplate" +) + +// PodTemplateReconciler reconciles a PodTemplate object. +type PodTemplateReconciler struct { + client.Client + Log logr.Logger + Scheme *runtime.Scheme + Recorder record.EventRecorder + Options podtemplate.ReconcilerOptions + internal *podtemplate.Reconciler +} + +// +kubebuilder:rbac:groups=datadoghq.com,resources=extendeddaemonsets,verbs=get;list;watch +// +kubebuilder:rbac:groups="",resources=podtemplates,verbs=get;list;watch;create;update;patch;delete + +// Reconcile loop for PodTemplate. +func (r *PodTemplateReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + return r.internal.Reconcile(ctx, req) +} + +// SetupWithManager creates a new PodTemplate controller. +func (r *PodTemplateReconciler) SetupWithManager(mgr ctrl.Manager) error { + internal, err := podtemplate.NewReconciler(r.Options, r.Client, r.Scheme, r.Log, r.Recorder) + if err != nil { + return err + } + + r.internal = internal + + return ctrl.NewControllerManagedBy(mgr). + For(&datadoghqv1alpha1.ExtendedDaemonSet{}). + Owns(&corev1.PodTemplate{}). + Complete(r) +} diff --git a/controllers/setup.go b/controllers/setup.go index b5356ae3..d7aca4dc 100644 --- a/controllers/setup.go +++ b/controllers/setup.go @@ -15,6 +15,7 @@ import ( "github.com/DataDog/extendeddaemonset/controllers/extendeddaemonset" "github.com/DataDog/extendeddaemonset/controllers/extendeddaemonsetreplicaset" "github.com/DataDog/extendeddaemonset/controllers/extendeddaemonsetsetting" + "github.com/DataDog/extendeddaemonset/controllers/podtemplate" ) // SetupControllers start all controllers (also used by unit and e2e tests). @@ -53,5 +54,15 @@ func SetupControllers(mgr manager.Manager, nodeAffinityMatchSupport bool, defaul return fmt.Errorf("unable to create controller ExtendedDaemonSetReplicaSet: %w", err) } + if err := (&PodTemplateReconciler{ + Client: mgr.GetClient(), + Log: ctrl.Log.WithName("controllers").WithName("PodTemplate"), + Scheme: mgr.GetScheme(), + Recorder: mgr.GetEventRecorderFor("PodTemplate"), + Options: podtemplate.ReconcilerOptions{}, + }).SetupWithManager(mgr); err != nil { + return fmt.Errorf("unable to create controller PodTemplate: %w", err) + } + return nil }