add remoteWrite.status field (#935)

* add remoteWrite.status field

* make remoteWrite.status optional

* patch remoteWrite status

* fmt.Sprintf

* rbac add remotewrites/status

* try status update

* remove patch

* debug log

* debug objectKind

* populate remoteWrite kind

* update test

* try gvk

* try RESTClient

* fix status

* status subresource

* Revert "fix status"

This reverts commit 456613d.

* Revert "try RESTClient"

This reverts commit 84e358f.

* Revert "try gvk"

This reverts commit 5e9a85d.

* replace remoteWrite.Status.ConfiguredPrometheuses with corev1.ObjectReference

* add ensureStatus

* add ctx, fix err

* lol ci

* implements cleanup logic

* Apply suggestions

* Update changelog

* Implement cleanup of secrets once selector has changed

* Add Check for ensureCleanupSecretsv

* Test Convert ToRemoteWrite

* Refactor Cleanup of secrets

* Remove unsed code

Co-authored-by: Mohamed Chiheb <[email protected]>
Co-authored-by: Quentin Bisson <[email protected]>

CHANGELOG.md

@@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Add unit tests for remotewrite resource
 - Add Secrets field in the RemoteWrite CR
 - Implement sync RemoteWrite Secrets logic
+- Adding RemoteWrite.status field to ensure cleanup
 - Add psp and service account for prometheus and alertmanager
 
 ### Changed

api/v1alpha1/remoteWrite_types.go

@@ -2,16 +2,20 @@ package v1alpha1
 
 import (
 	pov1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 // +kubebuilder:resource:categories=common;giantswarm
+// +kubebuilder:subresource:status
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // RemoteWrite represents schema for managed RemoteWrites in Prometheus. Reconciled by prometheus-meta-operator.
 type RemoteWrite struct {
 	metav1.TypeMeta   `json:",inline"`
 	metav1.ObjectMeta `json:"metadata"`
 	Spec              RemoteWriteSpec `json:"spec"`
+	// +kubebuilder:validation:Optional
+	Status RemoteWriteStatus `json:"status"`
 }
 
 type RemoteWriteSpec struct {
@@ -32,6 +36,11 @@ type RemoteWriteSecretSpec struct {
 	Data map[string][]byte `json:"data,omitempty"`
 }
 
+type RemoteWriteStatus struct {
+	ConfiguredPrometheuses []corev1.ObjectReference `json:"configuredPrometheuses,omitempty"`
+	SyncedSecrets          []corev1.ObjectReference `json:"syncedSecrets,omitempty"`
+}
+
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 type RemoteWriteList struct {
 	metav1.TypeMeta `json:",inline"`

config/crd/monitoring.giantswarm.io_remotewrites.yaml

@@ -580,12 +580,141 @@ spec:
             - clusterSelector
             - remoteWrite
             type: object
+          status:
+            properties:
+              configuredPrometheuses:
+                items:
+                  description: 'ObjectReference contains enough information to let
+                    you inspect or modify the referred object. --- New uses of this
+                    type are discouraged because of difficulty describing its usage
+                    when embedded in APIs. 1. Ignored fields.  It includes many fields
+                    which are not generally honored.  For instance, ResourceVersion
+                    and FieldPath are both very rarely valid in actual usage. 2. Invalid
+                    usage help.  It is impossible to add specific help for individual
+                    usage.  In most embedded usages, there are particular restrictions
+                    like, "must refer only to types A and B" or "UID not honored"
+                    or "name must be restricted". Those cannot be well described when
+                    embedded. 3. Inconsistent validation.  Because the usages are
+                    different, the validation rules are different by usage, which
+                    makes it hard for users to predict what will happen. 4. The fields
+                    are both imprecise and overly precise.  Kind is not a precise
+                    mapping to a URL. This can produce ambiguity during interpretation
+                    and require a REST mapping.  In most cases, the dependency is
+                    on the group,resource tuple and the version of the actual struct
+                    is irrelevant. 5. We cannot easily change it.  Because this type
+                    is embedded in many locations, updates to this type will affect
+                    numerous schemas.  Don''t make new APIs embed an underspecified
+                    API type they do not control. Instead of using this type, create
+                    a locally provided and used type that is well-focused on your
+                    reference. For example, ServiceReferences for admission registration:
+                    https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533
+                    .'
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                        an entire object, this string should contain a valid JSON/Go
+                        field access statement, such as desiredState.manifest.containers[2].
+                        For example, if the object reference is to a container within
+                        a pod, this would take on a value like: "spec.containers{name}"
+                        (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]"
+                        (container with index 2 in this pod). This syntax is chosen
+                        only to have some well-defined way of referencing a part of
+                        an object. TODO: this design is not final and this field is
+                        subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+                type: array
+              syncedSecrets:
+                items:
+                  description: 'ObjectReference contains enough information to let
+                    you inspect or modify the referred object. --- New uses of this
+                    type are discouraged because of difficulty describing its usage
+                    when embedded in APIs. 1. Ignored fields.  It includes many fields
+                    which are not generally honored.  For instance, ResourceVersion
+                    and FieldPath are both very rarely valid in actual usage. 2. Invalid
+                    usage help.  It is impossible to add specific help for individual
+                    usage.  In most embedded usages, there are particular restrictions
+                    like, "must refer only to types A and B" or "UID not honored"
+                    or "name must be restricted". Those cannot be well described when
+                    embedded. 3. Inconsistent validation.  Because the usages are
+                    different, the validation rules are different by usage, which
+                    makes it hard for users to predict what will happen. 4. The fields
+                    are both imprecise and overly precise.  Kind is not a precise
+                    mapping to a URL. This can produce ambiguity during interpretation
+                    and require a REST mapping.  In most cases, the dependency is
+                    on the group,resource tuple and the version of the actual struct
+                    is irrelevant. 5. We cannot easily change it.  Because this type
+                    is embedded in many locations, updates to this type will affect
+                    numerous schemas.  Don''t make new APIs embed an underspecified
+                    API type they do not control. Instead of using this type, create
+                    a locally provided and used type that is well-focused on your
+                    reference. For example, ServiceReferences for admission registration:
+                    https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533
+                    .'
+                  properties:
+                    apiVersion:
+                      description: API version of the referent.
+                      type: string
+                    fieldPath:
+                      description: 'If referring to a piece of an object instead of
+                        an entire object, this string should contain a valid JSON/Go
+                        field access statement, such as desiredState.manifest.containers[2].
+                        For example, if the object reference is to a container within
+                        a pod, this would take on a value like: "spec.containers{name}"
+                        (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]"
+                        (container with index 2 in this pod). This syntax is chosen
+                        only to have some well-defined way of referencing a part of
+                        an object. TODO: this design is not final and this field is
+                        subject to change in the future.'
+                      type: string
+                    kind:
+                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      type: string
+                    name:
+                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      type: string
+                    namespace:
+                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      type: string
+                    resourceVersion:
+                      description: 'Specific resourceVersion to which this reference
+                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                      type: string
+                    uid:
+                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      type: string
+                  type: object
+                type: array
+            type: object
         required:
         - metadata
         - spec
         type: object
     served: true
     storage: true
+    subresources:
+      status: {}
 status:
   acceptedNames:
     kind: ""

helm/prometheus-meta-operator/templates/rbac.yaml

@@ -156,6 +156,14 @@ rules:
       - list
       - patch
       - watch
+  - apiGroups:
+      - monitoring.giantswarm.io
+    resources:
+      - remotewrites/status
+    verbs:
+      - get
+      - patch
+      - update
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding

service/controller/resource/prometheusremotewrite/create.go

@@ -5,9 +5,11 @@ import (
 	"fmt"
 
 	"github.com/giantswarm/microerror"
-	"github.com/giantswarm/operatorkit/v7/pkg/controller/context/resourcecanceledcontext"
+	promv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
+	pmov1alpha1 "github.com/giantswarm/prometheus-meta-operator/api/v1alpha1"
 	"github.com/giantswarm/prometheus-meta-operator/pkg/remotewriteutils"
 )
 
@@ -24,31 +26,18 @@ func (r *Resource) EnsureCreated(ctx context.Context, obj interface{}) error {
 		if err != nil {
 			return microerror.Mask(err)
 		}
-		if prometheusList == nil || len(prometheusList.Items) == 0 {
-			r.logger.Debugf(ctx, "no prometheus found, cancel reconciliation")
-			resourcecanceledcontext.SetCanceled(ctx)
-			return nil
-		}
 
 		for _, current := range prometheusList.Items {
-
-			desired, ok := r.ensurePrometheusRemoteWrite(*remoteWrite, *current)
-			if !ok {
-				r.logger.Debugf(ctx, fmt.Sprintf("no update required for Prometheus CR %#q in namespace %#q", desired.Name, desired.Namespace))
-				continue
-			}
-
-			r.logger.Debugf(ctx, fmt.Sprintf("updating Prometheus CR %#q in namespace %#q", desired.Name, desired.Namespace))
-
-			updateMeta(current, desired)
-			_, err = r.prometheusClient.MonitoringV1().
-				Prometheuses(current.GetNamespace()).
-				Update(ctx, desired, metav1.UpdateOptions{})
+			err = r.setRemoteWrite(ctx, remoteWrite, current)
 			if err != nil {
 				return microerror.Mask(err)
 			}
 		}
 
+		err = r.ensureCleanUp(ctx, remoteWrite, prometheusList.Items)
+		if err != nil {
+			return microerror.Mask(err)
+		}
 	}
 
 	r.logger.Debugf(ctx, "ensured prometheus remoteWrite config")
@@ -66,3 +55,82 @@ func updateMeta(c, d metav1.Object) {
 	d.SetDeletionGracePeriodSeconds(c.GetDeletionGracePeriodSeconds())
 	d.SetManagedFields(c.GetManagedFields())
 }
+
+func (r *Resource) setRemoteWrite(ctx context.Context, remoteWrite *pmov1alpha1.RemoteWrite, prometheus *promv1.Prometheus) error {
+
+	desired, ok := r.ensurePrometheusRemoteWrite(*remoteWrite, *prometheus)
+	if !ok {
+		r.logger.Debugf(ctx, fmt.Sprintf("no update required for Prometheus CR %#q in namespace %#q", desired.Name, desired.Namespace))
+		return nil
+	}
+
+	r.logger.Debugf(ctx, fmt.Sprintf("updating Prometheus CR %#q in namespace %#q", desired.Name, desired.Namespace))
+
+	updateMeta(prometheus, desired)
+	_, err := r.prometheusClient.MonitoringV1().
+		Prometheuses(prometheus.GetNamespace()).
+		Update(ctx, desired, metav1.UpdateOptions{})
+	if err != nil {
+		return microerror.Mask(err)
+	}
+	err = r.ensureStatusCreated(ctx, remoteWrite, prometheus)
+	if err != nil {
+		return microerror.Mask(err)
+	}
+
+	return nil
+}
+
+func (r *Resource) ensureStatusCreated(ctx context.Context, remoteWrite *pmov1alpha1.RemoteWrite, prometheus *promv1.Prometheus) error {
+	for _, ref := range remoteWrite.Status.ConfiguredPrometheuses {
+		if ref.Name == prometheus.GetName() && ref.Namespace == prometheus.GetNamespace() {
+			return nil
+		}
+	}
+
+	newStatus := corev1.ObjectReference{
+		Name:      prometheus.GetName(),
+		Namespace: prometheus.GetNamespace(),
+	}
+	remoteWrite.Status.ConfiguredPrometheuses = append(remoteWrite.Status.ConfiguredPrometheuses, newStatus)
+
+	err := r.k8sClient.CtrlClient().Status().Update(ctx, remoteWrite)
+	if err != nil {
+		return microerror.Mask(err)
+	}
+
+	return nil
+}
+
+func (r *Resource) ensureCleanUp(ctx context.Context, remoteWrite *pmov1alpha1.RemoteWrite, prometheuses []*promv1.Prometheus) error {
+	// Copy the statuses, because it will be overwritten later on.
+	statuses := remoteWrite.Status.ConfiguredPrometheuses
+
+	for _, statusRef := range statuses {
+		if !inList(statusRef, prometheuses) {
+			p, err := r.prometheusClient.MonitoringV1().
+				Prometheuses(statusRef.Namespace).
+				Get(ctx, statusRef.Name, metav1.GetOptions{})
+			if err != nil {
+				return microerror.Mask(err)
+			}
+
+			err = r.unsetRemoteWrite(ctx, remoteWrite, p)
+			if err != nil {
+				return microerror.Mask(err)
+			}
+		}
+	}
+
+	return nil
+}
+
+func inList(o corev1.ObjectReference, list []*promv1.Prometheus) bool {
+	for _, item := range list {
+		if o.Name == item.GetName() && o.Namespace == item.GetNamespace() {
+			return true
+		}
+	}
+
+	return false
+}