From 9e6ee73b7a09d3fd74f5ca3113d6d44c8c3435ff Mon Sep 17 00:00:00 2001 From: odubajDT Date: Thu, 6 Oct 2022 16:01:40 +0200 Subject: [PATCH] feat(operator): Introduce OTel tracing for Task controller Signed-off-by: odubajDT --- operator/api/v1alpha1/semconv/semconv.go | 14 +++++++++++++ operator/controllers/keptntask/controller.go | 20 +++++++++++++++++++ .../keptnworkloadinstance/controller.go | 20 +++++++++++++++++-- operator/main.go | 2 ++ 4 files changed, 54 insertions(+), 2 deletions(-) diff --git a/operator/api/v1alpha1/semconv/semconv.go b/operator/api/v1alpha1/semconv/semconv.go index 7b9243b30cf..525079413b7 100644 --- a/operator/api/v1alpha1/semconv/semconv.go +++ b/operator/api/v1alpha1/semconv/semconv.go @@ -12,6 +12,20 @@ func AddAttributeFromWorkload(s trace.Span, w v1alpha1.KeptnWorkload) { s.SetAttributes(common.Version.String(w.Spec.Version)) } +func AddAttributeFromWorkloadInstance(s trace.Span, w v1alpha1.KeptnWorkloadInstance) { + s.SetAttributes(common.ApplicationName.String(w.Spec.AppName)) + s.SetAttributes(common.Workload.String(w.Name)) + s.SetAttributes(common.Version.String(w.Spec.Version)) +} + +func AddAttributeFromTask(s trace.Span, t v1alpha1.KeptnTask) { + s.SetAttributes(common.ApplicationName.String(t.Spec.AppName)) + s.SetAttributes(common.Workload.String(t.Spec.Workload)) + s.SetAttributes(common.Version.String(t.Spec.WorkloadVersion)) + s.SetAttributes(common.TaskName.String(t.Name)) + s.SetAttributes(common.TaskType.String(string(t.Spec.Type))) +} + func AddAttributeFromAnnotations(s trace.Span, annotations map[string]string) { s.SetAttributes(common.ApplicationName.String(annotations[common.AppAnnotation])) s.SetAttributes(common.Workload.String(annotations[common.WorkloadAnnotation])) diff --git a/operator/controllers/keptntask/controller.go b/operator/controllers/keptntask/controller.go index a79ebadc60d..bc2c4fa42ca 100644 --- a/operator/controllers/keptntask/controller.go +++ b/operator/controllers/keptntask/controller.go @@ -24,6 +24,11 @@ import ( "github.com/go-logr/logr" klcv1alpha1 "github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1" "github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1/common" + "github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1/semconv" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" batchv1 "k8s.io/api/batch/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -41,6 +46,7 @@ type KeptnTaskReconciler struct { Recorder record.EventRecorder Log logr.Logger Meters common.KeptnMeters + Tracer trace.Tracer } //+kubebuilder:rbac:groups=lifecycle.keptn.sh,resources=keptntasks,verbs=get;list;watch;create;update;patch;delete @@ -53,6 +59,14 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( r.Log.Info("Reconciling KeptnTask") task := &klcv1alpha1.KeptnTask{} + traceContextCarrier := propagation.MapCarrier(task.Annotations) + ctx = otel.GetTextMapPropagator().Extract(ctx, traceContextCarrier) + + ctx, span := r.Tracer.Start(ctx, "reconcile_task", trace.WithSpanKind(trace.SpanKindConsumer)) + defer span.End() + + semconv.AddAttributeFromTask(span, *task) + if err := r.Client.Get(ctx, req.NamespacedName, task); err != nil { if errors.IsNotFound(err) { // taking down all associated K8s resources is handled by K8s @@ -60,6 +74,7 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{}, nil } r.Log.Error(err, "Failed to get the KeptnTask") + span.SetStatus(codes.Error, err.Error()) return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil } @@ -71,18 +86,21 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( err := r.Client.Status().Update(ctx, task) if err != nil { + span.SetStatus(codes.Error, err.Error()) return ctrl.Result{Requeue: true}, err } jobExists, err := r.JobExists(ctx, *task, req.Namespace) if err != nil { r.Log.Error(err, "Could not check if job is running") + span.SetStatus(codes.Error, err.Error()) return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil } if !jobExists { err = r.createJob(ctx, req, task) if err != nil { + span.SetStatus(codes.Error, err.Error()) return ctrl.Result{Requeue: true}, err } return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil @@ -91,6 +109,7 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if !task.Status.Status.IsCompleted() { err := r.updateJob(ctx, req, task) if err != nil { + span.SetStatus(codes.Error, err.Error()) return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, err } return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil @@ -108,6 +127,7 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( err = r.Client.Status().Update(ctx, task) if err != nil { + span.SetStatus(codes.Error, err.Error()) return ctrl.Result{Requeue: true}, err } diff --git a/operator/controllers/keptnworkloadinstance/controller.go b/operator/controllers/keptnworkloadinstance/controller.go index d06bbc879de..31c4bbc1a7b 100644 --- a/operator/controllers/keptnworkloadinstance/controller.go +++ b/operator/controllers/keptnworkloadinstance/controller.go @@ -24,6 +24,9 @@ import ( "github.com/go-logr/logr" "github.com/google/uuid" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -37,6 +40,7 @@ import ( "github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1" klcv1alpha1 "github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1" "github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1/common" + "github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1/semconv" "k8s.io/apimachinery/pkg/runtime" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" @@ -56,6 +60,7 @@ type KeptnWorkloadInstanceReconciler struct { Recorder record.EventRecorder Log logr.Logger Meters common.KeptnMeters + Tracer trace.Tracer } //+kubebuilder:rbac:groups=lifecycle.keptn.sh,resources=keptnworkloadinstances,verbs=get;list;watch;create;update;patch;delete @@ -223,10 +228,21 @@ func generateTaskName(checkType common.CheckType, taskName string) string { } func (r *KeptnWorkloadInstanceReconciler) createKeptnTask(ctx context.Context, namespace string, workloadInstance *klcv1alpha1.KeptnWorkloadInstance, taskDefinition string, checkType common.CheckType) (string, error) { + ctx, span := r.Tracer.Start(ctx, "create_task", trace.WithSpanKind(trace.SpanKindProducer)) + defer span.End() + + semconv.AddAttributeFromWorkloadInstance(span, *workloadInstance) + + // create TraceContext + // follow up with a Keptn propagator that JSON-encoded the OTel map into our own key + traceContextCarrier := propagation.MapCarrier{} + otel.GetTextMapPropagator().Inject(ctx, traceContextCarrier) + newTask := &klcv1alpha1.KeptnTask{ ObjectMeta: metav1.ObjectMeta{ - Name: generateTaskName(checkType, taskDefinition), - Namespace: namespace, + Annotations: traceContextCarrier, + Name: generateTaskName(checkType, taskDefinition), + Namespace: namespace, }, Spec: klcv1alpha1.KeptnTaskSpec{ Workload: workloadInstance.Spec.WorkloadName, diff --git a/operator/main.go b/operator/main.go index 8dfe95e6ea5..f1a91917625 100644 --- a/operator/main.go +++ b/operator/main.go @@ -203,6 +203,7 @@ func main() { Log: ctrl.Log.WithName("KeptnTask Controller"), Recorder: mgr.GetEventRecorderFor("keptntask-controller"), Meters: meters, + Tracer: otel.Tracer("keptn/operator/task"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "KeptnTask") os.Exit(1) @@ -239,6 +240,7 @@ func main() { Log: ctrl.Log.WithName("KeptnWorkloadInstance Controller"), Recorder: mgr.GetEventRecorderFor("keptnworkloadinstance-controller"), Meters: meters, + Tracer: otel.Tracer("keptn/operator/workload_instance"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "KeptnWorkloadInstance") os.Exit(1)