From 0baf7a9d8058877247bc264eb6fdb645b0a77a60 Mon Sep 17 00:00:00 2001 From: odubajDT <93584209+odubajDT@users.noreply.github.com> Date: Fri, 7 Oct 2022 10:50:50 +0200 Subject: [PATCH] feat(operator): Introduce OTel tracing for Task controller (#128) Signed-off-by: odubajDT --- operator/api/v1alpha1/semconv/semconv.go | 8 ++++++++ operator/controllers/keptntask/controller.go | 19 +++++++++++++++++++ operator/main.go | 1 + 3 files changed, 28 insertions(+) diff --git a/operator/api/v1alpha1/semconv/semconv.go b/operator/api/v1alpha1/semconv/semconv.go index b22cc41c9f..525079413b 100644 --- a/operator/api/v1alpha1/semconv/semconv.go +++ b/operator/api/v1alpha1/semconv/semconv.go @@ -18,6 +18,14 @@ func AddAttributeFromWorkloadInstance(s trace.Span, w v1alpha1.KeptnWorkloadInst s.SetAttributes(common.Version.String(w.Spec.Version)) } +func AddAttributeFromTask(s trace.Span, t v1alpha1.KeptnTask) { + s.SetAttributes(common.ApplicationName.String(t.Spec.AppName)) + s.SetAttributes(common.Workload.String(t.Spec.Workload)) + s.SetAttributes(common.Version.String(t.Spec.WorkloadVersion)) + s.SetAttributes(common.TaskName.String(t.Name)) + s.SetAttributes(common.TaskType.String(string(t.Spec.Type))) +} + func AddAttributeFromAnnotations(s trace.Span, annotations map[string]string) { s.SetAttributes(common.ApplicationName.String(annotations[common.AppAnnotation])) s.SetAttributes(common.Workload.String(annotations[common.WorkloadAnnotation])) diff --git a/operator/controllers/keptntask/controller.go b/operator/controllers/keptntask/controller.go index a79ebadc60..3f34fa6201 100644 --- a/operator/controllers/keptntask/controller.go +++ b/operator/controllers/keptntask/controller.go @@ -24,6 +24,11 @@ import ( "github.com/go-logr/logr" klcv1alpha1 "github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1" "github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1/common" + "github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1/semconv" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/propagation" + "go.opentelemetry.io/otel/trace" batchv1 "k8s.io/api/batch/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" @@ -41,6 +46,7 @@ type KeptnTaskReconciler struct { Recorder record.EventRecorder Log logr.Logger Meters common.KeptnMeters + Tracer trace.Tracer } //+kubebuilder:rbac:groups=lifecycle.keptn.sh,resources=keptntasks,verbs=get;list;watch;create;update;patch;delete @@ -63,6 +69,14 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil } + traceContextCarrier := propagation.MapCarrier(task.Annotations) + ctx = otel.GetTextMapPropagator().Extract(ctx, traceContextCarrier) + + ctx, span := r.Tracer.Start(ctx, "reconcile_task", trace.WithSpanKind(trace.SpanKindConsumer)) + defer span.End() + + semconv.AddAttributeFromTask(span, *task) + if !task.IsStartTimeSet() { // metrics: increment active task counter r.Meters.TaskActive.Add(ctx, 1, task.GetActiveMetricsAttributes()...) @@ -71,18 +85,21 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( err := r.Client.Status().Update(ctx, task) if err != nil { + span.SetStatus(codes.Error, err.Error()) return ctrl.Result{Requeue: true}, err } jobExists, err := r.JobExists(ctx, *task, req.Namespace) if err != nil { r.Log.Error(err, "Could not check if job is running") + span.SetStatus(codes.Error, err.Error()) return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil } if !jobExists { err = r.createJob(ctx, req, task) if err != nil { + span.SetStatus(codes.Error, err.Error()) return ctrl.Result{Requeue: true}, err } return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil @@ -91,6 +108,7 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( if !task.Status.Status.IsCompleted() { err := r.updateJob(ctx, req, task) if err != nil { + span.SetStatus(codes.Error, err.Error()) return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, err } return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil @@ -108,6 +126,7 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) ( err = r.Client.Status().Update(ctx, task) if err != nil { + span.SetStatus(codes.Error, err.Error()) return ctrl.Result{Requeue: true}, err } diff --git a/operator/main.go b/operator/main.go index 026723b44d..8943696842 100644 --- a/operator/main.go +++ b/operator/main.go @@ -203,6 +203,7 @@ func main() { Log: ctrl.Log.WithName("KeptnTask Controller"), Recorder: mgr.GetEventRecorderFor("keptntask-controller"), Meters: meters, + Tracer: otel.Tracer("keptn/operator/task"), }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "KeptnTask") os.Exit(1)