feat(operator): Introduce OTel tracing for Task controller

Signed-off-by: odubajDT <[email protected]>
keptn · Oct 6, 2022 · 9e6ee73 · 9e6ee73
1 parent bc03709
commit 9e6ee73
Show file tree

Hide file tree

Showing 4 changed files with 54 additions and 2 deletions.
diff --git a/operator/api/v1alpha1/semconv/semconv.go b/operator/api/v1alpha1/semconv/semconv.go
@@ -12,6 +12,20 @@ func AddAttributeFromWorkload(s trace.Span, w v1alpha1.KeptnWorkload) {
 	s.SetAttributes(common.Version.String(w.Spec.Version))
 }
 
+func AddAttributeFromWorkloadInstance(s trace.Span, w v1alpha1.KeptnWorkloadInstance) {
+	s.SetAttributes(common.ApplicationName.String(w.Spec.AppName))
+	s.SetAttributes(common.Workload.String(w.Name))
+	s.SetAttributes(common.Version.String(w.Spec.Version))
+}
+
+func AddAttributeFromTask(s trace.Span, t v1alpha1.KeptnTask) {
+	s.SetAttributes(common.ApplicationName.String(t.Spec.AppName))
+	s.SetAttributes(common.Workload.String(t.Spec.Workload))
+	s.SetAttributes(common.Version.String(t.Spec.WorkloadVersion))
+	s.SetAttributes(common.TaskName.String(t.Name))
+	s.SetAttributes(common.TaskType.String(string(t.Spec.Type)))
+}
+
 func AddAttributeFromAnnotations(s trace.Span, annotations map[string]string) {
 	s.SetAttributes(common.ApplicationName.String(annotations[common.AppAnnotation]))
 	s.SetAttributes(common.Workload.String(annotations[common.WorkloadAnnotation]))

diff --git a/operator/controllers/keptntask/controller.go b/operator/controllers/keptntask/controller.go
@@ -24,6 +24,11 @@ import (
 	"github.com/go-logr/logr"
 	klcv1alpha1 "github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1"
 	"github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1/common"
+	"github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1/semconv"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/trace"
 	batchv1 "k8s.io/api/batch/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -41,6 +46,7 @@ type KeptnTaskReconciler struct {
 	Recorder record.EventRecorder
 	Log      logr.Logger
 	Meters   common.KeptnMeters
+	Tracer   trace.Tracer
 }
 
 //+kubebuilder:rbac:groups=lifecycle.keptn.sh,resources=keptntasks,verbs=get;list;watch;create;update;patch;delete
@@ -53,13 +59,22 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	r.Log.Info("Reconciling KeptnTask")
 	task := &klcv1alpha1.KeptnTask{}
 
+	traceContextCarrier := propagation.MapCarrier(task.Annotations)
+	ctx = otel.GetTextMapPropagator().Extract(ctx, traceContextCarrier)
+
+	ctx, span := r.Tracer.Start(ctx, "reconcile_task", trace.WithSpanKind(trace.SpanKindConsumer))
+	defer span.End()
+
+	semconv.AddAttributeFromTask(span, *task)
+
 	if err := r.Client.Get(ctx, req.NamespacedName, task); err != nil {
 		if errors.IsNotFound(err) {
 			// taking down all associated K8s resources is handled by K8s
 			r.Log.Info("KeptnTask resource not found. Ignoring since object must be deleted")
 			return ctrl.Result{}, nil
 		}
 		r.Log.Error(err, "Failed to get the KeptnTask")
+		span.SetStatus(codes.Error, err.Error())
 		return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
 	}
 
@@ -71,18 +86,21 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 
 	err := r.Client.Status().Update(ctx, task)
 	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
 		return ctrl.Result{Requeue: true}, err
 	}
 
 	jobExists, err := r.JobExists(ctx, *task, req.Namespace)
 	if err != nil {
 		r.Log.Error(err, "Could not check if job is running")
+		span.SetStatus(codes.Error, err.Error())
 		return ctrl.Result{Requeue: true, RequeueAfter: 30 * time.Second}, nil
 	}
 
 	if !jobExists {
 		err = r.createJob(ctx, req, task)
 		if err != nil {
+			span.SetStatus(codes.Error, err.Error())
 			return ctrl.Result{Requeue: true}, err
 		}
 		return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil
@@ -91,6 +109,7 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 	if !task.Status.Status.IsCompleted() {
 		err := r.updateJob(ctx, req, task)
 		if err != nil {
+			span.SetStatus(codes.Error, err.Error())
 			return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, err
 		}
 		return ctrl.Result{Requeue: true, RequeueAfter: 10 * time.Second}, nil
@@ -108,6 +127,7 @@ func (r *KeptnTaskReconciler) Reconcile(ctx context.Context, req ctrl.Request) (
 
 	err = r.Client.Status().Update(ctx, task)
 	if err != nil {
+		span.SetStatus(codes.Error, err.Error())
 		return ctrl.Result{Requeue: true}, err
 	}
 

diff --git a/operator/controllers/keptnworkloadinstance/controller.go b/operator/controllers/keptnworkloadinstance/controller.go
@@ -24,6 +24,9 @@ import (
 
 	"github.com/go-logr/logr"
 	"github.com/google/uuid"
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/trace"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
@@ -37,6 +40,7 @@ import (
 	"github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1"
 	klcv1alpha1 "github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1"
 	"github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1/common"
+	"github.com/keptn-sandbox/lifecycle-controller/operator/api/v1alpha1/semconv"
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -56,6 +60,7 @@ type KeptnWorkloadInstanceReconciler struct {
 	Recorder record.EventRecorder
 	Log      logr.Logger
 	Meters   common.KeptnMeters
+	Tracer   trace.Tracer
 }
 
 //+kubebuilder:rbac:groups=lifecycle.keptn.sh,resources=keptnworkloadinstances,verbs=get;list;watch;create;update;patch;delete
@@ -223,10 +228,21 @@ func generateTaskName(checkType common.CheckType, taskName string) string {
 }
 
 func (r *KeptnWorkloadInstanceReconciler) createKeptnTask(ctx context.Context, namespace string, workloadInstance *klcv1alpha1.KeptnWorkloadInstance, taskDefinition string, checkType common.CheckType) (string, error) {
+	ctx, span := r.Tracer.Start(ctx, "create_task", trace.WithSpanKind(trace.SpanKindProducer))
+	defer span.End()
+
+	semconv.AddAttributeFromWorkloadInstance(span, *workloadInstance)
+
+	// create TraceContext
+	// follow up with a Keptn propagator that JSON-encoded the OTel map into our own key
+	traceContextCarrier := propagation.MapCarrier{}
+	otel.GetTextMapPropagator().Inject(ctx, traceContextCarrier)
+
 	newTask := &klcv1alpha1.KeptnTask{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      generateTaskName(checkType, taskDefinition),
-			Namespace: namespace,
+			Annotations: traceContextCarrier,
+			Name:        generateTaskName(checkType, taskDefinition),
+			Namespace:   namespace,
 		},
 		Spec: klcv1alpha1.KeptnTaskSpec{
 			Workload:         workloadInstance.Spec.WorkloadName,

diff --git a/operator/main.go b/operator/main.go
@@ -203,6 +203,7 @@ func main() {
 		Log:      ctrl.Log.WithName("KeptnTask Controller"),
 		Recorder: mgr.GetEventRecorderFor("keptntask-controller"),
 		Meters:   meters,
+		Tracer:   otel.Tracer("keptn/operator/task"),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "KeptnTask")
 		os.Exit(1)
@@ -239,6 +240,7 @@ func main() {
 		Log:      ctrl.Log.WithName("KeptnWorkloadInstance Controller"),
 		Recorder: mgr.GetEventRecorderFor("keptnworkloadinstance-controller"),
 		Meters:   meters,
+		Tracer:   otel.Tracer("keptn/operator/workload_instance"),
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "KeptnWorkloadInstance")
 		os.Exit(1)