feat(operator): rework Task and Evaluation span structure

operator/api/v1alpha1/common/common.go

@@ -64,47 +64,47 @@ func (k KeptnState) IsPending() bool {
 
 type StatusSummary struct {
 	Total       int
-	progressing int
-	failed      int
-	succeeded   int
-	pending     int
-	unknown     int
-	cancelled   int
+	Progressing int
+	Failed      int
+	Succeeded   int
+	Pending     int
+	Unknown     int
+	Cancelled   int
 }
 
 func UpdateStatusSummary(status KeptnState, summary StatusSummary) StatusSummary {
 	switch status {
 	case StateFailed:
-		summary.failed++
+		summary.Failed++
 	case StateCancelled:
-		summary.cancelled++
+		summary.Cancelled++
 	case StateSucceeded:
-		summary.succeeded++
+		summary.Succeeded++
 	case StateProgressing:
-		summary.progressing++
+		summary.Progressing++
 	case StatePending, "":
-		summary.pending++
+		summary.Pending++
 	case StateUnknown:
-		summary.unknown++
+		summary.Unknown++
 	}
 	return summary
 }
 
 func (s StatusSummary) GetTotalCount() int {
-	return s.failed + s.succeeded + s.progressing + s.pending + s.unknown + s.cancelled
+	return s.Failed + s.Succeeded + s.Progressing + s.Pending + s.Unknown + s.Cancelled
 }
 
 func GetOverallState(s StatusSummary) KeptnState {
-	if s.failed > 0 || s.cancelled > 0 {
+	if s.Failed > 0 || s.Cancelled > 0 {
 		return StateFailed
 	}
-	if s.progressing > 0 {
+	if s.Progressing > 0 {
 		return StateProgressing
 	}
-	if s.pending > 0 {
+	if s.Pending > 0 {
 		return StatePending
 	}
-	if s.unknown > 0 || s.GetTotalCount() != s.Total {
+	if s.Unknown > 0 || s.GetTotalCount() != s.Total {
 		return StateUnknown
 	}
 	return StateSucceeded

operator/api/v1alpha1/keptnappversion_types.go

@@ -299,12 +299,11 @@ func (a KeptnAppVersion) GetVersion() string {
 	return a.Spec.Version
 }
 
-func (a KeptnAppVersion) GenerateTask(traceContextCarrier propagation.MapCarrier, taskDefinition string, checkType common.CheckType) KeptnTask {
+func (a KeptnAppVersion) GenerateTask(taskDefinition string, checkType common.CheckType) KeptnTask {
 	return KeptnTask{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:        common.GenerateTaskName(checkType, taskDefinition),
-			Namespace:   a.Namespace,
-			Annotations: traceContextCarrier,
+			Name:      common.GenerateTaskName(checkType, taskDefinition),
+			Namespace: a.Namespace,
 		},
 		Spec: KeptnTaskSpec{
 			AppVersion:       a.GetVersion(),
@@ -317,12 +316,11 @@ func (a KeptnAppVersion) GenerateTask(traceContextCarrier propagation.MapCarrier
 	}
 }
 
-func (a KeptnAppVersion) GenerateEvaluation(traceContextCarrier propagation.MapCarrier, evaluationDefinition string, checkType common.CheckType) KeptnEvaluation {
+func (a KeptnAppVersion) GenerateEvaluation(evaluationDefinition string, checkType common.CheckType) KeptnEvaluation {
 	return KeptnEvaluation{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:        common.GenerateEvaluationName(checkType, evaluationDefinition),
-			Namespace:   a.Namespace,
-			Annotations: traceContextCarrier,
+			Name:      common.GenerateEvaluationName(checkType, evaluationDefinition),
+			Namespace: a.Namespace,
 		},
 		Spec: KeptnEvaluationSpec{
 			AppVersion:           a.Spec.Version,

operator/api/v1alpha1/keptnevaluation_types.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/keptn/lifecycle-toolkit/operator/api/v1alpha1/common"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/trace"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -175,3 +176,15 @@ func (e KeptnEvaluation) GetSpanAttributes() []attribute.KeyValue {
 		common.EvaluationType.String(string(e.Spec.Type)),
 	}
 }
+
+func (e *KeptnEvaluation) SetPhaseTraceID(phase string, carrier propagation.MapCarrier) {
+	// present due to SpanItem interface
+}
+
+func (e KeptnEvaluation) GetSpanKey(phase string) string {
+	return e.Name
+}
+
+func (e KeptnEvaluation) GetSpanName(phase string) string {
+	return e.Name
+}

operator/api/v1alpha1/keptntask_types.go

@@ -21,6 +21,7 @@ import (
 
 	"github.com/keptn/lifecycle-toolkit/operator/api/v1alpha1/common"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/trace"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -183,3 +184,15 @@ func (t KeptnTask) GetSpanAttributes() []attribute.KeyValue {
 		common.TaskType.String(string(t.Spec.Type)),
 	}
 }
+
+func (t *KeptnTask) SetPhaseTraceID(phase string, carrier propagation.MapCarrier) {
+	// present due to SpanItem interface
+}
+
+func (t KeptnTask) GetSpanKey(phase string) string {
+	return t.Name
+}
+
+func (t KeptnTask) GetSpanName(phase string) string {
+	return t.Name
+}

operator/api/v1alpha1/keptnworkloadinstance_types.go

@@ -336,12 +336,11 @@ func (w KeptnWorkloadInstance) GetVersion() string {
 	return w.Spec.Version
 }
 
-func (w KeptnWorkloadInstance) GenerateTask(traceContextCarrier propagation.MapCarrier, taskDefinition string, checkType common.CheckType) KeptnTask {
+func (w KeptnWorkloadInstance) GenerateTask(taskDefinition string, checkType common.CheckType) KeptnTask {
 	return KeptnTask{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:        common.GenerateTaskName(checkType, taskDefinition),
-			Namespace:   w.Namespace,
-			Annotations: traceContextCarrier,
+			Name:      common.GenerateTaskName(checkType, taskDefinition),
+			Namespace: w.Namespace,
 		},
 		Spec: KeptnTaskSpec{
 			AppName:          w.GetAppName(),
@@ -355,12 +354,11 @@ func (w KeptnWorkloadInstance) GenerateTask(traceContextCarrier propagation.MapC
 	}
 }
 
-func (w KeptnWorkloadInstance) GenerateEvaluation(traceContextCarrier propagation.MapCarrier, evaluationDefinition string, checkType common.CheckType) KeptnEvaluation {
+func (w KeptnWorkloadInstance) GenerateEvaluation(evaluationDefinition string, checkType common.CheckType) KeptnEvaluation {
 	return KeptnEvaluation{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:        common.GenerateEvaluationName(checkType, evaluationDefinition),
-			Namespace:   w.Namespace,
-			Annotations: traceContextCarrier,
+			Name:      common.GenerateEvaluationName(checkType, evaluationDefinition),
+			Namespace: w.Namespace,
 		},
 		Spec: KeptnEvaluationSpec{
 			AppName:              w.GetAppName(),

operator/api/v1alpha1/tests/keptnappversion_test.go

@@ -179,7 +179,7 @@ func TestKeptnAppVersion(t *testing.T) {
 
 	require.Equal(t, "trace1.appname.version.phase", app.GetSpanKey("phase"))
 
-	task := app.GenerateTask(map[string]string{}, "taskdef", common.PostDeploymentCheckType)
+	task := app.GenerateTask("taskdef", common.PostDeploymentCheckType)
 	require.Equal(t, v1alpha1.KeptnTaskSpec{
 		AppVersion:       app.GetVersion(),
 		AppName:          app.GetParentName(),
@@ -189,7 +189,7 @@ func TestKeptnAppVersion(t *testing.T) {
 		Type:             common.PostDeploymentCheckType,
 	}, task.Spec)
 
-	evaluation := app.GenerateEvaluation(map[string]string{}, "taskdef", common.PostDeploymentCheckType)
+	evaluation := app.GenerateEvaluation("taskdef", common.PostDeploymentCheckType)
 	require.Equal(t, v1alpha1.KeptnEvaluationSpec{
 		AppVersion:           app.GetVersion(),
 		AppName:              app.GetParentName(),

operator/api/v1alpha1/tests/keptnevaluation_test.go

@@ -25,6 +25,24 @@ func TestKeptnEvaluation(t *testing.T) {
 		},
 	}
 
+	evaluation.SetPhaseTraceID("", nil)
+	require.Equal(t, v1alpha1.KeptnEvaluation{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "evaluation",
+		},
+		Spec: v1alpha1.KeptnEvaluationSpec{
+			AppName:    "app",
+			AppVersion: "appversion",
+			Type:       common.PostDeploymentCheckType,
+		},
+		Status: v1alpha1.KeptnEvaluationStatus{
+			OverallStatus: common.StateFailed,
+		},
+	}, *evaluation)
+
+	require.Equal(t, "evaluation", evaluation.GetSpanKey(""))
+	require.Equal(t, "evaluation", evaluation.GetSpanName(""))
+
 	require.False(t, evaluation.IsEndTimeSet())
 	require.False(t, evaluation.IsStartTimeSet())
 

operator/api/v1alpha1/tests/keptntask_test.go

@@ -25,6 +25,24 @@ func TestKeptnTask(t *testing.T) {
 		},
 	}
 
+	task.SetPhaseTraceID("", nil)
+	require.Equal(t, v1alpha1.KeptnTask{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "task",
+		},
+		Spec: v1alpha1.KeptnTaskSpec{
+			AppName:    "app",
+			AppVersion: "appversion",
+			Type:       common.PostDeploymentCheckType,
+		},
+		Status: v1alpha1.KeptnTaskStatus{
+			Status: common.StateFailed,
+		},
+	}, *task)
+
+	require.Equal(t, "task", task.GetSpanKey(""))
+	require.Equal(t, "task", task.GetSpanName(""))
+
 	require.False(t, task.IsEndTimeSet())
 	require.False(t, task.IsStartTimeSet())
 

operator/api/v1alpha1/tests/keptnworkloadinstance_test.go

@@ -183,7 +183,7 @@ func TestKeptnWorkloadInstance(t *testing.T) {
 
 	require.Equal(t, "trace1.workloadname.version.phase", workload.GetSpanKey("phase"))
 
-	task := workload.GenerateTask(map[string]string{}, "taskdef", common.PostDeploymentCheckType)
+	task := workload.GenerateTask("taskdef", common.PostDeploymentCheckType)
 	require.Equal(t, v1alpha1.KeptnTaskSpec{
 		AppName:          workload.GetAppName(),
 		WorkloadVersion:  workload.GetVersion(),
@@ -194,7 +194,7 @@ func TestKeptnWorkloadInstance(t *testing.T) {
 		Type:             common.PostDeploymentCheckType,
 	}, task.Spec)
 
-	evaluation := workload.GenerateEvaluation(map[string]string{}, "taskdef", common.PostDeploymentCheckType)
+	evaluation := workload.GenerateEvaluation("taskdef", common.PostDeploymentCheckType)
 	require.Equal(t, v1alpha1.KeptnEvaluationSpec{
 		AppName:              workload.GetAppName(),
 		WorkloadVersion:      workload.GetVersion(),

operator/controllers/common/errors.go

@@ -6,6 +6,7 @@ var ErrCannotWrapToPhaseItem = fmt.Errorf("provided object does not implement Ph
 var ErrCannotWrapToListItem = fmt.Errorf("provided object does not implement ListItem interface")
 var ErrCannotWrapToMetricsObject = fmt.Errorf("provided object does not implement MetricsObject interface")
 var ErrCannotWrapToActiveMetricsObject = fmt.Errorf("provided object does not implement ActiveMetricsObject interface")
+var ErrCannotWrapToSpanItem = fmt.Errorf("provided object does not implement SpanItem interface")
 var ErrRetryCountExceeded = fmt.Errorf("retryCount for evaluation exceeded")
 var ErrNoValues = fmt.Errorf("no values")
 var ErrInvalidOperator = fmt.Errorf("invalid operator")
@@ -21,3 +22,4 @@ var ErrNoLabelsFoundTask = "no labels found for task: %s"
 var ErrNoConfigMapMsg = "No ConfigMap specified or HTTP source specified in TaskDefinition) / Namespace: %s, Name: %s"
 var ErrCannotGetFunctionConfigMap = "could not get function configMap: %w"
 var ErrCannotFetchAppVersionForWorkloadInstanceMsg = "could not fetch AppVersion for KeptnWorkloadInstance: "
+var ErrCouldNotUnbindSpan = "could not unbind span for %s"

operator/controllers/common/evaluationhandler.go

@@ -3,12 +3,12 @@ package common
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"github.com/go-logr/logr"
 	klcv1alpha1 "github.com/keptn/lifecycle-toolkit/operator/api/v1alpha1"
 	apicommon "github.com/keptn/lifecycle-toolkit/operator/api/v1alpha1/common"
-	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/propagation"
+	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/trace"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -20,10 +20,11 @@ import (
 
 type EvaluationHandler struct {
 	client.Client
-	Recorder record.EventRecorder
-	Log      logr.Logger
-	Tracer   trace.Tracer
-	Scheme   *runtime.Scheme
+	Recorder    record.EventRecorder
+	Log         logr.Logger
+	Tracer      trace.Tracer
+	Scheme      *runtime.Scheme
+	SpanHandler ISpanHandler
 }
 
 type EvaluationCreateAttributes struct {
@@ -32,7 +33,7 @@ type EvaluationCreateAttributes struct {
 	CheckType            apicommon.CheckType
 }
 
-func (r EvaluationHandler) ReconcileEvaluations(ctx context.Context, reconcileObject client.Object, evaluationCreateAttributes EvaluationCreateAttributes) ([]klcv1alpha1.EvaluationStatus, apicommon.StatusSummary, error) {
+func (r EvaluationHandler) ReconcileEvaluations(ctx context.Context, phaseCtx context.Context, reconcileObject client.Object, evaluationCreateAttributes EvaluationCreateAttributes) ([]klcv1alpha1.EvaluationStatus, apicommon.StatusSummary, error) {
 	piWrapper, err := NewPhaseItemWrapperFromClientObject(reconcileObject)
 	if err != nil {
 		return nil, apicommon.StatusSummary{}, err
@@ -101,10 +102,30 @@ func (r EvaluationHandler) ReconcileEvaluations(ctx context.Context, reconcileOb
 			}
 			evaluationStatus.EvaluationName = evaluationName
 			evaluationStatus.SetStartTime()
+			_, _, err = r.SpanHandler.GetSpan(phaseCtx, r.Tracer, evaluation, "")
+			if err != nil {
+				r.Log.Error(err, "could not get span")
+			}
 		} else {
+			_, spanEvaluationTrace, err := r.SpanHandler.GetSpan(phaseCtx, r.Tracer, evaluation, "")
+			if err != nil {
+				r.Log.Error(err, "could not get span")
+			}
 			// Update state of Evaluation if it is already created
 			evaluationStatus.Status = evaluation.Status.OverallStatus
 			if evaluationStatus.Status.IsCompleted() {
+				if evaluationStatus.Status.IsSucceeded() {
+					spanEvaluationTrace.AddEvent(evaluation.Name + " has finished")
+					spanEvaluationTrace.SetStatus(codes.Ok, "Finished")
+				} else {
+					spanEvaluationTrace.AddEvent(evaluation.Name + " has failed")
+					r.setEvaluationFailureEvents(evaluation, spanEvaluationTrace)
+					spanEvaluationTrace.SetStatus(codes.Error, "Failed")
+				}
+				spanEvaluationTrace.End()
+				if err := r.SpanHandler.UnbindSpan(evaluation, ""); err != nil {
+					r.Log.Error(err, ErrCouldNotUnbindSpan, evaluation.Name)
+				}
 				evaluationStatus.SetEndTime()
 			}
 		}
@@ -127,22 +148,12 @@ func (r EvaluationHandler) CreateKeptnEvaluation(ctx context.Context, namespace
 		return "", err
 	}
 
-	ctx, span := r.Tracer.Start(ctx, evaluationCreateAttributes.SpanName, trace.WithSpanKind(trace.SpanKindProducer))
-	defer span.End()
-
-	piWrapper.SetSpanAttributes(span)
-
-	// create TraceContext
-	// follow up with a Keptn propagator that JSON-encoded the OTel map into our own key
-	traceContextCarrier := propagation.MapCarrier{}
-	otel.GetTextMapPropagator().Inject(ctx, traceContextCarrier)
-
 	phase := apicommon.KeptnPhaseType{
 		ShortName: "KeptnEvaluationCreate",
 		LongName:  "Keptn Evaluation Create",
 	}
 
-	newEvaluation := piWrapper.GenerateEvaluation(traceContextCarrier, evaluationCreateAttributes.EvaluationDefinition, evaluationCreateAttributes.CheckType)
+	newEvaluation := piWrapper.GenerateEvaluation(evaluationCreateAttributes.EvaluationDefinition, evaluationCreateAttributes.CheckType)
 	err = controllerutil.SetControllerReference(reconcileObject, &newEvaluation, r.Scheme)
 	if err != nil {
 		r.Log.Error(err, "could not set controller reference:")
@@ -157,3 +168,11 @@ func (r EvaluationHandler) CreateKeptnEvaluation(ctx context.Context, namespace
 
 	return newEvaluation.Name, nil
 }
+
+func (r EvaluationHandler) setEvaluationFailureEvents(evaluation *klcv1alpha1.KeptnEvaluation, spanTrace trace.Span) {
+	for k, v := range evaluation.Status.EvaluationStatus {
+		if v.Status == apicommon.StateFailed {
+			spanTrace.AddEvent(fmt.Sprintf("evaluation of '%s' failed with value: '%s' and reason: '%s'", k, v.Value, v.Message), trace.WithTimestamp(time.Now().UTC()))
+		}
+	}
+}