Skip to content

Commit

Permalink
feat(operator): rework Task and Evaluation span structure (#465)
Browse files Browse the repository at this point in the history
Signed-off-by: odubajDT <[email protected]>
  • Loading branch information
odubajDT authored Nov 25, 2022
1 parent 247d026 commit e5717c6
Show file tree
Hide file tree
Showing 33 changed files with 1,246 additions and 738 deletions.
34 changes: 17 additions & 17 deletions operator/api/v1alpha1/common/common.go
Original file line number Diff line number Diff line change
Expand Up @@ -64,47 +64,47 @@ func (k KeptnState) IsPending() bool {

type StatusSummary struct {
Total int
progressing int
failed int
succeeded int
pending int
unknown int
cancelled int
Progressing int
Failed int
Succeeded int
Pending int
Unknown int
Cancelled int
}

func UpdateStatusSummary(status KeptnState, summary StatusSummary) StatusSummary {
switch status {
case StateFailed:
summary.failed++
summary.Failed++
case StateCancelled:
summary.cancelled++
summary.Cancelled++
case StateSucceeded:
summary.succeeded++
summary.Succeeded++
case StateProgressing:
summary.progressing++
summary.Progressing++
case StatePending, "":
summary.pending++
summary.Pending++
case StateUnknown:
summary.unknown++
summary.Unknown++
}
return summary
}

func (s StatusSummary) GetTotalCount() int {
return s.failed + s.succeeded + s.progressing + s.pending + s.unknown + s.cancelled
return s.Failed + s.Succeeded + s.Progressing + s.Pending + s.Unknown + s.Cancelled
}

func GetOverallState(s StatusSummary) KeptnState {
if s.failed > 0 || s.cancelled > 0 {
if s.Failed > 0 || s.Cancelled > 0 {
return StateFailed
}
if s.progressing > 0 {
if s.Progressing > 0 {
return StateProgressing
}
if s.pending > 0 {
if s.Pending > 0 {
return StatePending
}
if s.unknown > 0 || s.GetTotalCount() != s.Total {
if s.Unknown > 0 || s.GetTotalCount() != s.Total {
return StateUnknown
}
return StateSucceeded
Expand Down
14 changes: 6 additions & 8 deletions operator/api/v1alpha1/keptnappversion_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -299,12 +299,11 @@ func (a KeptnAppVersion) GetVersion() string {
return a.Spec.Version
}

func (a KeptnAppVersion) GenerateTask(traceContextCarrier propagation.MapCarrier, taskDefinition string, checkType common.CheckType) KeptnTask {
func (a KeptnAppVersion) GenerateTask(taskDefinition string, checkType common.CheckType) KeptnTask {
return KeptnTask{
ObjectMeta: metav1.ObjectMeta{
Name: common.GenerateTaskName(checkType, taskDefinition),
Namespace: a.Namespace,
Annotations: traceContextCarrier,
Name: common.GenerateTaskName(checkType, taskDefinition),
Namespace: a.Namespace,
},
Spec: KeptnTaskSpec{
AppVersion: a.GetVersion(),
Expand All @@ -317,12 +316,11 @@ func (a KeptnAppVersion) GenerateTask(traceContextCarrier propagation.MapCarrier
}
}

func (a KeptnAppVersion) GenerateEvaluation(traceContextCarrier propagation.MapCarrier, evaluationDefinition string, checkType common.CheckType) KeptnEvaluation {
func (a KeptnAppVersion) GenerateEvaluation(evaluationDefinition string, checkType common.CheckType) KeptnEvaluation {
return KeptnEvaluation{
ObjectMeta: metav1.ObjectMeta{
Name: common.GenerateEvaluationName(checkType, evaluationDefinition),
Namespace: a.Namespace,
Annotations: traceContextCarrier,
Name: common.GenerateEvaluationName(checkType, evaluationDefinition),
Namespace: a.Namespace,
},
Spec: KeptnEvaluationSpec{
AppVersion: a.Spec.Version,
Expand Down
13 changes: 13 additions & 0 deletions operator/api/v1alpha1/keptnevaluation_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

"github.com/keptn/lifecycle-toolkit/operator/api/v1alpha1/common"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand Down Expand Up @@ -175,3 +176,15 @@ func (e KeptnEvaluation) GetSpanAttributes() []attribute.KeyValue {
common.EvaluationType.String(string(e.Spec.Type)),
}
}

func (e *KeptnEvaluation) SetPhaseTraceID(phase string, carrier propagation.MapCarrier) {
// present due to SpanItem interface
}

func (e KeptnEvaluation) GetSpanKey(phase string) string {
return e.Name
}

func (e KeptnEvaluation) GetSpanName(phase string) string {
return e.Name
}
13 changes: 13 additions & 0 deletions operator/api/v1alpha1/keptntask_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (

"github.com/keptn/lifecycle-toolkit/operator/api/v1alpha1/common"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/trace"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
Expand Down Expand Up @@ -183,3 +184,15 @@ func (t KeptnTask) GetSpanAttributes() []attribute.KeyValue {
common.TaskType.String(string(t.Spec.Type)),
}
}

func (t *KeptnTask) SetPhaseTraceID(phase string, carrier propagation.MapCarrier) {
// present due to SpanItem interface
}

func (t KeptnTask) GetSpanKey(phase string) string {
return t.Name
}

func (t KeptnTask) GetSpanName(phase string) string {
return t.Name
}
14 changes: 6 additions & 8 deletions operator/api/v1alpha1/keptnworkloadinstance_types.go
Original file line number Diff line number Diff line change
Expand Up @@ -336,12 +336,11 @@ func (w KeptnWorkloadInstance) GetVersion() string {
return w.Spec.Version
}

func (w KeptnWorkloadInstance) GenerateTask(traceContextCarrier propagation.MapCarrier, taskDefinition string, checkType common.CheckType) KeptnTask {
func (w KeptnWorkloadInstance) GenerateTask(taskDefinition string, checkType common.CheckType) KeptnTask {
return KeptnTask{
ObjectMeta: metav1.ObjectMeta{
Name: common.GenerateTaskName(checkType, taskDefinition),
Namespace: w.Namespace,
Annotations: traceContextCarrier,
Name: common.GenerateTaskName(checkType, taskDefinition),
Namespace: w.Namespace,
},
Spec: KeptnTaskSpec{
AppName: w.GetAppName(),
Expand All @@ -355,12 +354,11 @@ func (w KeptnWorkloadInstance) GenerateTask(traceContextCarrier propagation.MapC
}
}

func (w KeptnWorkloadInstance) GenerateEvaluation(traceContextCarrier propagation.MapCarrier, evaluationDefinition string, checkType common.CheckType) KeptnEvaluation {
func (w KeptnWorkloadInstance) GenerateEvaluation(evaluationDefinition string, checkType common.CheckType) KeptnEvaluation {
return KeptnEvaluation{
ObjectMeta: metav1.ObjectMeta{
Name: common.GenerateEvaluationName(checkType, evaluationDefinition),
Namespace: w.Namespace,
Annotations: traceContextCarrier,
Name: common.GenerateEvaluationName(checkType, evaluationDefinition),
Namespace: w.Namespace,
},
Spec: KeptnEvaluationSpec{
AppName: w.GetAppName(),
Expand Down
4 changes: 2 additions & 2 deletions operator/api/v1alpha1/tests/keptnappversion_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ func TestKeptnAppVersion(t *testing.T) {

require.Equal(t, "trace1.appname.version.phase", app.GetSpanKey("phase"))

task := app.GenerateTask(map[string]string{}, "taskdef", common.PostDeploymentCheckType)
task := app.GenerateTask("taskdef", common.PostDeploymentCheckType)
require.Equal(t, v1alpha1.KeptnTaskSpec{
AppVersion: app.GetVersion(),
AppName: app.GetParentName(),
Expand All @@ -189,7 +189,7 @@ func TestKeptnAppVersion(t *testing.T) {
Type: common.PostDeploymentCheckType,
}, task.Spec)

evaluation := app.GenerateEvaluation(map[string]string{}, "taskdef", common.PostDeploymentCheckType)
evaluation := app.GenerateEvaluation("taskdef", common.PostDeploymentCheckType)
require.Equal(t, v1alpha1.KeptnEvaluationSpec{
AppVersion: app.GetVersion(),
AppName: app.GetParentName(),
Expand Down
18 changes: 18 additions & 0 deletions operator/api/v1alpha1/tests/keptnevaluation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,24 @@ func TestKeptnEvaluation(t *testing.T) {
},
}

evaluation.SetPhaseTraceID("", nil)
require.Equal(t, v1alpha1.KeptnEvaluation{
ObjectMeta: metav1.ObjectMeta{
Name: "evaluation",
},
Spec: v1alpha1.KeptnEvaluationSpec{
AppName: "app",
AppVersion: "appversion",
Type: common.PostDeploymentCheckType,
},
Status: v1alpha1.KeptnEvaluationStatus{
OverallStatus: common.StateFailed,
},
}, *evaluation)

require.Equal(t, "evaluation", evaluation.GetSpanKey(""))
require.Equal(t, "evaluation", evaluation.GetSpanName(""))

require.False(t, evaluation.IsEndTimeSet())
require.False(t, evaluation.IsStartTimeSet())

Expand Down
18 changes: 18 additions & 0 deletions operator/api/v1alpha1/tests/keptntask_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,24 @@ func TestKeptnTask(t *testing.T) {
},
}

task.SetPhaseTraceID("", nil)
require.Equal(t, v1alpha1.KeptnTask{
ObjectMeta: metav1.ObjectMeta{
Name: "task",
},
Spec: v1alpha1.KeptnTaskSpec{
AppName: "app",
AppVersion: "appversion",
Type: common.PostDeploymentCheckType,
},
Status: v1alpha1.KeptnTaskStatus{
Status: common.StateFailed,
},
}, *task)

require.Equal(t, "task", task.GetSpanKey(""))
require.Equal(t, "task", task.GetSpanName(""))

require.False(t, task.IsEndTimeSet())
require.False(t, task.IsStartTimeSet())

Expand Down
4 changes: 2 additions & 2 deletions operator/api/v1alpha1/tests/keptnworkloadinstance_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -183,7 +183,7 @@ func TestKeptnWorkloadInstance(t *testing.T) {

require.Equal(t, "trace1.workloadname.version.phase", workload.GetSpanKey("phase"))

task := workload.GenerateTask(map[string]string{}, "taskdef", common.PostDeploymentCheckType)
task := workload.GenerateTask("taskdef", common.PostDeploymentCheckType)
require.Equal(t, v1alpha1.KeptnTaskSpec{
AppName: workload.GetAppName(),
WorkloadVersion: workload.GetVersion(),
Expand All @@ -194,7 +194,7 @@ func TestKeptnWorkloadInstance(t *testing.T) {
Type: common.PostDeploymentCheckType,
}, task.Spec)

evaluation := workload.GenerateEvaluation(map[string]string{}, "taskdef", common.PostDeploymentCheckType)
evaluation := workload.GenerateEvaluation("taskdef", common.PostDeploymentCheckType)
require.Equal(t, v1alpha1.KeptnEvaluationSpec{
AppName: workload.GetAppName(),
WorkloadVersion: workload.GetVersion(),
Expand Down
2 changes: 2 additions & 0 deletions operator/controllers/common/errors.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ var ErrCannotWrapToPhaseItem = fmt.Errorf("provided object does not implement Ph
var ErrCannotWrapToListItem = fmt.Errorf("provided object does not implement ListItem interface")
var ErrCannotWrapToMetricsObject = fmt.Errorf("provided object does not implement MetricsObject interface")
var ErrCannotWrapToActiveMetricsObject = fmt.Errorf("provided object does not implement ActiveMetricsObject interface")
var ErrCannotWrapToSpanItem = fmt.Errorf("provided object does not implement SpanItem interface")
var ErrRetryCountExceeded = fmt.Errorf("retryCount for evaluation exceeded")
var ErrNoValues = fmt.Errorf("no values")
var ErrInvalidOperator = fmt.Errorf("invalid operator")
Expand All @@ -21,3 +22,4 @@ var ErrNoLabelsFoundTask = "no labels found for task: %s"
var ErrNoConfigMapMsg = "No ConfigMap specified or HTTP source specified in TaskDefinition) / Namespace: %s, Name: %s"
var ErrCannotGetFunctionConfigMap = "could not get function configMap: %w"
var ErrCannotFetchAppVersionForWorkloadInstanceMsg = "could not fetch AppVersion for KeptnWorkloadInstance: "
var ErrCouldNotUnbindSpan = "could not unbind span for %s"
55 changes: 37 additions & 18 deletions operator/controllers/common/evaluationhandler.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,12 @@ package common
import (
"context"
"fmt"
"time"

"github.com/go-logr/logr"
klcv1alpha1 "github.com/keptn/lifecycle-toolkit/operator/api/v1alpha1"
apicommon "github.com/keptn/lifecycle-toolkit/operator/api/v1alpha1/common"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"
Expand All @@ -20,10 +20,11 @@ import (

type EvaluationHandler struct {
client.Client
Recorder record.EventRecorder
Log logr.Logger
Tracer trace.Tracer
Scheme *runtime.Scheme
Recorder record.EventRecorder
Log logr.Logger
Tracer trace.Tracer
Scheme *runtime.Scheme
SpanHandler ISpanHandler
}

type EvaluationCreateAttributes struct {
Expand All @@ -32,7 +33,7 @@ type EvaluationCreateAttributes struct {
CheckType apicommon.CheckType
}

func (r EvaluationHandler) ReconcileEvaluations(ctx context.Context, reconcileObject client.Object, evaluationCreateAttributes EvaluationCreateAttributes) ([]klcv1alpha1.EvaluationStatus, apicommon.StatusSummary, error) {
func (r EvaluationHandler) ReconcileEvaluations(ctx context.Context, phaseCtx context.Context, reconcileObject client.Object, evaluationCreateAttributes EvaluationCreateAttributes) ([]klcv1alpha1.EvaluationStatus, apicommon.StatusSummary, error) {
piWrapper, err := NewPhaseItemWrapperFromClientObject(reconcileObject)
if err != nil {
return nil, apicommon.StatusSummary{}, err
Expand Down Expand Up @@ -101,10 +102,30 @@ func (r EvaluationHandler) ReconcileEvaluations(ctx context.Context, reconcileOb
}
evaluationStatus.EvaluationName = evaluationName
evaluationStatus.SetStartTime()
_, _, err = r.SpanHandler.GetSpan(phaseCtx, r.Tracer, evaluation, "")
if err != nil {
r.Log.Error(err, "could not get span")
}
} else {
_, spanEvaluationTrace, err := r.SpanHandler.GetSpan(phaseCtx, r.Tracer, evaluation, "")
if err != nil {
r.Log.Error(err, "could not get span")
}
// Update state of Evaluation if it is already created
evaluationStatus.Status = evaluation.Status.OverallStatus
if evaluationStatus.Status.IsCompleted() {
if evaluationStatus.Status.IsSucceeded() {
spanEvaluationTrace.AddEvent(evaluation.Name + " has finished")
spanEvaluationTrace.SetStatus(codes.Ok, "Finished")
} else {
spanEvaluationTrace.AddEvent(evaluation.Name + " has failed")
r.setEvaluationFailureEvents(evaluation, spanEvaluationTrace)
spanEvaluationTrace.SetStatus(codes.Error, "Failed")
}
spanEvaluationTrace.End()
if err := r.SpanHandler.UnbindSpan(evaluation, ""); err != nil {
r.Log.Error(err, ErrCouldNotUnbindSpan, evaluation.Name)
}
evaluationStatus.SetEndTime()
}
}
Expand All @@ -127,22 +148,12 @@ func (r EvaluationHandler) CreateKeptnEvaluation(ctx context.Context, namespace
return "", err
}

ctx, span := r.Tracer.Start(ctx, evaluationCreateAttributes.SpanName, trace.WithSpanKind(trace.SpanKindProducer))
defer span.End()

piWrapper.SetSpanAttributes(span)

// create TraceContext
// follow up with a Keptn propagator that JSON-encoded the OTel map into our own key
traceContextCarrier := propagation.MapCarrier{}
otel.GetTextMapPropagator().Inject(ctx, traceContextCarrier)

phase := apicommon.KeptnPhaseType{
ShortName: "KeptnEvaluationCreate",
LongName: "Keptn Evaluation Create",
}

newEvaluation := piWrapper.GenerateEvaluation(traceContextCarrier, evaluationCreateAttributes.EvaluationDefinition, evaluationCreateAttributes.CheckType)
newEvaluation := piWrapper.GenerateEvaluation(evaluationCreateAttributes.EvaluationDefinition, evaluationCreateAttributes.CheckType)
err = controllerutil.SetControllerReference(reconcileObject, &newEvaluation, r.Scheme)
if err != nil {
r.Log.Error(err, "could not set controller reference:")
Expand All @@ -157,3 +168,11 @@ func (r EvaluationHandler) CreateKeptnEvaluation(ctx context.Context, namespace

return newEvaluation.Name, nil
}

func (r EvaluationHandler) setEvaluationFailureEvents(evaluation *klcv1alpha1.KeptnEvaluation, spanTrace trace.Span) {
for k, v := range evaluation.Status.EvaluationStatus {
if v.Status == apicommon.StateFailed {
spanTrace.AddEvent(fmt.Sprintf("evaluation of '%s' failed with value: '%s' and reason: '%s'", k, v.Value, v.Message), trace.WithTimestamp(time.Now().UTC()))
}
}
}
Loading

0 comments on commit e5717c6

Please sign in to comment.