From 817bc20f49f7c17fa65a8dc226afc2bcb7285f5d Mon Sep 17 00:00:00 2001 From: Florian Bacher Date: Tue, 11 Oct 2022 14:51:10 +0200 Subject: [PATCH 1/2] feat(scheduler): Add support for OTel collector Signed-off-by: Florian Bacher --- scheduler/cmd/scheduler/main.go | 72 ++++++++++++++++--- scheduler/go.mod | 1 + scheduler/go.sum | 2 + .../manifests/install/base/deployment.yaml | 3 + 4 files changed, 68 insertions(+), 10 deletions(-) diff --git a/scheduler/cmd/scheduler/main.go b/scheduler/cmd/scheduler/main.go index 8a9e4ddde7..7c2b95e303 100644 --- a/scheduler/cmd/scheduler/main.go +++ b/scheduler/cmd/scheduler/main.go @@ -19,6 +19,11 @@ package main import ( "context" "fmt" + "github.com/kelseyhightower/envconfig" + "go.opentelemetry.io/otel/exporters/otlp" + "go.opentelemetry.io/otel/exporters/otlp/otlpgrpc" + "google.golang.org/grpc" + "k8s.io/klog/v2" "log" "os" "time" @@ -34,9 +39,17 @@ import ( sdktrace "go.opentelemetry.io/otel/sdk/trace" ) +type envConfig struct { + OTelCollectorURL string `envconfig:"OTEL_COLLECTOR_URL" default:""` +} + func main() { + var env envConfig + if err := envconfig.Process("", &env); err != nil { + log.Fatalf("Failed to process env var: %s", err) + } - tp := initOTel() + tp := initOTel(env) rand.Seed(time.Now().UnixNano()) command := app.NewSchedulerCommand( @@ -53,18 +66,57 @@ func main() { } -func initOTel() *sdktrace.TracerProvider { - var err error - exp, err := stdout.NewExporter(stdout.WithPrettyPrint()) +func initOTel(env envConfig) *sdktrace.TracerProvider { + tpOptions, err := getOTelTracerProviderOptions(env) if err != nil { - log.Panicf("failed to initialize stdout exporter %v\n", err) + log.Panicf("failed to initialize OTel options") } - bsp := sdktrace.NewSimpleSpanProcessor(exp) - tp := sdktrace.NewTracerProvider( - sdktrace.WithSampler(sdktrace.AlwaysSample()), - sdktrace.WithSpanProcessor(bsp), - ) + tp := sdktrace.NewTracerProvider(tpOptions...) otel.SetTracerProvider(tp) otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{})) return tp } + +func getOTelTracerProviderOptions(env envConfig) ([]sdktrace.TracerProviderOption, error) { + tracerProviderOptions := []sdktrace.TracerProviderOption{} + + stdOutExp, err := newStdOutExporter() + if err != nil { + return nil, fmt.Errorf("could not create stdout OTel exporter: %w", err) + } + tracerProviderOptions = append(tracerProviderOptions, sdktrace.WithBatcher(stdOutExp)) + + if env.OTelCollectorURL != "" { + // try to set OTel exporter for Jaeger + otelExporter, err := newOTelExporter(env) + if err != nil { + // log the error, but do not break if Jaeger exporter cannot be created + klog.Errorf("Could not set up OTel exporter: %v", err) + } else if otelExporter != nil { + tracerProviderOptions = append(tracerProviderOptions, sdktrace.WithBatcher(otelExporter)) + } + } + tracerProviderOptions = append(tracerProviderOptions, sdktrace.WithSampler(sdktrace.AlwaysSample())) + + return tracerProviderOptions, nil +} + +func newStdOutExporter() (sdktrace.SpanExporter, error) { + return stdout.NewExporter(stdout.WithPrettyPrint()) +} + +func newOTelExporter(env envConfig) (sdktrace.SpanExporter, error) { + ctx, cancel := context.WithTimeout(context.TODO(), 3*time.Second) + defer cancel() + + driver := otlpgrpc.NewDriver( + otlpgrpc.WithInsecure(), + otlpgrpc.WithEndpoint(env.OTelCollectorURL), + otlpgrpc.WithDialOption(grpc.WithBlock()), // useful for testing + ) + traceExporter, err := otlp.NewExporter(ctx, driver) + if err != nil { + return nil, err + } + return traceExporter, nil +} diff --git a/scheduler/go.mod b/scheduler/go.mod index 0b5b3f8e0d..b913f4ad8d 100644 --- a/scheduler/go.mod +++ b/scheduler/go.mod @@ -48,6 +48,7 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect diff --git a/scheduler/go.sum b/scheduler/go.sum index 5b8c81b103..21b066953e 100644 --- a/scheduler/go.sum +++ b/scheduler/go.sum @@ -396,6 +396,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk= +github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= +github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= diff --git a/scheduler/manifests/install/base/deployment.yaml b/scheduler/manifests/install/base/deployment.yaml index 74826b8e73..e9e32e0764 100644 --- a/scheduler/manifests/install/base/deployment.yaml +++ b/scheduler/manifests/install/base/deployment.yaml @@ -21,6 +21,9 @@ spec: - /bin/kube-scheduler - --config=/etc/kubernetes/scheduler-config.yaml image: klfc-scheduler:latest + env: + - name: OTEL_COLLECTOR_URL + value: otel-collector:4317 livenessProbe: httpGet: path: /healthz From 83e1e8af9d3ccf6eeda9efa261c3c605c04bff7c Mon Sep 17 00:00:00 2001 From: Florian Bacher Date: Wed, 12 Oct 2022 08:14:23 +0200 Subject: [PATCH 2/2] update go.mod Signed-off-by: Florian Bacher --- scheduler/go.mod | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scheduler/go.mod b/scheduler/go.mod index b913f4ad8d..50132c74b9 100644 --- a/scheduler/go.mod +++ b/scheduler/go.mod @@ -3,10 +3,13 @@ module github.com/keptn-sandbox/lifecycle-controller/scheduler go 1.18 require ( + github.com/kelseyhightower/envconfig v1.4.0 go.opentelemetry.io/otel v0.20.0 + go.opentelemetry.io/otel/exporters/otlp v0.20.0 go.opentelemetry.io/otel/exporters/stdout v0.20.0 go.opentelemetry.io/otel/sdk v0.20.0 go.opentelemetry.io/otel/trace v0.20.0 + google.golang.org/grpc v1.40.0 k8s.io/api v0.24.3 k8s.io/apimachinery v0.24.3 k8s.io/client-go v0.24.3 @@ -48,7 +51,6 @@ require ( github.com/inconshreveable/mousetrap v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect @@ -72,7 +74,6 @@ require ( go.opentelemetry.io/contrib v0.20.0 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect - go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect go.opentelemetry.io/otel/metric v0.20.0 // indirect go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect @@ -92,7 +93,6 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect - google.golang.org/grpc v1.40.0 // indirect google.golang.org/protobuf v1.27.1 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect