feat(scheduler): Add support for OTel collector (#146)

Signed-off-by: Florian Bacher <[email protected]>

scheduler/cmd/scheduler/main.go

@@ -19,6 +19,11 @@ package main
 import (
 	"context"
 	"fmt"
+	"github.com/kelseyhightower/envconfig"
+	"go.opentelemetry.io/otel/exporters/otlp"
+	"go.opentelemetry.io/otel/exporters/otlp/otlpgrpc"
+	"google.golang.org/grpc"
+	"k8s.io/klog/v2"
 	"log"
 	"os"
 	"time"
@@ -34,9 +39,17 @@ import (
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 )
 
+type envConfig struct {
+	OTelCollectorURL string `envconfig:"OTEL_COLLECTOR_URL" default:""`
+}
+
 func main() {
+	var env envConfig
+	if err := envconfig.Process("", &env); err != nil {
+		log.Fatalf("Failed to process env var: %s", err)
+	}
 
-	tp := initOTel()
+	tp := initOTel(env)
 
 	rand.Seed(time.Now().UnixNano())
 	command := app.NewSchedulerCommand(
@@ -53,18 +66,57 @@ func main() {
 
 }
 
-func initOTel() *sdktrace.TracerProvider {
-	var err error
-	exp, err := stdout.NewExporter(stdout.WithPrettyPrint())
+func initOTel(env envConfig) *sdktrace.TracerProvider {
+	tpOptions, err := getOTelTracerProviderOptions(env)
 	if err != nil {
-		log.Panicf("failed to initialize stdout exporter %v\n", err)
+		log.Panicf("failed to initialize OTel options")
 	}
-	bsp := sdktrace.NewSimpleSpanProcessor(exp)
-	tp := sdktrace.NewTracerProvider(
-		sdktrace.WithSampler(sdktrace.AlwaysSample()),
-		sdktrace.WithSpanProcessor(bsp),
-	)
+	tp := sdktrace.NewTracerProvider(tpOptions...)
 	otel.SetTracerProvider(tp)
 	otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator(propagation.TraceContext{}, propagation.Baggage{}))
 	return tp
 }
+
+func getOTelTracerProviderOptions(env envConfig) ([]sdktrace.TracerProviderOption, error) {
+	tracerProviderOptions := []sdktrace.TracerProviderOption{}
+
+	stdOutExp, err := newStdOutExporter()
+	if err != nil {
+		return nil, fmt.Errorf("could not create stdout OTel exporter: %w", err)
+	}
+	tracerProviderOptions = append(tracerProviderOptions, sdktrace.WithBatcher(stdOutExp))
+
+	if env.OTelCollectorURL != "" {
+		// try to set OTel exporter for Jaeger
+		otelExporter, err := newOTelExporter(env)
+		if err != nil {
+			// log the error, but do not break if Jaeger exporter cannot be created
+			klog.Errorf("Could not set up OTel exporter: %v", err)
+		} else if otelExporter != nil {
+			tracerProviderOptions = append(tracerProviderOptions, sdktrace.WithBatcher(otelExporter))
+		}
+	}
+	tracerProviderOptions = append(tracerProviderOptions, sdktrace.WithSampler(sdktrace.AlwaysSample()))
+
+	return tracerProviderOptions, nil
+}
+
+func newStdOutExporter() (sdktrace.SpanExporter, error) {
+	return stdout.NewExporter(stdout.WithPrettyPrint())
+}
+
+func newOTelExporter(env envConfig) (sdktrace.SpanExporter, error) {
+	ctx, cancel := context.WithTimeout(context.TODO(), 3*time.Second)
+	defer cancel()
+
+	driver := otlpgrpc.NewDriver(
+		otlpgrpc.WithInsecure(),
+		otlpgrpc.WithEndpoint(env.OTelCollectorURL),
+		otlpgrpc.WithDialOption(grpc.WithBlock()), // useful for testing
+	)
+	traceExporter, err := otlp.NewExporter(ctx, driver)
+	if err != nil {
+		return nil, err
+	}
+	return traceExporter, nil
+}

scheduler/go.mod

@@ -3,10 +3,13 @@ module github.com/keptn-sandbox/lifecycle-controller/scheduler
 go 1.18
 
 require (
+	github.com/kelseyhightower/envconfig v1.4.0
 	go.opentelemetry.io/otel v0.20.0
+	go.opentelemetry.io/otel/exporters/otlp v0.20.0
 	go.opentelemetry.io/otel/exporters/stdout v0.20.0
 	go.opentelemetry.io/otel/sdk v0.20.0
 	go.opentelemetry.io/otel/trace v0.20.0
+	google.golang.org/grpc v1.40.0
 	k8s.io/api v0.24.3
 	k8s.io/apimachinery v0.24.3
 	k8s.io/client-go v0.24.3
@@ -71,7 +74,6 @@ require (
 	go.opentelemetry.io/contrib v0.20.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp v0.20.0 // indirect
 	go.opentelemetry.io/otel/metric v0.20.0 // indirect
 	go.opentelemetry.io/otel/sdk/export/metric v0.20.0 // indirect
 	go.opentelemetry.io/otel/sdk/metric v0.20.0 // indirect
@@ -91,7 +93,6 @@ require (
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368 // indirect
-	google.golang.org/grpc v1.40.0 // indirect
 	google.golang.org/protobuf v1.27.1 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect

scheduler/go.sum

@@ -396,6 +396,8 @@ github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7V
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
 github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes=
 github.com/karrick/godirwalk v1.16.1/go.mod h1:j4mkqPuvaLI8mp1DroR3P6ad7cyYd4c1qeJ3RV7ULlk=
+github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=
+github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg=
 github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
 github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=

scheduler/manifests/install/base/deployment.yaml

@@ -21,6 +21,9 @@ spec:
             - /bin/kube-scheduler
             - --config=/etc/kubernetes/scheduler-config.yaml
           image: klfc-scheduler:latest
+          env:
+            - name: OTEL_COLLECTOR_URL
+              value: otel-collector:4317
           livenessProbe:
             httpGet:
               path: /healthz