feat: OpenTelemetry module integration

build/run-e2e-suite.sh

@@ -83,6 +83,7 @@ kubectl run --rm \
   --restart=Never \
   --env="E2E_NODES=${E2E_NODES}" \
   --env="FOCUS=${FOCUS}" \
+  --env="IS_CHROOT=${IS_CHROOT:-false}"\
   --env="E2E_CHECK_LEAKS=${E2E_CHECK_LEAKS}" \
   --env="NGINX_BASE_IMAGE=${NGINX_BASE_IMAGE}" \
   --overrides='{ "apiVersion": "v1", "spec":{"serviceAccountName": "ingress-nginx-e2e"}}' \

internal/ingress/annotations/annotations.go

@@ -20,6 +20,7 @@ import (
 	"github.com/imdario/mergo"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/canary"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/modsecurity"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/opentelemetry"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/proxyssl"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/sslcipher"
 	"k8s.io/ingress-nginx/internal/ingress/annotations/streamsnippet"
@@ -93,6 +94,7 @@ type Ingress struct {
 	EnableGlobalAuth   bool
 	HTTP2PushPreload   bool
 	Opentracing        opentracing.Config
+	Opentelemetry      opentelemetry.Config
 	Proxy              proxy.Config
 	ProxySSL           proxyssl.Config
 	RateLimit          ratelimit.Config
@@ -143,6 +145,7 @@ func NewAnnotationExtractor(cfg resolver.Resolver) Extractor {
 			"EnableGlobalAuth":     authreqglobal.NewParser(cfg),
 			"HTTP2PushPreload":     http2pushpreload.NewParser(cfg),
 			"Opentracing":          opentracing.NewParser(cfg),
+			"Opentelemetry":        opentelemetry.NewParser(cfg),
 			"Proxy":                proxy.NewParser(cfg),
 			"ProxySSL":             proxyssl.NewParser(cfg),
 			"RateLimit":            ratelimit.NewParser(cfg),

internal/ingress/annotations/opentelemetry/main.go

@@ -0,0 +1,85 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package opentelemetry
+
+import (
+	networking "k8s.io/api/networking/v1"
+
+	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
+	"k8s.io/ingress-nginx/internal/ingress/resolver"
+)
+
+type opentelemetry struct {
+	r resolver.Resolver
+}
+
+// Config contains the configuration to be used in the Ingress
+type Config struct {
+	Enabled       bool   `json:"enabled"`
+	Set           bool   `json:"set"`
+	TrustEnabled  bool   `json:"trust-enabled"`
+	TrustSet      bool   `json:"trust-set"`
+	OperationName string `json:"operation-name"`
+}
+
+// Equal tests for equality between two Config types
+func (bd1 *Config) Equal(bd2 *Config) bool {
+
+	if bd1.Enabled != bd2.Enabled {
+		return false
+	}
+
+	if bd1.TrustEnabled != bd2.TrustEnabled {
+		return false
+	}
+
+	return true
+}
+
+func NewParser(r resolver.Resolver) parser.IngressAnnotation {
+	return opentelemetry{r}
+}
+
+// Parse parses the annotations to look for opentelemetry configurations
+func (c opentelemetry) Parse(ing *networking.Ingress) (interface{}, error) {
+	cfg := Config{}
+	enabled, err := parser.GetBoolAnnotation("enable-opentelemetry", ing)
+	if err != nil {
+		return &cfg, nil
+	}
+	cfg.Set = true
+	cfg.Enabled = enabled
+	if !enabled {
+		return &cfg, nil
+	}
+
+	trustEnabled, err := parser.GetBoolAnnotation("opentelemetry-trust-incoming-span", ing)
+	if err != nil {
+		operationName, err := parser.GetStringAnnotation("opentelemetry-operation-name", ing)
+		if err != nil {
+			return &cfg, nil
+		}
+		cfg.OperationName = operationName
+		return &cfg, nil
+	}
+
+	cfg.TrustSet = true
+	cfg.TrustEnabled = trustEnabled
+	operationName, err := parser.GetStringAnnotation("opentelemetry-operation-name", ing)
+	if err != nil {
+		return &cfg, nil
+	}
+	cfg.OperationName = operationName
+	return &cfg, nil
+}

internal/ingress/annotations/opentelemetry/main_test.go

@@ -0,0 +1,150 @@
+/*
+Copyright 2022 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package opentelemetry
+
+import (
+	"testing"
+
+	api "k8s.io/api/core/v1"
+	networking "k8s.io/api/networking/v1"
+	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/ingress-nginx/internal/ingress/annotations/parser"
+	"k8s.io/ingress-nginx/internal/ingress/resolver"
+)
+
+func buildIngress() *networking.Ingress {
+	defaultBackend := networking.IngressBackend{
+		Service: &networking.IngressServiceBackend{
+			Name: "default-backend",
+			Port: networking.ServiceBackendPort{
+				Number: 80,
+			},
+		},
+	}
+
+	return &networking.Ingress{
+		ObjectMeta: meta_v1.ObjectMeta{
+			Name:      "foo",
+			Namespace: api.NamespaceDefault,
+		},
+		Spec: networking.IngressSpec{
+			DefaultBackend: &networking.IngressBackend{
+				Service: &networking.IngressServiceBackend{
+					Name: "default-backend",
+					Port: networking.ServiceBackendPort{
+						Number: 80,
+					},
+				},
+			},
+			Rules: []networking.IngressRule{
+				{
+					Host: "foo.bar.com",
+					IngressRuleValue: networking.IngressRuleValue{
+						HTTP: &networking.HTTPIngressRuleValue{
+							Paths: []networking.HTTPIngressPath{
+								{
+									Path:    "/foo",
+									Backend: defaultBackend,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	}
+}
+
+func TestIngressAnnotationOpentelemetrySetTrue(t *testing.T) {
+	ing := buildIngress()
+
+	data := map[string]string{}
+	data[parser.GetAnnotationWithPrefix("enable-opentelemetry")] = "true"
+	ing.SetAnnotations(data)
+
+	val, _ := NewParser(&resolver.Mock{}).Parse(ing)
+	openTelemetry, ok := val.(*Config)
+	if !ok {
+		t.Errorf("expected a Config type")
+	}
+
+	if !openTelemetry.Enabled {
+		t.Errorf("expected annotation value to be true, got false")
+	}
+}
+
+func TestIngressAnnotationOpentelemetrySetFalse(t *testing.T) {
+	ing := buildIngress()
+
+	// Test with explicitly set to false
+	data := map[string]string{}
+	data[parser.GetAnnotationWithPrefix("enable-opentelemetry")] = "false"
+	ing.SetAnnotations(data)
+
+	val, _ := NewParser(&resolver.Mock{}).Parse(ing)
+	openTelemetry, ok := val.(*Config)
+	if !ok {
+		t.Errorf("expected a Config type")
+	}
+
+	if openTelemetry.Enabled {
+		t.Errorf("expected annotation value to be false, got true")
+	}
+}
+
+func TestIngressAnnotationOpentelemetryTrustSetTrue(t *testing.T) {
+	ing := buildIngress()
+
+	data := map[string]string{}
+	opName := "foo-op"
+	data[parser.GetAnnotationWithPrefix("enable-opentelemetry")] = "true"
+	data[parser.GetAnnotationWithPrefix("opentelemetry-trust-incoming-span")] = "true"
+	data[parser.GetAnnotationWithPrefix("opentelemetry-operation-name")] = opName
+	ing.SetAnnotations(data)
+
+	val, _ := NewParser(&resolver.Mock{}).Parse(ing)
+	openTelemetry, ok := val.(*Config)
+	if !ok {
+		t.Errorf("expected a Config type")
+	}
+
+	if !openTelemetry.Enabled {
+		t.Errorf("expected annotation value to be true, got false")
+	}
+
+	if !openTelemetry.TrustEnabled {
+		t.Errorf("expected annotation value to be true, got false")
+	}
+
+	if openTelemetry.OperationName != opName {
+		t.Errorf("expected annotation value to be %v, got %v", opName, openTelemetry.OperationName)
+	}
+}
+
+func TestIngressAnnotationOpentelemetryUnset(t *testing.T) {
+	ing := buildIngress()
+
+	// Test with no annotation specified
+	data := map[string]string{}
+	ing.SetAnnotations(data)
+
+	val, _ := NewParser(&resolver.Mock{}).Parse(ing)
+	_, ok := val.(*Config)
+	if !ok {
+		t.Errorf("expected a Config type")
+	}
+}

internal/ingress/controller/config/config.go

@@ -559,6 +559,54 @@ type Configuration struct {
 	// Default: true
 	OpentracingTrustIncomingSpan bool `json:"opentracing-trust-incoming-span"`
 
+	// EnableOpentelemetry enables the nginx Opentelemetry extension
+	// By default this is disabled
+	EnableOpentelemetry bool `json:"enable-opentelemetry"`
+
+	// OpentelemetryConfig sets the opentelemetry config file
+	// Default: /etc/nginx/opentelemtry.toml
+	OpentelemetryConfig string `json:"opentelemetry-config"`
+
+	// OpentelemetryOperationName specifies a custom name for the server span
+	OpentelemetryOperationName string `json:"opentelemetry-operation-name"`
+
+	// OpentelemetryTrustIncomingSpan sets whether or not to trust incoming trace spans
+	// If false, incoming span headers will be rejected
+	// Default: true
+	OpentelemetryTrustIncomingSpan bool `json:"opentelemetry-trust-incoming-span"`
+
+	// OtlpCollectorHost specifies the host to use when uploading traces
+	OtlpCollectorHost string `json:"otlp-collector-host"`
+
+	// OtlpCollectorPort specifies the port to use when uploading traces
+	// Default: 4317
+	OtlpCollectorPort string `json:"otlp-collector-port"`
+
+	// OtelServiceName specifies the service name to use for any traces created
+	// Default: nginx
+	OtelServiceName string `json:"otel-service-name"`
+
+	// OtelSampler specifies the sampler to use for any traces created
+	// Default: AlwaysOff
+	OtelSampler string `json:"otel-sampler"`
+
+	// OtelSamplerRatio specifies the sampler ratio to use for any traces created
+	// Default: 0.01
+	OtelSamplerRatio float32 `json:"otel-sampler-ratio"`
+
+	//OtelSamplerParantBased specifies the parent based sampler to be use for any traces created
+	// Default: false
+	OtelSamplerParantBased bool `json:"otel-sampler-parent-based"`
+
+	// MaxQueueSize specifies the max queue size for uploading traces
+	OtelMaxQueueSize int32 `json:"otel-max-queuesize"`
+
+	// ScheduleDelayMillis specifies the max delay between uploading traces
+	OtelScheduleDelayMillis int32 `json:"otel-schedule-delay-millis"`
+
+	// MaxExportBatchSize specifies the max export batch size to used when uploading traces
+	OtelMaxExportBatchSize int32 `json:"otel-max-export-batch-size"`
+
 	// ZipkinCollectorHost specifies the host to use when uploading traces
 	ZipkinCollectorHost string `json:"zipkin-collector-host"`
 
@@ -908,6 +956,13 @@ func NewDefault() Configuration {
 		BindAddressIpv4:                        defBindAddress,
 		BindAddressIpv6:                        defBindAddress,
 		OpentracingTrustIncomingSpan:           true,
+		OpentelemetryTrustIncomingSpan:         true,
+		OpentelemetryConfig:                    "/etc/nginx/opentelemtry.toml",
+		OtlpCollectorPort:                      "4317",
+		OtelServiceName:                        "nginx",
+		OtelSampler:                            "AlwaysOff",
+		OtelSamplerRatio:                       0.01,
+		OtelSamplerParantBased:                 false,
 		ZipkinCollectorPort:                    9411,
 		ZipkinServiceName:                      "nginx",
 		ZipkinSampleRate:                       1.0,

internal/ingress/controller/controller.go

@@ -1405,6 +1405,7 @@ func locationApplyAnnotations(loc *ingress.Location, anns *annotations.Ingress)
 	loc.EnableGlobalAuth = anns.EnableGlobalAuth
 	loc.HTTP2PushPreload = anns.HTTP2PushPreload
 	loc.Opentracing = anns.Opentracing
+	loc.Opentelemetry = anns.Opentelemetry
 	loc.Proxy = anns.Proxy
 	loc.ProxySSL = anns.ProxySSL
 	loc.RateLimit = anns.RateLimit