Python auto-instrumentation: handle musl based containers

Build and and inject musl based python auto-instrumentation if proper
annotation is configured:

instrumentation.opentelemetry.io/otel-python-wheel-kind: "musllinux"

Refs #2264

autoinstrumentation/python/Dockerfile

@@ -1,12 +1,12 @@
 # To build one auto-instrumentation image for Python, please:
-# - Ensure the packages are installed in the `/autoinstrumentation` directory. This is required as when instrumenting the pod,
-#   one init container will be created to copy all the content in `/autoinstrumentation` directory to your app's container. Then
+# - Ensure the packages are installed in the `/autoinstrumentation,{musl}` directory. This is required as when instrumenting the pod,
+#   one init container will be created to copy all the content in `/autoinstrumentation{,-musl}` directory to your app's container. Then
 #   update the `PYTHONPATH` environment variable accordingly. To achieve this, you can mimic the one in `autoinstrumentation/python/Dockerfile`
 #   by using multi-stage builds. In the first stage, install all the required packages in one custom directory with `pip install --target`.
-#   Then in the second stage, copy the directory to `/autoinstrumentation`.
+#   Then in the second stage, copy the directory to `/autoinstrumentation{,-musl}`.
 # - Ensure you have `opentelemetry-distro` and `opentelemetry-instrumentation` or your customized alternatives installed.
 #   Those two packages are essential to Python auto-instrumentation.
-# - Grant the necessary access to `/autoinstrumentation` directory. `chmod -R go+r /autoinstrumentation`
+# - Grant the necessary access to `/autoinstrumentation{,-musl}` directory. `chmod -R go+r /autoinstrumentation`
 # - For auto-instrumentation by container injection, the Linux command cp is
 #   used and must be availabe in the image.
 FROM python:3.11 AS build
@@ -17,8 +17,19 @@ ADD requirements.txt .
 
 RUN mkdir workspace && pip install --target workspace -r requirements.txt
 
+FROM python:3.11-alpine AS build-musl
+
+WORKDIR /operator-build
+
+ADD requirements.txt .
+
+RUN apk add gcc python3-dev musl-dev linux-headers
+RUN mkdir workspace && pip install --target workspace -r requirements.txt
+
 FROM busybox
 
 COPY --from=build /operator-build/workspace /autoinstrumentation
+COPY --from=build-musl /operator-build/workspace /autoinstrumentation-musl
 
 RUN chmod -R go+r /autoinstrumentation
+RUN chmod -R go+r /autoinstrumentation-musl

pkg/instrumentation/annotation.go

@@ -30,6 +30,7 @@ const (
 	annotationInjectNodeJSContainersName      = "instrumentation.opentelemetry.io/nodejs-container-names"
 	annotationInjectPython                    = "instrumentation.opentelemetry.io/inject-python"
 	annotationInjectPythonContainersName      = "instrumentation.opentelemetry.io/python-container-names"
+	annotationPythonWheelKind                 = "instrumentation.opentelemetry.io/python-wheel-kind"
 	annotationInjectDotNet                    = "instrumentation.opentelemetry.io/inject-dotnet"
 	annotationDotNetRuntime                   = "instrumentation.opentelemetry.io/otel-dotnet-auto-runtime"
 	annotationInjectDotnetContainersName      = "instrumentation.opentelemetry.io/dotnet-container-names"

pkg/instrumentation/podmutator.go

@@ -320,6 +320,7 @@ func (pm *instPodMutator) Mutate(ctx context.Context, ns corev1.Namespace, pod c
 	}
 	if pm.config.EnablePythonAutoInstrumentation() || inst == nil {
 		insts.Python.Instrumentation = inst
+		insts.Python.AdditionalAnnotations = map[string]string{annotationPythonWheelKind: annotationValue(ns.ObjectMeta, pod.ObjectMeta, annotationPythonWheelKind)}
 	} else {
 		logger.Error(nil, "support for Python auto instrumentation is not enabled")
 		pm.Recorder.Event(pod.DeepCopy(), "Warning", "InstrumentationRequestRejected", "support for Python auto instrumentation is not enabled")

pkg/instrumentation/python.go

@@ -23,18 +23,22 @@ import (
 )
 
 const (
-	envPythonPath               = "PYTHONPATH"
-	envOtelTracesExporter       = "OTEL_TRACES_EXPORTER"
-	envOtelMetricsExporter      = "OTEL_METRICS_EXPORTER"
-	envOtelExporterOTLPProtocol = "OTEL_EXPORTER_OTLP_PROTOCOL"
-	pythonPathPrefix            = "/otel-auto-instrumentation-python/opentelemetry/instrumentation/auto_instrumentation"
-	pythonPathSuffix            = "/otel-auto-instrumentation-python"
-	pythonInstrMountPath        = "/otel-auto-instrumentation-python"
-	pythonVolumeName            = volumeName + "-python"
-	pythonInitContainerName     = initContainerName + "-python"
+	envPythonPath                   = "PYTHONPATH"
+	envOtelTracesExporter           = "OTEL_TRACES_EXPORTER"
+	envOtelMetricsExporter          = "OTEL_METRICS_EXPORTER"
+	envOtelExporterOTLPProtocol     = "OTEL_EXPORTER_OTLP_PROTOCOL"
+	manyLinuxAutoInstrumentationSrc = "/autoinstrumentation/."
+	muslLinuxAutoInstrumentationSrc = "/autoinstrumentation-musl/."
+	pythonPathPrefix                = "/otel-auto-instrumentation-python/opentelemetry/instrumentation/auto_instrumentation"
+	pythonPathSuffix                = "/otel-auto-instrumentation-python"
+	pythonInstrMountPath            = "/otel-auto-instrumentation-python"
+	pythonVolumeName                = volumeName + "-python"
+	pythonInitContainerName         = initContainerName + "-python"
+	wheelKindManyLinux              = "manylinux"
+	wheelKindMuslLinux              = "musllinux"
 )
 
-func injectPythonSDK(pythonSpec v1alpha1.Python, pod corev1.Pod, index int) (corev1.Pod, error) {
+func injectPythonSDK(pythonSpec v1alpha1.Python, pod corev1.Pod, index int, wheelKind string) (corev1.Pod, error) {
 	// caller checks if there is at least one container.
 	container := &pod.Spec.Containers[index]
 
@@ -43,6 +47,16 @@ func injectPythonSDK(pythonSpec v1alpha1.Python, pod corev1.Pod, index int) (cor
 		return pod, err
 	}
 
+	autoInstrumentationSrc := ""
+	switch wheelKind {
+	case "", wheelKindManyLinux:
+		autoInstrumentationSrc = manyLinuxAutoInstrumentationSrc
+	case wheelKindMuslLinux:
+		autoInstrumentationSrc = muslLinuxAutoInstrumentationSrc
+	default:
+		return pod, fmt.Errorf("provided instrumentation.opentelemetry.io/python-wheel-kind annotation value '%s' is not supported", wheelKind)
+	}
+
 	// inject Python instrumentation spec env vars.
 	for _, env := range pythonSpec.Env {
 		idx := getIndexOfEnv(container.Env, env.Name)
@@ -106,7 +120,7 @@ func injectPythonSDK(pythonSpec v1alpha1.Python, pod corev1.Pod, index int) (cor
 		pod.Spec.InitContainers = append(pod.Spec.InitContainers, corev1.Container{
 			Name:      pythonInitContainerName,
 			Image:     pythonSpec.Image,
-			Command:   []string{"cp", "-r", "/autoinstrumentation/.", pythonInstrMountPath},
+			Command:   []string{"cp", "-r", autoInstrumentationSrc, pythonInstrMountPath},
 			Resources: pythonSpec.Resources,
 			VolumeMounts: []corev1.VolumeMount{{
 				Name:      pythonVolumeName,

pkg/instrumentation/python_test.go

@@ -29,6 +29,7 @@ func TestInjectPythonSDK(t *testing.T) {
 		name string
 		v1alpha1.Python
 		pod      corev1.Pod
+		wheelKind string
 		expected corev1.Pod
 		err      error
 	}{
@@ -42,6 +43,7 @@ func TestInjectPythonSDK(t *testing.T) {
 					},
 				},
 			},
+			wheelKind: "manylinux",
 			expected: corev1.Pod{
 				Spec: corev1.PodSpec{
 					Volumes: []corev1.Volume{
@@ -114,6 +116,7 @@ func TestInjectPythonSDK(t *testing.T) {
 					},
 				},
 			},
+			wheelKind: "manylinux",
 			expected: corev1.Pod{
 				Spec: corev1.PodSpec{
 					Volumes: []corev1.Volume{
@@ -187,6 +190,7 @@ func TestInjectPythonSDK(t *testing.T) {
 					},
 				},
 			},
+			wheelKind: "manylinux",
 			expected: corev1.Pod{
 				Spec: corev1.PodSpec{
 					Volumes: []corev1.Volume{
@@ -259,6 +263,7 @@ func TestInjectPythonSDK(t *testing.T) {
 					},
 				},
 			},
+			wheelKind: "manylinux",
 			expected: corev1.Pod{
 				Spec: corev1.PodSpec{
 					Volumes: []corev1.Volume{
@@ -331,6 +336,7 @@ func TestInjectPythonSDK(t *testing.T) {
 					},
 				},
 			},
+			wheelKind: "manylinux",
 			expected: corev1.Pod{
 				Spec: corev1.PodSpec{
 					Volumes: []corev1.Volume{
@@ -403,6 +409,7 @@ func TestInjectPythonSDK(t *testing.T) {
 					},
 				},
 			},
+			wheelKind: "manylinux",
 			expected: corev1.Pod{
 				Spec: corev1.PodSpec{
 					Containers: []corev1.Container{
@@ -419,11 +426,163 @@ func TestInjectPythonSDK(t *testing.T) {
 			},
 			err: fmt.Errorf("the container defines env var value via ValueFrom, envVar: %s", envPythonPath),
 		},
+		{
+			name:   "musllinux wheelKind defined",
+			Python: v1alpha1.Python{Image: "foo/bar:1"},
+			pod: corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{},
+					},
+				},
+			},
+			wheelKind: "musllinux",
+			expected: corev1.Pod{
+				Spec: corev1.PodSpec{
+					Volumes: []corev1.Volume{
+						{
+							Name: pythonVolumeName,
+							VolumeSource: corev1.VolumeSource{
+								EmptyDir: &corev1.EmptyDirVolumeSource{
+									SizeLimit: &defaultVolumeLimitSize,
+								},
+							},
+						},
+					},
+					InitContainers: []corev1.Container{
+						{
+							Name:    "opentelemetry-auto-instrumentation-python",
+							Image:   "foo/bar:1",
+							Command: []string{"cp", "-r", "/autoinstrumentation-musl/.", "/otel-auto-instrumentation-python"},
+							VolumeMounts: []corev1.VolumeMount{{
+								Name:      "opentelemetry-auto-instrumentation-python",
+								MountPath: "/otel-auto-instrumentation-python",
+							}},
+						},
+					},
+					Containers: []corev1.Container{
+						{
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "opentelemetry-auto-instrumentation-python",
+									MountPath: "/otel-auto-instrumentation-python",
+								},
+							},
+							Env: []corev1.EnvVar{
+								{
+									Name:  "PYTHONPATH",
+									Value: fmt.Sprintf("%s:%s", "/otel-auto-instrumentation-python/opentelemetry/instrumentation/auto_instrumentation", "/otel-auto-instrumentation-python"),
+								},
+								{
+									Name:  "OTEL_EXPORTER_OTLP_PROTOCOL",
+									Value: "http/protobuf",
+								},
+								{
+									Name:  "OTEL_TRACES_EXPORTER",
+									Value: "otlp",
+								},
+								{
+									Name:  "OTEL_METRICS_EXPORTER",
+									Value: "otlp",
+								},
+							},
+						},
+					},
+				},
+			},
+			err: nil,
+		},
+		{
+			name:   "wheelKind not defined",
+			Python: v1alpha1.Python{Image: "foo/bar:1"},
+			pod: corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{},
+					},
+				},
+			},
+			wheelKind: "",
+			expected: corev1.Pod{
+				Spec: corev1.PodSpec{
+					Volumes: []corev1.Volume{
+						{
+							Name: pythonVolumeName,
+							VolumeSource: corev1.VolumeSource{
+								EmptyDir: &corev1.EmptyDirVolumeSource{
+									SizeLimit: &defaultVolumeLimitSize,
+								},
+							},
+						},
+					},
+					InitContainers: []corev1.Container{
+						{
+							Name:    "opentelemetry-auto-instrumentation-python",
+							Image:   "foo/bar:1",
+							Command: []string{"cp", "-r", "/autoinstrumentation/.", "/otel-auto-instrumentation-python"},
+							VolumeMounts: []corev1.VolumeMount{{
+								Name:      "opentelemetry-auto-instrumentation-python",
+								MountPath: "/otel-auto-instrumentation-python",
+							}},
+						},
+					},
+					Containers: []corev1.Container{
+						{
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      "opentelemetry-auto-instrumentation-python",
+									MountPath: "/otel-auto-instrumentation-python",
+								},
+							},
+							Env: []corev1.EnvVar{
+								{
+									Name:  "PYTHONPATH",
+									Value: fmt.Sprintf("%s:%s", "/otel-auto-instrumentation-python/opentelemetry/instrumentation/auto_instrumentation", "/otel-auto-instrumentation-python"),
+								},
+								{
+									Name:  "OTEL_EXPORTER_OTLP_PROTOCOL",
+									Value: "http/protobuf",
+								},
+								{
+									Name:  "OTEL_TRACES_EXPORTER",
+									Value: "otlp",
+								},
+								{
+									Name:  "OTEL_METRICS_EXPORTER",
+									Value: "otlp",
+								},
+							},
+						},
+					},
+				},
+			},
+			err: nil,
+		},
+		{
+			name:   "wheelKind not supported",
+			Python: v1alpha1.Python{Image: "foo/bar:1"},
+			pod: corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{},
+					},
+				},
+			},
+			wheelKind: "not supported",
+			expected: corev1.Pod{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{},
+					},
+				},
+			},
+			err: fmt.Errorf("provided instrumentation.opentelemetry.io/python-wheel-kind annotation value 'not supported' is not supported"),
+		},
 	}
 
 	for _, test := range tests {
 		t.Run(test.name, func(t *testing.T) {
-			pod, err := injectPythonSDK(test.Python, test.pod, 0)
+			pod, err := injectPythonSDK(test.Python, test.pod, 0, test.wheelKind)
 			assert.Equal(t, test.expected, pod)
 			assert.Equal(t, test.err, err)
 		})

pkg/instrumentation/sdk.go

@@ -110,7 +110,7 @@ func (i *sdkInjector) inject(ctx context.Context, insts languageInstrumentations
 
 		for _, container := range insts.Python.Containers {
 			index := getContainerIndex(container, pod)
-			pod, err = injectPythonSDK(otelinst.Spec.Python, pod, index)
+			pod, err = injectPythonSDK(otelinst.Spec.Python, pod, index, insts.Python.AdditionalAnnotations[annotationPythonWheelKind])
 			if err != nil {
 				i.logger.Info("Skipping Python SDK injection", "reason", err.Error(), "container", pod.Spec.Containers[index].Name)
 			} else {

tests/e2e-instrumentation/instrumentation-python-musl/00-install-collector.yaml

@@ -0,0 +1,22 @@
+apiVersion: opentelemetry.io/v1alpha1
+kind: OpenTelemetryCollector
+metadata:
+  name: sidecar
+spec:
+  config: |
+    receivers:
+      otlp:
+        protocols:
+          grpc:
+          http:
+    processors:
+
+    exporters:
+      debug:
+
+    service:
+      pipelines:
+        traces:
+          receivers: [otlp]
+          exporters: [debug]
+  mode: sidecar

tests/e2e-instrumentation/instrumentation-python-musl/00-install-instrumentation.yaml

@@ -0,0 +1,30 @@
+apiVersion: opentelemetry.io/v1alpha1
+kind: Instrumentation
+metadata:
+  name: python-musl
+spec:
+  env:
+    - name: OTEL_EXPORTER_OTLP_TIMEOUT
+      value: "20"
+    - name: OTEL_TRACES_SAMPLER
+      value: parentbased_traceidratio
+    - name: OTEL_TRACES_SAMPLER_ARG
+      value: "0.85"
+    - name: SPLUNK_TRACE_RESPONSE_HEADER_ENABLED
+      value: "true"
+  exporter:
+    endpoint: http://localhost:4317
+  propagators:
+    - jaeger
+    - b3
+  sampler:
+    type: parentbased_traceidratio
+    argument: "0.25"
+  python:
+    env:
+      - name: OTEL_LOG_LEVEL
+        value: "debug"
+      - name: OTEL_TRACES_EXPORTER
+        value: otlp
+      - name: OTEL_EXPORTER_OTLP_ENDPOINT
+        value: http://localhost:4318