Use kprobe for unreliable recvmsg return probe (#1095)

grafana · Aug 16, 2024 · fe36f90 · fe36f90
1 parent c8f170d
commit fe36f90
Show file tree

Hide file tree

Showing 25 changed files with 63 additions and 15 deletions.
diff --git a/bpf/http_sock.c b/bpf/http_sock.c
@@ -434,24 +434,12 @@ int BPF_KPROBE(kprobe_tcp_recvmsg, struct sock *sk, struct msghdr *msg, size_t l
     return 0;
 }
 
-SEC("kretprobe/tcp_recvmsg")
-int BPF_KRETPROBE(kretprobe_tcp_recvmsg, int copied_len) {
-    u64 id = bpf_get_current_pid_tgid();
-
-    if (!valid_pid(id)) {
-        return 0;
-    }
-
+static __always_inline int return_recvmsg(void *ctx, u64 id, int copied_len) {
     recv_args_t *args = bpf_map_lookup_elem(&active_recv_args, &id);
 
-    bpf_dbg_printk("=== tcp_recvmsg ret id=%d args=%llx copied_len %d ===", id, args, copied_len);
+    bpf_dbg_printk("=== return recvmsg id=%d args=%llx copied_len %d ===", id, args, copied_len);
 
-    if (!args) {
-        goto done;
-    }
-
-    if (copied_len <= 0) {
-        bpf_map_delete_elem(&active_recv_args, &id);
+    if (!args || (copied_len <= 0)) {
         goto done;
     }
 
@@ -502,6 +490,32 @@ int BPF_KRETPROBE(kretprobe_tcp_recvmsg, int copied_len) {
     return 0;
 }
 
+SEC("kprobe/tcp_cleanup_rbuf")
+int BPF_KPROBE(kprobe_tcp_cleanup_rbuf, struct sock *sk, int copied) {
+    u64 id = bpf_get_current_pid_tgid();
+
+    if (!valid_pid(id)) {
+        return 0;
+    }
+
+    bpf_dbg_printk("=== tcp_cleanup_rbuf id=%d copied_len %d ===", id, copied);
+
+    return return_recvmsg(ctx, id, copied);
+}
+
+SEC("kretprobe/tcp_recvmsg")
+int BPF_KRETPROBE(kretprobe_tcp_recvmsg, int copied_len) {
+    u64 id = bpf_get_current_pid_tgid();
+
+    if (!valid_pid(id)) {
+        return 0;
+    }
+
+    bpf_dbg_printk("=== kretprobe_tcp_recvmsg id=%d copied_len %d ===", id, copied_len);
+
+    return return_recvmsg(ctx, id, copied_len);
+}
+
 // Fall-back in case we don't see kretprobe on tcp_recvmsg in high network volume situations
 SEC("socket/http_filter")
 int socket__http_filter(struct __sk_buff *skb) {

diff --git a/pkg/internal/ebpf/httpfltr/bpf_arm64_bpfel.go b/pkg/internal/ebpf/httpfltr/bpf_arm64_bpfel.go
diff --git a/pkg/internal/ebpf/httpfltr/bpf_arm64_bpfel.o b/pkg/internal/ebpf/httpfltr/bpf_arm64_bpfel.o
diff --git a/pkg/internal/ebpf/httpfltr/bpf_debug_arm64_bpfel.go b/pkg/internal/ebpf/httpfltr/bpf_debug_arm64_bpfel.go
diff --git a/pkg/internal/ebpf/httpfltr/bpf_debug_arm64_bpfel.o b/pkg/internal/ebpf/httpfltr/bpf_debug_arm64_bpfel.o
diff --git a/pkg/internal/ebpf/httpfltr/bpf_debug_x86_bpfel.go b/pkg/internal/ebpf/httpfltr/bpf_debug_x86_bpfel.go
diff --git a/pkg/internal/ebpf/httpfltr/bpf_debug_x86_bpfel.o b/pkg/internal/ebpf/httpfltr/bpf_debug_x86_bpfel.o
diff --git a/pkg/internal/ebpf/httpfltr/bpf_tp_arm64_bpfel.go b/pkg/internal/ebpf/httpfltr/bpf_tp_arm64_bpfel.go
diff --git a/pkg/internal/ebpf/httpfltr/bpf_tp_arm64_bpfel.o b/pkg/internal/ebpf/httpfltr/bpf_tp_arm64_bpfel.o
diff --git a/pkg/internal/ebpf/httpfltr/bpf_tp_debug_arm64_bpfel.go b/pkg/internal/ebpf/httpfltr/bpf_tp_debug_arm64_bpfel.go
diff --git a/pkg/internal/ebpf/httpfltr/bpf_tp_debug_arm64_bpfel.o b/pkg/internal/ebpf/httpfltr/bpf_tp_debug_arm64_bpfel.o
diff --git a/pkg/internal/ebpf/httpfltr/bpf_tp_debug_x86_bpfel.go b/pkg/internal/ebpf/httpfltr/bpf_tp_debug_x86_bpfel.go
diff --git a/pkg/internal/ebpf/httpfltr/bpf_tp_debug_x86_bpfel.o b/pkg/internal/ebpf/httpfltr/bpf_tp_debug_x86_bpfel.o
diff --git a/pkg/internal/ebpf/httpfltr/bpf_tp_x86_bpfel.go b/pkg/internal/ebpf/httpfltr/bpf_tp_x86_bpfel.go
diff --git a/pkg/internal/ebpf/httpfltr/bpf_tp_x86_bpfel.o b/pkg/internal/ebpf/httpfltr/bpf_tp_x86_bpfel.o
diff --git a/pkg/internal/ebpf/httpfltr/bpf_x86_bpfel.go b/pkg/internal/ebpf/httpfltr/bpf_x86_bpfel.go
diff --git a/pkg/internal/ebpf/httpfltr/bpf_x86_bpfel.o b/pkg/internal/ebpf/httpfltr/bpf_x86_bpfel.o
diff --git a/pkg/internal/ebpf/httpfltr/httpfltr.go b/pkg/internal/ebpf/httpfltr/httpfltr.go
@@ -228,6 +228,9 @@ func (p *Tracer) KProbes() map[string]ebpfcommon.FunctionPrograms {
 			Start:    p.bpfObjects.KprobeTcpRecvmsg,
 			End:      p.bpfObjects.KretprobeTcpRecvmsg,
 		},
+		"tcp_cleanup_rbuf": {
+			Start: p.bpfObjects.KprobeTcpCleanupRbuf, // this kprobe runs the same code as recvmsg return, we use it because kretprobes can be unreliable.
+		},
 		"sys_clone": {
 			Required: true,
 			End:      p.bpfObjects.KretprobeSysClone,

diff --git a/test/oats/kafka/docker-compose-beyla-java-kafka.yml b/test/oats/kafka/docker-compose-beyla-java-kafka.yml
@@ -54,6 +54,7 @@ services:
       BEYLA_METRICS_INTERVAL: "10ms"
       BEYLA_BPF_BATCH_TIMEOUT: "10ms"
       BEYLA_LOG_LEVEL: "DEBUG"
+      BEYLA_BPF_DEBUG: "true"
       OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector:4318"
     depends_on:
       testserver:

diff --git a/test/oats/kafka/docker-compose-beyla-python-kafka.yml b/test/oats/kafka/docker-compose-beyla-python-kafka.yml
@@ -61,6 +61,7 @@ services:
       BEYLA_METRICS_INTERVAL: "10ms"
       BEYLA_BPF_BATCH_TIMEOUT: "10ms"
       BEYLA_LOG_LEVEL: "DEBUG"
+      BEYLA_BPF_DEBUG: "true"
       OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector:4318"
     depends_on:
       testserver:

diff --git a/test/oats/redis/docker-compose-beyla-go-redis.yml b/test/oats/redis/docker-compose-beyla-go-redis.yml
@@ -37,6 +37,7 @@ services:
       BEYLA_METRICS_INTERVAL: "10ms"
       BEYLA_BPF_BATCH_TIMEOUT: "10ms"
       BEYLA_LOG_LEVEL: "DEBUG"
+      BEYLA_BPF_DEBUG: "true"
       OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector:4318"
     depends_on:
       testserver:

diff --git a/test/oats/redis/docker-compose-beyla-redis.yml b/test/oats/redis/docker-compose-beyla-redis.yml
@@ -37,6 +37,7 @@ services:
       BEYLA_METRICS_INTERVAL: "10ms"
       BEYLA_BPF_BATCH_TIMEOUT: "10ms"
       BEYLA_LOG_LEVEL: "DEBUG"
+      BEYLA_BPF_DEBUG: "true"
       OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector:4318"
     depends_on:
       testserver:

diff --git a/test/oats/sql/docker-compose-beyla-gosqlclient-statement.yml b/test/oats/sql/docker-compose-beyla-gosqlclient-statement.yml
@@ -31,6 +31,7 @@ services:
       BEYLA_METRICS_INTERVAL: "10ms"
       BEYLA_BPF_BATCH_TIMEOUT: "10ms"
       BEYLA_LOG_LEVEL: "DEBUG"
+      BEYLA_BPF_DEBUG: "true"
       OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector:4318"
     depends_on:
       testserver:

diff --git a/test/oats/sql/docker-compose-beyla-gosqlclient.yml b/test/oats/sql/docker-compose-beyla-gosqlclient.yml
@@ -31,6 +31,7 @@ services:
       BEYLA_METRICS_INTERVAL: "10ms"
       BEYLA_BPF_BATCH_TIMEOUT: "10ms"
       BEYLA_LOG_LEVEL: "DEBUG"
+      BEYLA_BPF_DEBUG: "true"
       OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector:4318"
     depends_on:
       testserver:

diff --git a/test/oats/sql/docker-compose-beyla-sql.yml b/test/oats/sql/docker-compose-beyla-sql.yml
@@ -42,6 +42,7 @@ services:
       BEYLA_METRICS_INTERVAL: "10ms"
       BEYLA_BPF_BATCH_TIMEOUT: "10ms"
       BEYLA_LOG_LEVEL: "DEBUG"
+      BEYLA_BPF_DEBUG: "true"
       OTEL_EXPORTER_OTLP_ENDPOINT: "http://collector:4318"
     depends_on:
       testserver: