[feat] Add ARM support (#82)

* Add ARM64 Support

Ported from odigos-io/opentelemetry-go-instrumentation#53

Co-authored-by: Eden Federman <[email protected]>

* update net/http probe to get args from ctx instead of goroutine

* fix c file formatting

* more c file formatting tidy

* get http header ctx pointer for net/http instrumentation

* switch back to goroutine

* set bpf2go target as bpfel

* revert bpf2go target to $TARGET

* remove generated probe go files

* set TARGET in build workflow to amd64

* add default docker image name to makefile

* remove TARGET and IMG env vars in build workflow

* set TARGETARCH in build workflow

* tidy up build workflow

* update go generate to use set targets directly

* add qemu

* lets see what happens with buildarch

* try buildx

* try buildx

* testing if we need buildx setup step

* remove --no-cache option

* switch back to specific eBPF arch targets

* tidy up docker/make files

* set default target as amd64

* readd buildx

* fix buildx arg

* fix image name

* wait for longer for e2e pods

* Revert "wait for longer for e2e pods"

This reverts commit 2ef3b6e.

* apply fix for instrumenting stripped binaries

ported from: odigos-io/opentelemetry-go-instrumentation#56

* Revert "apply fix for instrumenting stripped binaries"

This reverts commit 5d58216.

* use $TARGET for gin bpf2cmd

* update ptrace_linux_arm

* add changelog entry

* extend get_goroutine to work with go <1.17

* add missing licese files

* fix bad merge in Makefile

* remove unnessary additional funcname for net/http instrumentor

* update bpf2go generate commands to use both arm and amd

* Update changelog entry

Co-authored-by: Tyler Yahn <[email protected]>

* re-add comments to pt_regs struct

* add build guards to findReturnInstructions arch files

* unexport error const

* separate imports

* update mapSize description

* Fix changelog entry

Co-authored-by: Tyler Yahn <[email protected]>

---------

Co-authored-by: Eden Federman <[email protected]>
Co-authored-by: Purvi Kanal <[email protected]>
Co-authored-by: Robb Kidd <[email protected]>
Co-authored-by: Tyler Yahn <[email protected]>

.github/workflows/build.yaml

@@ -34,7 +34,7 @@ jobs:
         uses: actions/checkout@v3
       - name: Build auto-instrumentation
         run: |
-          IMG=otel-go-instrumentation make docker-build
+          make docker-build
   offsets:
     runs-on: ubuntu-latest
     steps:

.github/workflows/release.yaml

@@ -38,4 +38,4 @@ jobs:
           push: true
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          platforms: linux/amd64
+          platforms: linux/amd64,linux/arm64

CHANGELOG.md

@@ -11,6 +11,7 @@ OpenTelemetry Go Automatic Instrumentation adheres to [Semantic Versioning](http
 ### Added
 
 - Add [gin-gonic/gin](https://github.com/gin-gonic/gin) instrumentation. ([#100](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/100))
+- Add ARM64 support. ([#82](https://github.com/open-telemetry/opentelemetry-go-instrumentation/pull/82))
 
 ### Changed
 

Dockerfile

@@ -1,6 +1,7 @@
 FROM fedora:38 as builder
+ARG TARGETARCH
 RUN dnf install clang llvm make libbpf-devel -y
-RUN curl -LO https://go.dev/dl/go1.20.linux-amd64.tar.gz && tar -C /usr/local -xzf go*.linux-amd64.tar.gz
+RUN curl -LO https://go.dev/dl/go1.20.linux-${TARGETARCH}.tar.gz && tar -C /usr/local -xzf go*.linux-${TARGETARCH}.tar.gz
 ENV PATH="/usr/local/go/bin:${PATH}"
 WORKDIR /app
 COPY . .

Makefile

@@ -10,7 +10,7 @@ OTEL_GO_MOD_DIRS := $(filter-out $(TOOLS_MOD_DIR), $(ALL_GO_MOD_DIRS))
 
 # Build the list of include directories to compile the bpf program
 BPF_INCLUDE += -I${REPODIR}/include/libbpf
-BPF_INCLUDE+= -I${REPODIR}/include
+BPF_INCLUDE += -I${REPODIR}/include
 
 .DEFAULT_GOAL := precommit
 
@@ -30,6 +30,8 @@ $(TOOLS)/multimod: PACKAGE=go.opentelemetry.io/build-tools/multimod
 GOLICENSES = $(TOOLS)/go-licenses
 $(TOOLS)/go-licenses: PACKAGE=github.com/google/go-licenses
 
+IMG_NAME ?= otel-go-instrumentation
+
 GOLANGCI_LINT = $(TOOLS)/golangci-lint
 $(TOOLS)/golangci-lint: PACKAGE=github.com/golangci/golangci-lint/cmd/golangci-lint
 
@@ -48,6 +50,7 @@ test/%:
 .PHONY: generate
 generate: export CFLAGS := $(BPF_INCLUDE)
 generate: go-mod-tidy
+generate:
 	go generate ./...
 
 .PHONY: go-mod-tidy
@@ -68,11 +71,11 @@ golangci-lint/%: | $(GOLANGCI_LINT)
 
 .PHONY: build
 build: generate
-	GOOS=linux GOARCH=amd64 go build -o otel-go-instrumentation cli/main.go
+	GOOS=linux go build -o otel-go-instrumentation cli/main.go
 
 .PHONY: docker-build
 docker-build:
-	docker build -t $(IMG) .
+	docker buildx build -t $(IMG_NAME) .
 
 .PHONY: offsets
 offsets:
@@ -117,7 +120,7 @@ fixture-gorillamux: fixtures/gorillamux
 fixture-gin: fixtures/gin
 fixtures/%: LIBRARY=$*
 fixtures/%:
-	IMG=otel-go-instrumentation $(MAKE) docker-build
+	$(MAKE) docker-build
 	cd test/e2e/$(LIBRARY) && docker build -t sample-app .
 	kind create cluster
 	kind load docker-image otel-go-instrumentation sample-app

include/alloc.h

@@ -88,6 +88,12 @@ static __always_inline void *write_target_data(void *data, s32 size)
     {
         s32 start_index = 0;
         u64 updated_start = start + size;
+
+        // align updated_start to 8 bytes
+        if (updated_start % 8 != 0) {
+            updated_start += 8 - (updated_start % 8);
+        }
+
         bpf_map_update_elem(&alloc_map, &start_index, &updated_start, BPF_ANY);
         return target;
     }

include/arguments.h

@@ -13,6 +13,7 @@
 // limitations under the License.
 
 #include "common.h"
+#include "bpf_tracing.h"
 #include "bpf_helpers.h"
 #include <stdbool.h>
 
@@ -24,23 +25,23 @@ void *get_argument_by_reg(struct pt_regs *ctx, int index)
     switch (index)
     {
     case 1:
-        return (void *)(ctx->rax);
+        return (void *)GO_PARAM1(ctx);
     case 2:
-        return (void *)(ctx->rbx);
+        return (void *)GO_PARAM2(ctx);
     case 3:
-        return (void *)(ctx->rcx);
+        return (void *)GO_PARAM3(ctx);
     case 4:
-        return (void *)(ctx->rdi);
+        return (void *)GO_PARAM4(ctx);
     case 5:
-        return (void *)(ctx->rsi);
+        return (void *)GO_PARAM5(ctx);
     case 6:
-        return (void *)(ctx->r8);
+        return (void *)GO_PARAM6(ctx);
     case 7:
-        return (void *)(ctx->r9);
+        return (void *)GO_PARAM7(ctx);
     case 8:
-        return (void *)(ctx->r10);
+        return (void *)GO_PARAM8(ctx);
     case 9:
-        return (void *)(ctx->r11);
+        return (void *)GO_PARAM9(ctx);
     default:
         return NULL;
     }
@@ -49,7 +50,7 @@ void *get_argument_by_reg(struct pt_regs *ctx, int index)
 void *get_argument_by_stack(struct pt_regs *ctx, int index)
 {
     void *ptr = 0;
-    bpf_probe_read(&ptr, sizeof(ptr), (void *)(ctx->rsp + (index * 8)));
+    bpf_probe_read(&ptr, sizeof(ptr), (void *)(PT_REGS_SP(ctx) + (index * 8)));
     return ptr;
 }
 
@@ -63,8 +64,12 @@ void *get_argument(struct pt_regs *ctx, int index)
     return get_argument_by_stack(ctx, index);
 }
 
-// In x86, current goroutine is pointed by r14, according to
-// https://go.googlesource.com/go/+/refs/heads/dev.regabi/src/cmd/compile/internal-abi.md#amd64-architecture
-inline void *get_goroutine_address(struct pt_regs *ctx) {
-    return (void *)(ctx->r14);
+inline void *get_goroutine_address(struct pt_regs *ctx, int go_ctx_index)
+{
+    if (is_registers_abi)
+    {
+        return (void *)GOROUTINE(ctx);
+    }
+
+    return get_argument_by_stack(ctx, go_ctx_index);
 }

include/common.h

@@ -84,41 +84,68 @@ enum
 #define BPF_F_INDEX_MASK 0xffffffffULL
 #define BPF_F_CURRENT_CPU BPF_F_INDEX_MASK
 
-#define PT_REGS_RC(x) ((x)->rax)
-struct pt_regs
-{
+#if defined(__TARGET_ARCH_x86)
+struct pt_regs {
 	/*
 	 * C ABI says these regs are callee-preserved. They aren't saved on kernel entry
 	 * unless syscall needs a complete, fully filled "struct pt_regs".
 	 */
-	unsigned long r15;
-	unsigned long r14;
-	unsigned long r13;
-	unsigned long r12;
-	unsigned long rbp;
-	unsigned long rbx;
+	long unsigned int r15;
+	long unsigned int r14;
+	long unsigned int r13;
+	long unsigned int r12;
+	long unsigned int bp;
+	long unsigned int bx;
 	/* These regs are callee-clobbered. Always saved on kernel entry. */
-	unsigned long r11;
-	unsigned long r10;
-	unsigned long r9;
-	unsigned long r8;
-	unsigned long rax;
-	unsigned long rcx;
-	unsigned long rdx;
-	unsigned long rsi;
-	unsigned long rdi;
+	long unsigned int r11;
+	long unsigned int r10;
+	long unsigned int r9;
+	long unsigned int r8;
+	long unsigned int ax;
+	long unsigned int cx;
+	long unsigned int dx;
+	long unsigned int si;
+	long unsigned int di;
 	/*
 	 * On syscall entry, this is syscall#. On CPU exception, this is error code.
 	 * On hw interrupt, it's IRQ number:
 	 */
-	unsigned long orig_rax;
+	long unsigned int orig_ax;
 	/* Return frame for iretq */
-	unsigned long rip;
-	unsigned long cs;
-	unsigned long eflags;
-	unsigned long rsp;
-	unsigned long ss;
+	long unsigned int ip;
+	long unsigned int cs;
+	long unsigned int flags;
+	long unsigned int sp;
+	long unsigned int ss;
 	/* top of stack page */
 };
+#elif defined(__TARGET_ARCH_arm64)
+struct user_pt_regs {
+	__u64 regs[31];
+	__u64 sp;
+	__u64 pc;
+	__u64 pstate;
+};
+
+struct pt_regs {
+	union {
+		struct user_pt_regs user_regs;
+		struct {
+			u64 regs[31];
+			u64 sp;
+			u64 pc;
+			u64 pstate;
+		};
+	};
+	u64 orig_x0;
+	s32 syscallno;
+	u32 unused2;
+	u64 orig_addr_limit;
+	u64 pmr_save;
+	u64 stackframe[2];
+	u64 lockdep_hardirqs;
+	u64 exit_rcu;
+};
+#endif
 
 #endif /* __VMLINUX_H__ */

include/go_types.h

@@ -54,7 +54,11 @@ static __always_inline struct go_string write_user_go_string(char *str, u32 len)
     new_string.len = len;
 
     // Copy new string struct to userspace
-    write_target_data((void *)&new_string, sizeof(new_string));
+    void *res = write_target_data((void *)&new_string, sizeof(new_string));
+    if (res == NULL) {
+        new_string.len = 0;
+    }
+
     return new_string;
 }
 

include/libbpf/bpf_tracing.h

@@ -24,6 +24,9 @@
 #elif defined(__TARGET_ARCH_sparc)
 	#define bpf_target_sparc
 	#define bpf_target_defined
+#elif defined(__TARGET_ARCH_riscv)
+	#define bpf_target_riscv
+	#define bpf_target_defined
 #else
 
 /* Fall back to what the compiler says */
@@ -48,6 +51,9 @@
 #elif defined(__sparc__)
 	#define bpf_target_sparc
 	#define bpf_target_defined
+#elif defined(__riscv) && __riscv_xlen == 64
+	#define bpf_target_riscv
+	#define bpf_target_defined
 #endif /* no compiler target */
 
 #endif
@@ -60,6 +66,17 @@
 
 #if defined(__KERNEL__) || defined(__VMLINUX_H__)
 
+#define GO_PARAM1(x) ((x)->ax)
+#define GO_PARAM2(x) ((x)->bx)
+#define GO_PARAM3(x) ((x)->cx)
+#define GO_PARAM4(x) ((x)->di)
+#define GO_PARAM5(x) ((x)->si)
+#define GO_PARAM6(x) ((x)->r8)
+#define GO_PARAM7(x) ((x)->r9)
+#define GO_PARAM8(x) ((x)->r10)
+#define GO_PARAM9(x) ((x)->r11)
+#define GOROUTINE(x) ((x)->r14)
+
 #define PT_REGS_PARM1(x) ((x)->di)
 #define PT_REGS_PARM2(x) ((x)->si)
 #define PT_REGS_PARM3(x) ((x)->dx)
@@ -192,6 +209,18 @@ struct pt_regs;
 /* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */
 struct pt_regs;
 #define PT_REGS_ARM64 const volatile struct user_pt_regs
+
+#define GO_PARAM1(x) (((PT_REGS_ARM64 *)(x))->regs[0])
+#define GO_PARAM2(x) (((PT_REGS_ARM64 *)(x))->regs[1])
+#define GO_PARAM3(x) (((PT_REGS_ARM64 *)(x))->regs[2])
+#define GO_PARAM4(x) (((PT_REGS_ARM64 *)(x))->regs[3])
+#define GO_PARAM5(x) (((PT_REGS_ARM64 *)(x))->regs[4])
+#define GO_PARAM6(x) (((PT_REGS_ARM64 *)(x))->regs[5])
+#define GO_PARAM7(x) (((PT_REGS_ARM64 *)(x))->regs[6])
+#define GO_PARAM8(x) (((PT_REGS_ARM64 *)(x))->regs[7])
+#define GO_PARAM9(x) (((PT_REGS_ARM64 *)(x))->regs[8])
+#define GOROUTINE(x) (((PT_REGS_ARM64 *)(x))->regs[28])
+
 #define PT_REGS_PARM1(x) (((PT_REGS_ARM64 *)(x))->regs[0])
 #define PT_REGS_PARM2(x) (((PT_REGS_ARM64 *)(x))->regs[1])
 #define PT_REGS_PARM3(x) (((PT_REGS_ARM64 *)(x))->regs[2])
@@ -288,6 +317,32 @@ struct pt_regs;
 #define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), pc)
 #endif
 
+#elif defined(bpf_target_riscv)
+
+struct pt_regs;
+#define PT_REGS_RV const volatile struct user_regs_struct
+#define PT_REGS_PARM1(x) (((PT_REGS_RV *)(x))->a0)
+#define PT_REGS_PARM2(x) (((PT_REGS_RV *)(x))->a1)
+#define PT_REGS_PARM3(x) (((PT_REGS_RV *)(x))->a2)
+#define PT_REGS_PARM4(x) (((PT_REGS_RV *)(x))->a3)
+#define PT_REGS_PARM5(x) (((PT_REGS_RV *)(x))->a4)
+#define PT_REGS_RET(x) (((PT_REGS_RV *)(x))->ra)
+#define PT_REGS_FP(x) (((PT_REGS_RV *)(x))->s5)
+#define PT_REGS_RC(x) (((PT_REGS_RV *)(x))->a5)
+#define PT_REGS_SP(x) (((PT_REGS_RV *)(x))->sp)
+#define PT_REGS_IP(x) (((PT_REGS_RV *)(x))->epc)
+
+#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a0)
+#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a1)
+#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a2)
+#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a3)
+#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a4)
+#define PT_REGS_RET_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), ra)
+#define PT_REGS_FP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), fp)
+#define PT_REGS_RC_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a5)
+#define PT_REGS_SP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), sp)
+#define PT_REGS_IP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), epc)
+
 #endif
 
 #if defined(bpf_target_powerpc)

pkg/instrumentors/bpf/github.com/gin-gonic/gin/bpf/probe.bpf.c

@@ -46,6 +46,7 @@ struct {
 volatile const u64 method_ptr_pos;
 volatile const u64 url_ptr_pos;
 volatile const u64 path_ptr_pos;
+volatile const u64 ctx_ptr_pos;
 
 // This instrumentation attaches uprobe to the following function:
 // func (engine *Engine) ServeHTTP(w http.ResponseWriter, r *http.Request)
@@ -79,21 +80,18 @@ int uprobe_GinEngine_ServeHTTP(struct pt_regs *ctx) {
     bpf_probe_read(&httpReq.path, path_size, path_ptr);
 
     // Get goroutine pointer
-    void *goroutine = get_goroutine_address(ctx);
+    void *goroutine = get_goroutine_address(ctx, ctx_ptr_pos);
 
     // Write event
     httpReq.sc = generate_span_context();
     bpf_map_update_elem(&context_to_http_events, &goroutine, &httpReq, 0);
-    long res = bpf_map_update_elem(&spans_in_progress, &goroutine, &httpReq.sc, 0);
+    bpf_map_update_elem(&spans_in_progress, &goroutine, &httpReq.sc, 0);
     return 0;
 }
 
 SEC("uprobe/GinEngine_ServeHTTP")
 int uprobe_GinEngine_ServeHTTP_Returns(struct pt_regs *ctx) {
-    u64 request_pos = 4;
-    void *req_ptr = get_argument(ctx, request_pos);
-    void *goroutine = get_goroutine_address(ctx);
-
+    void *goroutine = get_goroutine_address(ctx, ctx_ptr_pos);
     void *httpReq_ptr = bpf_map_lookup_elem(&context_to_http_events, &goroutine);
     struct http_request_t httpReq = {};
     bpf_probe_read(&httpReq, sizeof(httpReq), httpReq_ptr);