From 49607d32dc51a9492d21872230469faef11849ea Mon Sep 17 00:00:00 2001 From: Eric Long Date: Wed, 24 Jul 2024 22:57:22 +0800 Subject: [PATCH] Add i386 BPF_PSEUDO_CALL workaround --- Makefile | 5 ++- bpf/egress.c | 6 ++-- bpf/ingress.c | 4 +-- bpf/mimic-impl.h | 84 +++++++++++++++++++++++++++++++++++++++++++++++ bpf/mimic.c | 85 +++--------------------------------------------- bpf/mimic.h | 9 ++++- 6 files changed, 105 insertions(+), 88 deletions(-) create mode 100644 bpf/mimic-impl.h diff --git a/Makefile b/Makefile index 757663a..43e1e7e 100644 --- a/Makefile +++ b/Makefile @@ -30,11 +30,14 @@ KERNEL_VMLINUX := /lib/modules/$(KERNEL_UNAME)/build/vmlinux else $(error vmlinux file not found) endif - else BPF_CFLAGS += -D_MIMIC_BPF_TARGET_ARCH_$(shell uname -m) endif +ifneq ($(findstring $(shell uname -m),i386 i486 i586 i686),) +BPF_CFLAGS += -D_MIMIC_BPF_INLINE_ALL_FUNCS +endif + mimic_common_headers := $(wildcard common/*.h) mimic_bpf_src := $(wildcard bpf/*.c) diff --git a/bpf/egress.c b/bpf/egress.c index 9bdd335..229e98f 100644 --- a/bpf/egress.c +++ b/bpf/egress.c @@ -9,7 +9,7 @@ #include "mimic.h" // Extend socket buffer and move n bytes from front to back. -static int mangle_data(struct __sk_buff* skb, __u16 offset, __be32* csum_diff) { +static __always_inline int mangle_data(struct __sk_buff* skb, __u16 offset, __be32* csum_diff) { __u16 data_len = skb->len - offset; try_shot(bpf_skb_change_tail(skb, skb->len + TCP_UDP_HEADER_DIFF, 0)); __u8 buf[TCP_UDP_HEADER_DIFF + 4] = {}; @@ -32,8 +32,8 @@ static int mangle_data(struct __sk_buff* skb, __u16 offset, __be32* csum_diff) { return TC_ACT_OK; } -static inline void update_tcp_header(struct tcphdr* tcp, __u16 payload_len, __u32 seq, - __u32 ack_seq, __u32 cwnd) { +static __always_inline void update_tcp_header(struct tcphdr* tcp, __u16 payload_len, __u32 seq, + __u32 ack_seq, __u32 cwnd) { tcp->seq = htonl(seq); tcp->ack_seq = htonl(ack_seq); tcp_flag_word(tcp) = 0; diff --git a/bpf/ingress.c b/bpf/ingress.c index e48526e..e904562 100644 --- a/bpf/ingress.c +++ b/bpf/ingress.c @@ -15,7 +15,7 @@ // applied to every data packet. For the same reason, middleboxes probably only append options like // MSS on handshake packets since there is no data at the end to move, so not finishing this TODO is // probably going to be fine. -static inline int restore_data(struct xdp_md* xdp, __u16 offset, __u32 buf_len, __be32* csum_diff) { +static __always_inline int restore_data(struct xdp_md* xdp, __u16 offset, __u32 buf_len, __be32* csum_diff) { __u8 buf[TCP_UDP_HEADER_DIFF + 4] = {}; __u16 data_len = buf_len - offset; __u32 copy_len = min(data_len, TCP_UDP_HEADER_DIFF); @@ -45,7 +45,7 @@ struct tcp_options { // TODO: more fields }; -static inline int read_tcp_options(struct xdp_md* xdp, struct tcphdr* tcp, __u32 ip_end, +static __always_inline int read_tcp_options(struct xdp_md* xdp, struct tcphdr* tcp, __u32 ip_end, struct tcp_options* opt) { __u8 opt_buf[80] = {}; __u32 len = (tcp->doff << 2) - sizeof(*tcp); diff --git a/bpf/mimic-impl.h b/bpf/mimic-impl.h new file mode 100644 index 0000000..e20b167 --- /dev/null +++ b/bpf/mimic-impl.h @@ -0,0 +1,84 @@ +#include "vmlinux.h" + +#include + +#include "common/try.h" +#include "kmod/mimic.h" +#include "mimic.h" + +_mimic_maybe_static_inline int send_ctrl_packet(struct conn_tuple* conn, __u16 flags, __u32 seq, __u32 ack_seq, __u32 cwnd) { + if (!conn) return -1; + struct rb_item* item = bpf_ringbuf_reserve(&mimic_rb, sizeof(*item), 0); + if (!item) return -1; + item->type = RB_ITEM_SEND_OPTIONS; + item->send_options = (struct send_options){ + .conn = *conn, + .syn = flags & SYN, + .ack = flags & ACK, + .rst = flags & RST, + .seq = seq, + .ack_seq = ack_seq, + .cwnd = cwnd, + }; + bpf_ringbuf_submit(item, 0); + return 0; +} + +_mimic_maybe_static_inline int store_packet(struct __sk_buff* skb, __u32 pkt_off, struct conn_tuple* key) { + int retcode; + __u32 data_len = skb->len - pkt_off; + if (!key || data_len > MAX_PACKET_SIZE) return TC_ACT_SHOT; + + bool has_remainder = data_len % SEGMENT_SIZE; + __u32 segments = data_len / SEGMENT_SIZE + has_remainder; + __u32 alloc_size = sizeof(struct rb_item) + segments * SEGMENT_SIZE; + struct bpf_dynptr ptr = {}; + if (bpf_ringbuf_reserve_dynptr(&mimic_rb, alloc_size, 0, &ptr) < 0) cleanup(TC_ACT_SHOT); + + struct rb_item* item = bpf_dynptr_data(&ptr, 0, sizeof(*item)); + if (!item) cleanup(TC_ACT_SHOT); + item->type = RB_ITEM_STORE_PACKET; + item->store_packet.conn_key = *key; + item->store_packet.len = data_len; + item->store_packet.l4_csum_partial = mimic_inspect_skb(skb)->ip_summed == CHECKSUM_PARTIAL; + + char* packet = NULL; + __u32 offset = 0, i = 0; + for (; i < segments - has_remainder; i++) { + if (i > MAX_PACKET_SIZE / SEGMENT_SIZE + 1) break; + offset = i * SEGMENT_SIZE; + packet = bpf_dynptr_data(&ptr, sizeof(*item) + offset, SEGMENT_SIZE); + if (!packet) cleanup(TC_ACT_SHOT); + if (bpf_skb_load_bytes(skb, pkt_off + offset, packet, SEGMENT_SIZE) < 0) cleanup(TC_ACT_SHOT); + } + if (has_remainder) { + offset = i * SEGMENT_SIZE; + __u32 copy_len = data_len - offset; + if (copy_len > 0 && copy_len < SEGMENT_SIZE) { + // HACK: see above + if (copy_len < 2) copy_len = 1; + if (copy_len > SEGMENT_SIZE - 2) copy_len = SEGMENT_SIZE - 1; + + packet = bpf_dynptr_data(&ptr, sizeof(*item) + offset, SEGMENT_SIZE); + if (!packet) cleanup(TC_ACT_SHOT); + if (bpf_skb_load_bytes(skb, pkt_off + offset, packet, copy_len) < 0) cleanup(TC_ACT_SHOT); + } + } + bpf_ringbuf_submit_dynptr(&ptr, 0); + return TC_ACT_STOLEN; +cleanup: + bpf_ringbuf_discard_dynptr(&ptr, 0); + return retcode; +} + +// Need to manually clear conn.pktbuf in eBPF +_mimic_maybe_static_inline int use_pktbuf(enum rb_item_type type, uintptr_t buf) { + if (type != RB_ITEM_CONSUME_PKTBUF && type != RB_ITEM_FREE_PKTBUF) return -1; + if (!buf) return 0; + struct rb_item* item = bpf_ringbuf_reserve(&mimic_rb, sizeof(*item), 0); + if (!item) return -1; + item->type = type; + item->pktbuf = buf; + bpf_ringbuf_submit(item, 0); + return 0; +} diff --git a/bpf/mimic.c b/bpf/mimic.c index 43ac19c..1814711 100644 --- a/bpf/mimic.c +++ b/bpf/mimic.c @@ -1,92 +1,15 @@ -#include "vmlinux.h" +#include "vmlinux.h" // IWYU pragma: keep #include -#include "common/try.h" -#include "kmod/mimic.h" -#include "mimic.h" - int log_verbosity = 0; struct mimic_whitelist_map mimic_whitelist SEC(".maps"); struct mimic_conns_map mimic_conns SEC(".maps"); struct mimic_rb_map mimic_rb SEC(".maps"); -int send_ctrl_packet(struct conn_tuple* conn, __u16 flags, __u32 seq, __u32 ack_seq, __u32 cwnd) { - if (!conn) return -1; - struct rb_item* item = bpf_ringbuf_reserve(&mimic_rb, sizeof(*item), 0); - if (!item) return -1; - item->type = RB_ITEM_SEND_OPTIONS; - item->send_options = (struct send_options){ - .conn = *conn, - .syn = flags & SYN, - .ack = flags & ACK, - .rst = flags & RST, - .seq = seq, - .ack_seq = ack_seq, - .cwnd = cwnd, - }; - bpf_ringbuf_submit(item, 0); - return 0; -} - -int store_packet(struct __sk_buff* skb, __u32 pkt_off, struct conn_tuple* key) { - int retcode; - __u32 data_len = skb->len - pkt_off; - if (!key || data_len > MAX_PACKET_SIZE) return TC_ACT_SHOT; - - bool has_remainder = data_len % SEGMENT_SIZE; - __u32 segments = data_len / SEGMENT_SIZE + has_remainder; - __u32 alloc_size = sizeof(struct rb_item) + segments * SEGMENT_SIZE; - struct bpf_dynptr ptr = {}; - if (bpf_ringbuf_reserve_dynptr(&mimic_rb, alloc_size, 0, &ptr) < 0) cleanup(TC_ACT_SHOT); - - struct rb_item* item = bpf_dynptr_data(&ptr, 0, sizeof(*item)); - if (!item) cleanup(TC_ACT_SHOT); - item->type = RB_ITEM_STORE_PACKET; - item->store_packet.conn_key = *key; - item->store_packet.len = data_len; - item->store_packet.l4_csum_partial = mimic_inspect_skb(skb)->ip_summed == CHECKSUM_PARTIAL; - - char* packet = NULL; - __u32 offset = 0, i = 0; - for (; i < segments - has_remainder; i++) { - if (i > MAX_PACKET_SIZE / SEGMENT_SIZE + 1) break; - offset = i * SEGMENT_SIZE; - packet = bpf_dynptr_data(&ptr, sizeof(*item) + offset, SEGMENT_SIZE); - if (!packet) cleanup(TC_ACT_SHOT); - if (bpf_skb_load_bytes(skb, pkt_off + offset, packet, SEGMENT_SIZE) < 0) cleanup(TC_ACT_SHOT); - } - if (has_remainder) { - offset = i * SEGMENT_SIZE; - __u32 copy_len = data_len - offset; - if (copy_len > 0 && copy_len < SEGMENT_SIZE) { - // HACK: see above - if (copy_len < 2) copy_len = 1; - if (copy_len > SEGMENT_SIZE - 2) copy_len = SEGMENT_SIZE - 1; - - packet = bpf_dynptr_data(&ptr, sizeof(*item) + offset, SEGMENT_SIZE); - if (!packet) cleanup(TC_ACT_SHOT); - if (bpf_skb_load_bytes(skb, pkt_off + offset, packet, copy_len) < 0) cleanup(TC_ACT_SHOT); - } - } - bpf_ringbuf_submit_dynptr(&ptr, 0); - return TC_ACT_STOLEN; -cleanup: - bpf_ringbuf_discard_dynptr(&ptr, 0); - return retcode; -} - -// Need to manually clear conn.pktbuf in eBPF -int use_pktbuf(enum rb_item_type type, uintptr_t buf) { - if (type != RB_ITEM_CONSUME_PKTBUF && type != RB_ITEM_FREE_PKTBUF) return -1; - if (!buf) return 0; - struct rb_item* item = bpf_ringbuf_reserve(&mimic_rb, sizeof(*item), 0); - if (!item) return -1; - item->type = type; - item->pktbuf = buf; - bpf_ringbuf_submit(item, 0); - return 0; -} +#ifndef _MIMIC_BPF_INLINE_ALL_FUNCS +#include "mimic-impl.h" // IWYU pragma: export +#endif char _license[] SEC("license") = "GPL"; diff --git a/bpf/mimic.h b/bpf/mimic.h index 642afd9..217561e 100644 --- a/bpf/mimic.h +++ b/bpf/mimic.h @@ -166,7 +166,7 @@ int use_pktbuf(enum rb_item_type type, uintptr_t buf); struct rb_item* item = bpf_ringbuf_reserve(&mimic_rb, sizeof(*item), 0); \ if (item) { \ item->log_event.level = (_l); \ - item->log_event.type = LOG_MSG; \ + item->log_event.type = LOG_MSG; \ _log_f(item->log_event.info.msg, sizeof(item->log_event.info.msg), _fmt, __VA_ARGS__); \ bpf_ringbuf_submit(item, 0); \ } \ @@ -196,4 +196,11 @@ static inline bool ipv6_is_ext(__u8 nexthdr) { } } +#ifdef _MIMIC_BPF_INLINE_ALL_FUNCS +#define _mimic_maybe_static_inline static __always_inline +#include "mimic-impl.h" // IWYU pragma: export +#else +#define _mimic_maybe_static_inline +#endif + #endif // _MIMIC_BPF_MIMIC_H