diff --git a/internal/include/arguments.h b/internal/include/arguments.h index 3fb11a5ac..8c9f67920 100644 --- a/internal/include/arguments.h +++ b/internal/include/arguments.h @@ -9,6 +9,38 @@ #include "bpf_helpers.h" #include +#if defined(bpf_target_x86) + +#if defined(__KERNEL__) || defined(__VMLINUX_H__) + +#define GO_PARAM1(x) ((x)->ax) +#define GO_PARAM2(x) ((x)->bx) +#define GO_PARAM3(x) ((x)->cx) +#define GO_PARAM4(x) ((x)->di) +#define GO_PARAM5(x) ((x)->si) +#define GO_PARAM6(x) ((x)->r8) +#define GO_PARAM7(x) ((x)->r9) +#define GO_PARAM8(x) ((x)->r10) +#define GO_PARAM9(x) ((x)->r11) +#define GOROUTINE(x) ((x)->r14) + +#endif + +#elif defined(bpf_target_arm64) + +#define GO_PARAM1(x) (__PT_REGS_CAST(x)->__PT_PARM1_REG) +#define GO_PARAM2(x) (__PT_REGS_CAST(x)->__PT_PARM2_REG) +#define GO_PARAM3(x) (__PT_REGS_CAST(x)->__PT_PARM3_REG) +#define GO_PARAM4(x) (__PT_REGS_CAST(x)->__PT_PARM4_REG) +#define GO_PARAM5(x) (__PT_REGS_CAST(x)->__PT_PARM5_REG) +#define GO_PARAM6(x) (__PT_REGS_CAST(x)->__PT_PARM6_REG) +#define GO_PARAM7(x) (__PT_REGS_CAST(x)->__PT_PARM7_REG) +#define GO_PARAM8(x) (__PT_REGS_CAST(x)->__PT_PARM8_REG) +#define GO_PARAM9(x) (__PT_REGS_CAST(x)->regs[8]) +#define GOROUTINE(x) (__PT_REGS_CAST(x)->regs[28]) + +#endif + // Injected in init volatile const bool is_registers_abi; diff --git a/internal/include/libbpf/bpf.h b/internal/include/libbpf/bpf.h deleted file mode 100644 index 6fffb3cdf..000000000 --- a/internal/include/libbpf/bpf.h +++ /dev/null @@ -1,293 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -/* - * common eBPF ELF operations. - * - * Copyright (C) 2013-2015 Alexei Starovoitov - * Copyright (C) 2015 Wang Nan - * Copyright (C) 2015 Huawei Inc. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU Lesser General Public - * License as published by the Free Software Foundation; - * version 2.1 of the License (not later!) - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Lesser General Public License for more details. - * - * You should have received a copy of the GNU Lesser General Public - * License along with this program; if not, see - */ -#ifndef __LIBBPF_BPF_H -#define __LIBBPF_BPF_H - -#include -#include -#include -#include - -#include "libbpf_common.h" - -#ifdef __cplusplus -extern "C" { -#endif - -struct bpf_create_map_attr { - const char *name; - enum bpf_map_type map_type; - __u32 map_flags; - __u32 key_size; - __u32 value_size; - __u32 max_entries; - __u32 numa_node; - __u32 btf_fd; - __u32 btf_key_type_id; - __u32 btf_value_type_id; - __u32 map_ifindex; - union { - __u32 inner_map_fd; - __u32 btf_vmlinux_value_type_id; - }; -}; - -LIBBPF_API int -bpf_create_map_xattr(const struct bpf_create_map_attr *create_attr); -LIBBPF_API int bpf_create_map_node(enum bpf_map_type map_type, const char *name, - int key_size, int value_size, - int max_entries, __u32 map_flags, int node); -LIBBPF_API int bpf_create_map_name(enum bpf_map_type map_type, const char *name, - int key_size, int value_size, - int max_entries, __u32 map_flags); -LIBBPF_API int bpf_create_map(enum bpf_map_type map_type, int key_size, - int value_size, int max_entries, __u32 map_flags); -LIBBPF_API int bpf_create_map_in_map_node(enum bpf_map_type map_type, - const char *name, int key_size, - int inner_map_fd, int max_entries, - __u32 map_flags, int node); -LIBBPF_API int bpf_create_map_in_map(enum bpf_map_type map_type, - const char *name, int key_size, - int inner_map_fd, int max_entries, - __u32 map_flags); - -struct bpf_load_program_attr { - enum bpf_prog_type prog_type; - enum bpf_attach_type expected_attach_type; - const char *name; - const struct bpf_insn *insns; - size_t insns_cnt; - const char *license; - union { - __u32 kern_version; - __u32 attach_prog_fd; - }; - union { - __u32 prog_ifindex; - __u32 attach_btf_id; - }; - __u32 prog_btf_fd; - __u32 func_info_rec_size; - const void *func_info; - __u32 func_info_cnt; - __u32 line_info_rec_size; - const void *line_info; - __u32 line_info_cnt; - __u32 log_level; - __u32 prog_flags; -}; - -/* Flags to direct loading requirements */ -#define MAPS_RELAX_COMPAT 0x01 - -/* Recommend log buffer size */ -#define BPF_LOG_BUF_SIZE (UINT32_MAX >> 8) /* verifier maximum in kernels <= 5.1 */ -LIBBPF_API int -bpf_load_program_xattr(const struct bpf_load_program_attr *load_attr, - char *log_buf, size_t log_buf_sz); -LIBBPF_API int bpf_load_program(enum bpf_prog_type type, - const struct bpf_insn *insns, size_t insns_cnt, - const char *license, __u32 kern_version, - char *log_buf, size_t log_buf_sz); -LIBBPF_API int bpf_verify_program(enum bpf_prog_type type, - const struct bpf_insn *insns, - size_t insns_cnt, __u32 prog_flags, - const char *license, __u32 kern_version, - char *log_buf, size_t log_buf_sz, - int log_level); - -LIBBPF_API int bpf_map_update_elem(int fd, const void *key, const void *value, - __u64 flags); - -LIBBPF_API int bpf_map_lookup_elem(int fd, const void *key, void *value); -LIBBPF_API int bpf_map_lookup_elem_flags(int fd, const void *key, void *value, - __u64 flags); -LIBBPF_API int bpf_map_lookup_and_delete_elem(int fd, const void *key, - void *value); -LIBBPF_API int bpf_map_lookup_and_delete_elem_flags(int fd, const void *key, - void *value, __u64 flags); -LIBBPF_API int bpf_map_delete_elem(int fd, const void *key); -LIBBPF_API int bpf_map_get_next_key(int fd, const void *key, void *next_key); -LIBBPF_API int bpf_map_freeze(int fd); - -struct bpf_map_batch_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u64 elem_flags; - __u64 flags; -}; -#define bpf_map_batch_opts__last_field flags - -LIBBPF_API int bpf_map_delete_batch(int fd, void *keys, - __u32 *count, - const struct bpf_map_batch_opts *opts); -LIBBPF_API int bpf_map_lookup_batch(int fd, void *in_batch, void *out_batch, - void *keys, void *values, __u32 *count, - const struct bpf_map_batch_opts *opts); -LIBBPF_API int bpf_map_lookup_and_delete_batch(int fd, void *in_batch, - void *out_batch, void *keys, - void *values, __u32 *count, - const struct bpf_map_batch_opts *opts); -LIBBPF_API int bpf_map_update_batch(int fd, void *keys, void *values, - __u32 *count, - const struct bpf_map_batch_opts *opts); - -LIBBPF_API int bpf_obj_pin(int fd, const char *pathname); -LIBBPF_API int bpf_obj_get(const char *pathname); - -struct bpf_prog_attach_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - unsigned int flags; - int replace_prog_fd; -}; -#define bpf_prog_attach_opts__last_field replace_prog_fd - -LIBBPF_API int bpf_prog_attach(int prog_fd, int attachable_fd, - enum bpf_attach_type type, unsigned int flags); -LIBBPF_API int bpf_prog_attach_xattr(int prog_fd, int attachable_fd, - enum bpf_attach_type type, - const struct bpf_prog_attach_opts *opts); -LIBBPF_API int bpf_prog_detach(int attachable_fd, enum bpf_attach_type type); -LIBBPF_API int bpf_prog_detach2(int prog_fd, int attachable_fd, - enum bpf_attach_type type); - -union bpf_iter_link_info; /* defined in up-to-date linux/bpf.h */ -struct bpf_link_create_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u32 flags; - union bpf_iter_link_info *iter_info; - __u32 iter_info_len; - __u32 target_btf_id; - union { - struct { - __u64 bpf_cookie; - } perf_event; - }; - size_t :0; -}; -#define bpf_link_create_opts__last_field perf_event - -LIBBPF_API int bpf_link_create(int prog_fd, int target_fd, - enum bpf_attach_type attach_type, - const struct bpf_link_create_opts *opts); - -LIBBPF_API int bpf_link_detach(int link_fd); - -struct bpf_link_update_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u32 flags; /* extra flags */ - __u32 old_prog_fd; /* expected old program FD */ -}; -#define bpf_link_update_opts__last_field old_prog_fd - -LIBBPF_API int bpf_link_update(int link_fd, int new_prog_fd, - const struct bpf_link_update_opts *opts); - -LIBBPF_API int bpf_iter_create(int link_fd); - -struct bpf_prog_test_run_attr { - int prog_fd; - int repeat; - const void *data_in; - __u32 data_size_in; - void *data_out; /* optional */ - __u32 data_size_out; /* in: max length of data_out - * out: length of data_out */ - __u32 retval; /* out: return code of the BPF program */ - __u32 duration; /* out: average per repetition in ns */ - const void *ctx_in; /* optional */ - __u32 ctx_size_in; - void *ctx_out; /* optional */ - __u32 ctx_size_out; /* in: max length of ctx_out - * out: length of cxt_out */ -}; - -LIBBPF_API int bpf_prog_test_run_xattr(struct bpf_prog_test_run_attr *test_attr); - -/* - * bpf_prog_test_run does not check that data_out is large enough. Consider - * using bpf_prog_test_run_xattr instead. - */ -LIBBPF_API int bpf_prog_test_run(int prog_fd, int repeat, void *data, - __u32 size, void *data_out, __u32 *size_out, - __u32 *retval, __u32 *duration); -LIBBPF_API int bpf_prog_get_next_id(__u32 start_id, __u32 *next_id); -LIBBPF_API int bpf_map_get_next_id(__u32 start_id, __u32 *next_id); -LIBBPF_API int bpf_btf_get_next_id(__u32 start_id, __u32 *next_id); -LIBBPF_API int bpf_link_get_next_id(__u32 start_id, __u32 *next_id); -LIBBPF_API int bpf_prog_get_fd_by_id(__u32 id); -LIBBPF_API int bpf_map_get_fd_by_id(__u32 id); -LIBBPF_API int bpf_btf_get_fd_by_id(__u32 id); -LIBBPF_API int bpf_link_get_fd_by_id(__u32 id); -LIBBPF_API int bpf_obj_get_info_by_fd(int bpf_fd, void *info, __u32 *info_len); -LIBBPF_API int bpf_prog_query(int target_fd, enum bpf_attach_type type, - __u32 query_flags, __u32 *attach_flags, - __u32 *prog_ids, __u32 *prog_cnt); -LIBBPF_API int bpf_raw_tracepoint_open(const char *name, int prog_fd); -LIBBPF_API int bpf_load_btf(const void *btf, __u32 btf_size, char *log_buf, - __u32 log_buf_size, bool do_log); -LIBBPF_API int bpf_task_fd_query(int pid, int fd, __u32 flags, char *buf, - __u32 *buf_len, __u32 *prog_id, __u32 *fd_type, - __u64 *probe_offset, __u64 *probe_addr); - -enum bpf_stats_type; /* defined in up-to-date linux/bpf.h */ -LIBBPF_API int bpf_enable_stats(enum bpf_stats_type type); - -struct bpf_prog_bind_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - __u32 flags; -}; -#define bpf_prog_bind_opts__last_field flags - -LIBBPF_API int bpf_prog_bind_map(int prog_fd, int map_fd, - const struct bpf_prog_bind_opts *opts); - -struct bpf_test_run_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - const void *data_in; /* optional */ - void *data_out; /* optional */ - __u32 data_size_in; - __u32 data_size_out; /* in: max length of data_out - * out: length of data_out - */ - const void *ctx_in; /* optional */ - void *ctx_out; /* optional */ - __u32 ctx_size_in; - __u32 ctx_size_out; /* in: max length of ctx_out - * out: length of cxt_out - */ - __u32 retval; /* out: return code of the BPF program */ - int repeat; - __u32 duration; /* out: average per repetition in ns */ - __u32 flags; - __u32 cpu; -}; -#define bpf_test_run_opts__last_field cpu - -LIBBPF_API int bpf_prog_test_run_opts(int prog_fd, - struct bpf_test_run_opts *opts); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* __LIBBPF_BPF_H */ diff --git a/internal/include/libbpf/bpf_core_read.h b/internal/include/libbpf/bpf_core_read.h deleted file mode 100644 index 09ebe3db5..000000000 --- a/internal/include/libbpf/bpf_core_read.h +++ /dev/null @@ -1,444 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef __BPF_CORE_READ_H__ -#define __BPF_CORE_READ_H__ - -/* - * enum bpf_field_info_kind is passed as a second argument into - * __builtin_preserve_field_info() built-in to get a specific aspect of - * a field, captured as a first argument. __builtin_preserve_field_info(field, - * info_kind) returns __u32 integer and produces BTF field relocation, which - * is understood and processed by libbpf during BPF object loading. See - * selftests/bpf for examples. - */ -enum bpf_field_info_kind { - BPF_FIELD_BYTE_OFFSET = 0, /* field byte offset */ - BPF_FIELD_BYTE_SIZE = 1, - BPF_FIELD_EXISTS = 2, /* field existence in target kernel */ - BPF_FIELD_SIGNED = 3, - BPF_FIELD_LSHIFT_U64 = 4, - BPF_FIELD_RSHIFT_U64 = 5, -}; - -/* second argument to __builtin_btf_type_id() built-in */ -enum bpf_type_id_kind { - BPF_TYPE_ID_LOCAL = 0, /* BTF type ID in local program */ - BPF_TYPE_ID_TARGET = 1, /* BTF type ID in target kernel */ -}; - -/* second argument to __builtin_preserve_type_info() built-in */ -enum bpf_type_info_kind { - BPF_TYPE_EXISTS = 0, /* type existence in target kernel */ - BPF_TYPE_SIZE = 1, /* type size in target kernel */ -}; - -/* second argument to __builtin_preserve_enum_value() built-in */ -enum bpf_enum_value_kind { - BPF_ENUMVAL_EXISTS = 0, /* enum value existence in kernel */ - BPF_ENUMVAL_VALUE = 1, /* enum value value relocation */ -}; - -#define __CORE_RELO(src, field, info) \ - __builtin_preserve_field_info((src)->field, BPF_FIELD_##info) - -#if __BYTE_ORDER == __LITTLE_ENDIAN -#define __CORE_BITFIELD_PROBE_READ(dst, src, fld) \ - bpf_probe_read_kernel( \ - (void *)dst, \ - __CORE_RELO(src, fld, BYTE_SIZE), \ - (const void *)src + __CORE_RELO(src, fld, BYTE_OFFSET)) -#else -/* semantics of LSHIFT_64 assumes loading values into low-ordered bytes, so - * for big-endian we need to adjust destination pointer accordingly, based on - * field byte size - */ -#define __CORE_BITFIELD_PROBE_READ(dst, src, fld) \ - bpf_probe_read_kernel( \ - (void *)dst + (8 - __CORE_RELO(src, fld, BYTE_SIZE)), \ - __CORE_RELO(src, fld, BYTE_SIZE), \ - (const void *)src + __CORE_RELO(src, fld, BYTE_OFFSET)) -#endif - -/* - * Extract bitfield, identified by s->field, and return its value as u64. - * All this is done in relocatable manner, so bitfield changes such as - * signedness, bit size, offset changes, this will be handled automatically. - * This version of macro is using bpf_probe_read_kernel() to read underlying - * integer storage. Macro functions as an expression and its return type is - * bpf_probe_read_kernel()'s return value: 0, on success, <0 on error. - */ -#define BPF_CORE_READ_BITFIELD_PROBED(s, field) ({ \ - unsigned long long val = 0; \ - \ - __CORE_BITFIELD_PROBE_READ(&val, s, field); \ - val <<= __CORE_RELO(s, field, LSHIFT_U64); \ - if (__CORE_RELO(s, field, SIGNED)) \ - val = ((long long)val) >> __CORE_RELO(s, field, RSHIFT_U64); \ - else \ - val = val >> __CORE_RELO(s, field, RSHIFT_U64); \ - val; \ -}) - -/* - * Extract bitfield, identified by s->field, and return its value as u64. - * This version of macro is using direct memory reads and should be used from - * BPF program types that support such functionality (e.g., typed raw - * tracepoints). - */ -#define BPF_CORE_READ_BITFIELD(s, field) ({ \ - const void *p = (const void *)s + __CORE_RELO(s, field, BYTE_OFFSET); \ - unsigned long long val; \ - \ - /* This is a so-called barrier_var() operation that makes specified \ - * variable "a black box" for optimizing compiler. \ - * It forces compiler to perform BYTE_OFFSET relocation on p and use \ - * its calculated value in the switch below, instead of applying \ - * the same relocation 4 times for each individual memory load. \ - */ \ - asm volatile("" : "=r"(p) : "0"(p)); \ - \ - switch (__CORE_RELO(s, field, BYTE_SIZE)) { \ - case 1: val = *(const unsigned char *)p; break; \ - case 2: val = *(const unsigned short *)p; break; \ - case 4: val = *(const unsigned int *)p; break; \ - case 8: val = *(const unsigned long long *)p; break; \ - } \ - val <<= __CORE_RELO(s, field, LSHIFT_U64); \ - if (__CORE_RELO(s, field, SIGNED)) \ - val = ((long long)val) >> __CORE_RELO(s, field, RSHIFT_U64); \ - else \ - val = val >> __CORE_RELO(s, field, RSHIFT_U64); \ - val; \ -}) - -/* - * Convenience macro to check that field actually exists in target kernel's. - * Returns: - * 1, if matching field is present in target kernel; - * 0, if no matching field found. - */ -#define bpf_core_field_exists(field) \ - __builtin_preserve_field_info(field, BPF_FIELD_EXISTS) - -/* - * Convenience macro to get the byte size of a field. Works for integers, - * struct/unions, pointers, arrays, and enums. - */ -#define bpf_core_field_size(field) \ - __builtin_preserve_field_info(field, BPF_FIELD_BYTE_SIZE) - -/* - * Convenience macro to get BTF type ID of a specified type, using a local BTF - * information. Return 32-bit unsigned integer with type ID from program's own - * BTF. Always succeeds. - */ -#define bpf_core_type_id_local(type) \ - __builtin_btf_type_id(*(typeof(type) *)0, BPF_TYPE_ID_LOCAL) - -/* - * Convenience macro to get BTF type ID of a target kernel's type that matches - * specified local type. - * Returns: - * - valid 32-bit unsigned type ID in kernel BTF; - * - 0, if no matching type was found in a target kernel BTF. - */ -#define bpf_core_type_id_kernel(type) \ - __builtin_btf_type_id(*(typeof(type) *)0, BPF_TYPE_ID_TARGET) - -/* - * Convenience macro to check that provided named type - * (struct/union/enum/typedef) exists in a target kernel. - * Returns: - * 1, if such type is present in target kernel's BTF; - * 0, if no matching type is found. - */ -#define bpf_core_type_exists(type) \ - __builtin_preserve_type_info(*(typeof(type) *)0, BPF_TYPE_EXISTS) - -/* - * Convenience macro to get the byte size of a provided named type - * (struct/union/enum/typedef) in a target kernel. - * Returns: - * >= 0 size (in bytes), if type is present in target kernel's BTF; - * 0, if no matching type is found. - */ -#define bpf_core_type_size(type) \ - __builtin_preserve_type_info(*(typeof(type) *)0, BPF_TYPE_SIZE) - -/* - * Convenience macro to check that provided enumerator value is defined in - * a target kernel. - * Returns: - * 1, if specified enum type and its enumerator value are present in target - * kernel's BTF; - * 0, if no matching enum and/or enum value within that enum is found. - */ -#define bpf_core_enum_value_exists(enum_type, enum_value) \ - __builtin_preserve_enum_value(*(typeof(enum_type) *)enum_value, BPF_ENUMVAL_EXISTS) - -/* - * Convenience macro to get the integer value of an enumerator value in - * a target kernel. - * Returns: - * 64-bit value, if specified enum type and its enumerator value are - * present in target kernel's BTF; - * 0, if no matching enum and/or enum value within that enum is found. - */ -#define bpf_core_enum_value(enum_type, enum_value) \ - __builtin_preserve_enum_value(*(typeof(enum_type) *)enum_value, BPF_ENUMVAL_VALUE) - -/* - * bpf_core_read() abstracts away bpf_probe_read_kernel() call and captures - * offset relocation for source address using __builtin_preserve_access_index() - * built-in, provided by Clang. - * - * __builtin_preserve_access_index() takes as an argument an expression of - * taking an address of a field within struct/union. It makes compiler emit - * a relocation, which records BTF type ID describing root struct/union and an - * accessor string which describes exact embedded field that was used to take - * an address. See detailed description of this relocation format and - * semantics in comments to struct bpf_field_reloc in libbpf_internal.h. - * - * This relocation allows libbpf to adjust BPF instruction to use correct - * actual field offset, based on target kernel BTF type that matches original - * (local) BTF, used to record relocation. - */ -#define bpf_core_read(dst, sz, src) \ - bpf_probe_read_kernel(dst, sz, (const void *)__builtin_preserve_access_index(src)) - -/* NOTE: see comments for BPF_CORE_READ_USER() about the proper types use. */ -#define bpf_core_read_user(dst, sz, src) \ - bpf_probe_read_user(dst, sz, (const void *)__builtin_preserve_access_index(src)) -/* - * bpf_core_read_str() is a thin wrapper around bpf_probe_read_str() - * additionally emitting BPF CO-RE field relocation for specified source - * argument. - */ -#define bpf_core_read_str(dst, sz, src) \ - bpf_probe_read_kernel_str(dst, sz, (const void *)__builtin_preserve_access_index(src)) - -/* NOTE: see comments for BPF_CORE_READ_USER() about the proper types use. */ -#define bpf_core_read_user_str(dst, sz, src) \ - bpf_probe_read_user_str(dst, sz, (const void *)__builtin_preserve_access_index(src)) - -#define ___concat(a, b) a ## b -#define ___apply(fn, n) ___concat(fn, n) -#define ___nth(_1, _2, _3, _4, _5, _6, _7, _8, _9, _10, __11, N, ...) N - -/* - * return number of provided arguments; used for switch-based variadic macro - * definitions (see ___last, ___arrow, etc below) - */ -#define ___narg(...) ___nth(_, ##__VA_ARGS__, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) -/* - * return 0 if no arguments are passed, N - otherwise; used for - * recursively-defined macros to specify termination (0) case, and generic - * (N) case (e.g., ___read_ptrs, ___core_read) - */ -#define ___empty(...) ___nth(_, ##__VA_ARGS__, N, N, N, N, N, N, N, N, N, N, 0) - -#define ___last1(x) x -#define ___last2(a, x) x -#define ___last3(a, b, x) x -#define ___last4(a, b, c, x) x -#define ___last5(a, b, c, d, x) x -#define ___last6(a, b, c, d, e, x) x -#define ___last7(a, b, c, d, e, f, x) x -#define ___last8(a, b, c, d, e, f, g, x) x -#define ___last9(a, b, c, d, e, f, g, h, x) x -#define ___last10(a, b, c, d, e, f, g, h, i, x) x -#define ___last(...) ___apply(___last, ___narg(__VA_ARGS__))(__VA_ARGS__) - -#define ___nolast2(a, _) a -#define ___nolast3(a, b, _) a, b -#define ___nolast4(a, b, c, _) a, b, c -#define ___nolast5(a, b, c, d, _) a, b, c, d -#define ___nolast6(a, b, c, d, e, _) a, b, c, d, e -#define ___nolast7(a, b, c, d, e, f, _) a, b, c, d, e, f -#define ___nolast8(a, b, c, d, e, f, g, _) a, b, c, d, e, f, g -#define ___nolast9(a, b, c, d, e, f, g, h, _) a, b, c, d, e, f, g, h -#define ___nolast10(a, b, c, d, e, f, g, h, i, _) a, b, c, d, e, f, g, h, i -#define ___nolast(...) ___apply(___nolast, ___narg(__VA_ARGS__))(__VA_ARGS__) - -#define ___arrow1(a) a -#define ___arrow2(a, b) a->b -#define ___arrow3(a, b, c) a->b->c -#define ___arrow4(a, b, c, d) a->b->c->d -#define ___arrow5(a, b, c, d, e) a->b->c->d->e -#define ___arrow6(a, b, c, d, e, f) a->b->c->d->e->f -#define ___arrow7(a, b, c, d, e, f, g) a->b->c->d->e->f->g -#define ___arrow8(a, b, c, d, e, f, g, h) a->b->c->d->e->f->g->h -#define ___arrow9(a, b, c, d, e, f, g, h, i) a->b->c->d->e->f->g->h->i -#define ___arrow10(a, b, c, d, e, f, g, h, i, j) a->b->c->d->e->f->g->h->i->j -#define ___arrow(...) ___apply(___arrow, ___narg(__VA_ARGS__))(__VA_ARGS__) - -#define ___type(...) typeof(___arrow(__VA_ARGS__)) - -#define ___read(read_fn, dst, src_type, src, accessor) \ - read_fn((void *)(dst), sizeof(*(dst)), &((src_type)(src))->accessor) - -/* "recursively" read a sequence of inner pointers using local __t var */ -#define ___rd_first(fn, src, a) ___read(fn, &__t, ___type(src), src, a); -#define ___rd_last(fn, ...) \ - ___read(fn, &__t, ___type(___nolast(__VA_ARGS__)), __t, ___last(__VA_ARGS__)); -#define ___rd_p1(fn, ...) const void *__t; ___rd_first(fn, __VA_ARGS__) -#define ___rd_p2(fn, ...) ___rd_p1(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p3(fn, ...) ___rd_p2(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p4(fn, ...) ___rd_p3(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p5(fn, ...) ___rd_p4(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p6(fn, ...) ___rd_p5(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p7(fn, ...) ___rd_p6(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p8(fn, ...) ___rd_p7(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___rd_p9(fn, ...) ___rd_p8(fn, ___nolast(__VA_ARGS__)) ___rd_last(fn, __VA_ARGS__) -#define ___read_ptrs(fn, src, ...) \ - ___apply(___rd_p, ___narg(__VA_ARGS__))(fn, src, __VA_ARGS__) - -#define ___core_read0(fn, fn_ptr, dst, src, a) \ - ___read(fn, dst, ___type(src), src, a); -#define ___core_readN(fn, fn_ptr, dst, src, ...) \ - ___read_ptrs(fn_ptr, src, ___nolast(__VA_ARGS__)) \ - ___read(fn, dst, ___type(src, ___nolast(__VA_ARGS__)), __t, \ - ___last(__VA_ARGS__)); -#define ___core_read(fn, fn_ptr, dst, src, a, ...) \ - ___apply(___core_read, ___empty(__VA_ARGS__))(fn, fn_ptr, dst, \ - src, a, ##__VA_ARGS__) - -/* - * BPF_CORE_READ_INTO() is a more performance-conscious variant of - * BPF_CORE_READ(), in which final field is read into user-provided storage. - * See BPF_CORE_READ() below for more details on general usage. - */ -#define BPF_CORE_READ_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_core_read, bpf_core_read, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* - * Variant of BPF_CORE_READ_INTO() for reading from user-space memory. - * - * NOTE: see comments for BPF_CORE_READ_USER() about the proper types use. - */ -#define BPF_CORE_READ_USER_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_core_read_user, bpf_core_read_user, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* Non-CO-RE variant of BPF_CORE_READ_INTO() */ -#define BPF_PROBE_READ_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_probe_read, bpf_probe_read, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* Non-CO-RE variant of BPF_CORE_READ_USER_INTO(). - * - * As no CO-RE relocations are emitted, source types can be arbitrary and are - * not restricted to kernel types only. - */ -#define BPF_PROBE_READ_USER_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_probe_read_user, bpf_probe_read_user, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* - * BPF_CORE_READ_STR_INTO() does same "pointer chasing" as - * BPF_CORE_READ() for intermediate pointers, but then executes (and returns - * corresponding error code) bpf_core_read_str() for final string read. - */ -#define BPF_CORE_READ_STR_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_core_read_str, bpf_core_read, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* - * Variant of BPF_CORE_READ_STR_INTO() for reading from user-space memory. - * - * NOTE: see comments for BPF_CORE_READ_USER() about the proper types use. - */ -#define BPF_CORE_READ_USER_STR_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_core_read_user_str, bpf_core_read_user, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* Non-CO-RE variant of BPF_CORE_READ_STR_INTO() */ -#define BPF_PROBE_READ_STR_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_probe_read_str, bpf_probe_read, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* - * Non-CO-RE variant of BPF_CORE_READ_USER_STR_INTO(). - * - * As no CO-RE relocations are emitted, source types can be arbitrary and are - * not restricted to kernel types only. - */ -#define BPF_PROBE_READ_USER_STR_INTO(dst, src, a, ...) ({ \ - ___core_read(bpf_probe_read_user_str, bpf_probe_read_user, \ - dst, (src), a, ##__VA_ARGS__) \ -}) - -/* - * BPF_CORE_READ() is used to simplify BPF CO-RE relocatable read, especially - * when there are few pointer chasing steps. - * E.g., what in non-BPF world (or in BPF w/ BCC) would be something like: - * int x = s->a.b.c->d.e->f->g; - * can be succinctly achieved using BPF_CORE_READ as: - * int x = BPF_CORE_READ(s, a.b.c, d.e, f, g); - * - * BPF_CORE_READ will decompose above statement into 4 bpf_core_read (BPF - * CO-RE relocatable bpf_probe_read_kernel() wrapper) calls, logically - * equivalent to: - * 1. const void *__t = s->a.b.c; - * 2. __t = __t->d.e; - * 3. __t = __t->f; - * 4. return __t->g; - * - * Equivalence is logical, because there is a heavy type casting/preservation - * involved, as well as all the reads are happening through - * bpf_probe_read_kernel() calls using __builtin_preserve_access_index() to - * emit CO-RE relocations. - * - * N.B. Only up to 9 "field accessors" are supported, which should be more - * than enough for any practical purpose. - */ -#define BPF_CORE_READ(src, a, ...) ({ \ - ___type((src), a, ##__VA_ARGS__) __r; \ - BPF_CORE_READ_INTO(&__r, (src), a, ##__VA_ARGS__); \ - __r; \ -}) - -/* - * Variant of BPF_CORE_READ() for reading from user-space memory. - * - * NOTE: all the source types involved are still *kernel types* and need to - * exist in kernel (or kernel module) BTF, otherwise CO-RE relocation will - * fail. Custom user types are not relocatable with CO-RE. - * The typical situation in which BPF_CORE_READ_USER() might be used is to - * read kernel UAPI types from the user-space memory passed in as a syscall - * input argument. - */ -#define BPF_CORE_READ_USER(src, a, ...) ({ \ - ___type((src), a, ##__VA_ARGS__) __r; \ - BPF_CORE_READ_USER_INTO(&__r, (src), a, ##__VA_ARGS__); \ - __r; \ -}) - -/* Non-CO-RE variant of BPF_CORE_READ() */ -#define BPF_PROBE_READ(src, a, ...) ({ \ - ___type((src), a, ##__VA_ARGS__) __r; \ - BPF_PROBE_READ_INTO(&__r, (src), a, ##__VA_ARGS__); \ - __r; \ -}) - -/* - * Non-CO-RE variant of BPF_CORE_READ_USER(). - * - * As no CO-RE relocations are emitted, source types can be arbitrary and are - * not restricted to kernel types only. - */ -#define BPF_PROBE_READ_USER(src, a, ...) ({ \ - ___type((src), a, ##__VA_ARGS__) __r; \ - BPF_PROBE_READ_USER_INTO(&__r, (src), a, ##__VA_ARGS__); \ - __r; \ -}) - -#endif - diff --git a/internal/include/libbpf/bpf_endian.h b/internal/include/libbpf/bpf_endian.h deleted file mode 100644 index ec9db4fec..000000000 --- a/internal/include/libbpf/bpf_endian.h +++ /dev/null @@ -1,99 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -#ifndef __BPF_ENDIAN__ -#define __BPF_ENDIAN__ - -/* - * Isolate byte #n and put it into byte #m, for __u##b type. - * E.g., moving byte #6 (nnnnnnnn) into byte #1 (mmmmmmmm) for __u64: - * 1) xxxxxxxx nnnnnnnn xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx mmmmmmmm xxxxxxxx - * 2) nnnnnnnn xxxxxxxx xxxxxxxx xxxxxxxx xxxxxxxx mmmmmmmm xxxxxxxx 00000000 - * 3) 00000000 00000000 00000000 00000000 00000000 00000000 00000000 nnnnnnnn - * 4) 00000000 00000000 00000000 00000000 00000000 00000000 nnnnnnnn 00000000 - */ -#define ___bpf_mvb(x, b, n, m) ((__u##b)(x) << (b-(n+1)*8) >> (b-8) << (m*8)) - -#define ___bpf_swab16(x) ((__u16)( \ - ___bpf_mvb(x, 16, 0, 1) | \ - ___bpf_mvb(x, 16, 1, 0))) - -#define ___bpf_swab32(x) ((__u32)( \ - ___bpf_mvb(x, 32, 0, 3) | \ - ___bpf_mvb(x, 32, 1, 2) | \ - ___bpf_mvb(x, 32, 2, 1) | \ - ___bpf_mvb(x, 32, 3, 0))) - -#define ___bpf_swab64(x) ((__u64)( \ - ___bpf_mvb(x, 64, 0, 7) | \ - ___bpf_mvb(x, 64, 1, 6) | \ - ___bpf_mvb(x, 64, 2, 5) | \ - ___bpf_mvb(x, 64, 3, 4) | \ - ___bpf_mvb(x, 64, 4, 3) | \ - ___bpf_mvb(x, 64, 5, 2) | \ - ___bpf_mvb(x, 64, 6, 1) | \ - ___bpf_mvb(x, 64, 7, 0))) - -/* LLVM's BPF target selects the endianness of the CPU - * it compiles on, or the user specifies (bpfel/bpfeb), - * respectively. The used __BYTE_ORDER__ is defined by - * the compiler, we cannot rely on __BYTE_ORDER from - * libc headers, since it doesn't reflect the actual - * requested byte order. - * - * Note, LLVM's BPF target has different __builtin_bswapX() - * semantics. It does map to BPF_ALU | BPF_END | BPF_TO_BE - * in bpfel and bpfeb case, which means below, that we map - * to cpu_to_be16(). We could use it unconditionally in BPF - * case, but better not rely on it, so that this header here - * can be used from application and BPF program side, which - * use different targets. - */ -#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ -# define __bpf_ntohs(x) __builtin_bswap16(x) -# define __bpf_htons(x) __builtin_bswap16(x) -# define __bpf_constant_ntohs(x) ___bpf_swab16(x) -# define __bpf_constant_htons(x) ___bpf_swab16(x) -# define __bpf_ntohl(x) __builtin_bswap32(x) -# define __bpf_htonl(x) __builtin_bswap32(x) -# define __bpf_constant_ntohl(x) ___bpf_swab32(x) -# define __bpf_constant_htonl(x) ___bpf_swab32(x) -# define __bpf_be64_to_cpu(x) __builtin_bswap64(x) -# define __bpf_cpu_to_be64(x) __builtin_bswap64(x) -# define __bpf_constant_be64_to_cpu(x) ___bpf_swab64(x) -# define __bpf_constant_cpu_to_be64(x) ___bpf_swab64(x) -#elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ -# define __bpf_ntohs(x) (x) -# define __bpf_htons(x) (x) -# define __bpf_constant_ntohs(x) (x) -# define __bpf_constant_htons(x) (x) -# define __bpf_ntohl(x) (x) -# define __bpf_htonl(x) (x) -# define __bpf_constant_ntohl(x) (x) -# define __bpf_constant_htonl(x) (x) -# define __bpf_be64_to_cpu(x) (x) -# define __bpf_cpu_to_be64(x) (x) -# define __bpf_constant_be64_to_cpu(x) (x) -# define __bpf_constant_cpu_to_be64(x) (x) -#else -# error "Fix your compiler's __BYTE_ORDER__?!" -#endif - -#define bpf_htons(x) \ - (__builtin_constant_p(x) ? \ - __bpf_constant_htons(x) : __bpf_htons(x)) -#define bpf_ntohs(x) \ - (__builtin_constant_p(x) ? \ - __bpf_constant_ntohs(x) : __bpf_ntohs(x)) -#define bpf_htonl(x) \ - (__builtin_constant_p(x) ? \ - __bpf_constant_htonl(x) : __bpf_htonl(x)) -#define bpf_ntohl(x) \ - (__builtin_constant_p(x) ? \ - __bpf_constant_ntohl(x) : __bpf_ntohl(x)) -#define bpf_cpu_to_be64(x) \ - (__builtin_constant_p(x) ? \ - __bpf_constant_cpu_to_be64(x) : __bpf_cpu_to_be64(x)) -#define bpf_be64_to_cpu(x) \ - (__builtin_constant_p(x) ? \ - __bpf_constant_be64_to_cpu(x) : __bpf_be64_to_cpu(x)) - -#endif /* __BPF_ENDIAN__ */ diff --git a/internal/include/libbpf/bpf_helper_defs.h b/internal/include/libbpf/bpf_helper_defs.h index 00929faad..6f13d1f1d 100644 --- a/internal/include/libbpf/bpf_helper_defs.h +++ b/internal/include/libbpf/bpf_helper_defs.h @@ -27,7 +27,9 @@ struct tcp_sock; struct tcp_timewait_sock; struct tcp_request_sock; struct udp6_sock; +struct unix_sock; struct task_struct; +struct cgroup; struct __sk_buff; struct sk_msg_md; struct xdp_md; @@ -37,6 +39,10 @@ struct inode; struct socket; struct file; struct bpf_timer; +struct mptcp_sock; +struct bpf_dynptr; +struct iphdr; +struct ipv6hdr; /* * bpf_map_lookup_elem @@ -47,7 +53,7 @@ struct bpf_timer; * Map value associated to *key*, or **NULL** if no entry was * found. */ -static void *(*bpf_map_lookup_elem)(void *map, const void *key) = (void *) 1; +static void *(* const bpf_map_lookup_elem)(void *map, const void *key) = (void *) 1; /* * bpf_map_update_elem @@ -69,7 +75,7 @@ static void *(*bpf_map_lookup_elem)(void *map, const void *key) = (void *) 1; * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_map_update_elem)(void *map, const void *key, const void *value, __u64 flags) = (void *) 2; +static long (* const bpf_map_update_elem)(void *map, const void *key, const void *value, __u64 flags) = (void *) 2; /* * bpf_map_delete_elem @@ -79,7 +85,7 @@ static long (*bpf_map_update_elem)(void *map, const void *key, const void *value * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_map_delete_elem)(void *map, const void *key) = (void *) 3; +static long (* const bpf_map_delete_elem)(void *map, const void *key) = (void *) 3; /* * bpf_probe_read @@ -93,7 +99,7 @@ static long (*bpf_map_delete_elem)(void *map, const void *key) = (void *) 3; * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_probe_read)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 4; +static long (* const bpf_probe_read)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 4; /* * bpf_ktime_get_ns @@ -105,24 +111,24 @@ static long (*bpf_probe_read)(void *dst, __u32 size, const void *unsafe_ptr) = ( * Returns * Current *ktime*. */ -static __u64 (*bpf_ktime_get_ns)(void) = (void *) 5; +static __u64 (* const bpf_ktime_get_ns)(void) = (void *) 5; /* * bpf_trace_printk * * This helper is a "printk()-like" facility for debugging. It * prints a message defined by format *fmt* (of size *fmt_size*) - * to file *\/sys/kernel/debug/tracing/trace* from DebugFS, if + * to file *\/sys/kernel/tracing/trace* from TraceFS, if * available. It can take up to three additional **u64** * arguments (as an eBPF helpers, the total number of arguments is * limited to five). * * Each time the helper is called, it appends a line to the trace. - * Lines are discarded while *\/sys/kernel/debug/tracing/trace* is - * open, use *\/sys/kernel/debug/tracing/trace_pipe* to avoid this. + * Lines are discarded while *\/sys/kernel/tracing/trace* is + * open, use *\/sys/kernel/tracing/trace_pipe* to avoid this. * The format of the trace is customizable, and the exact output * one will get depends on the options set in - * *\/sys/kernel/debug/tracing/trace_options* (see also the + * *\/sys/kernel/tracing/trace_options* (see also the * *README* file under the same directory). However, it usually * defaults to something like: * @@ -168,7 +174,7 @@ static __u64 (*bpf_ktime_get_ns)(void) = (void *) 5; * The number of bytes written to the buffer, or a negative error * in case of failure. */ -static long (*bpf_trace_printk)(const char *fmt, __u32 fmt_size, ...) = (void *) 6; +static long (* const bpf_trace_printk)(const char *fmt, __u32 fmt_size, ...) = (void *) 6; /* * bpf_get_prandom_u32 @@ -184,20 +190,20 @@ static long (*bpf_trace_printk)(const char *fmt, __u32 fmt_size, ...) = (void *) * Returns * A random 32-bit unsigned value. */ -static __u32 (*bpf_get_prandom_u32)(void) = (void *) 7; +static __u32 (* const bpf_get_prandom_u32)(void) = (void *) 7; /* * bpf_get_smp_processor_id * * Get the SMP (symmetric multiprocessing) processor id. Note that - * all programs run with preemption disabled, which means that the + * all programs run with migration disabled, which means that the * SMP processor id is stable during all the execution of the * program. * * Returns * The SMP id of the processor running the program. */ -static __u32 (*bpf_get_smp_processor_id)(void) = (void *) 8; +static __u32 (* const bpf_get_smp_processor_id)(void) = (void *) 8; /* * bpf_skb_store_bytes @@ -218,7 +224,7 @@ static __u32 (*bpf_get_smp_processor_id)(void) = (void *) 8; * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_store_bytes)(struct __sk_buff *skb, __u32 offset, const void *from, __u32 len, __u64 flags) = (void *) 9; +static long (* const bpf_skb_store_bytes)(struct __sk_buff *skb, __u32 offset, const void *from, __u32 len, __u64 flags) = (void *) 9; /* * bpf_l3_csum_replace @@ -247,7 +253,7 @@ static long (*bpf_skb_store_bytes)(struct __sk_buff *skb, __u32 offset, const vo * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_l3_csum_replace)(struct __sk_buff *skb, __u32 offset, __u64 from, __u64 to, __u64 size) = (void *) 10; +static long (* const bpf_l3_csum_replace)(struct __sk_buff *skb, __u32 offset, __u64 from, __u64 to, __u64 size) = (void *) 10; /* * bpf_l4_csum_replace @@ -283,7 +289,7 @@ static long (*bpf_l3_csum_replace)(struct __sk_buff *skb, __u32 offset, __u64 fr * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_l4_csum_replace)(struct __sk_buff *skb, __u32 offset, __u64 from, __u64 to, __u64 flags) = (void *) 11; +static long (* const bpf_l4_csum_replace)(struct __sk_buff *skb, __u32 offset, __u64 from, __u64 to, __u64 flags) = (void *) 11; /* * bpf_tail_call @@ -313,12 +319,12 @@ static long (*bpf_l4_csum_replace)(struct __sk_buff *skb, __u32 offset, __u64 fr * if the maximum number of tail calls has been reached for this * chain of programs. This limit is defined in the kernel by the * macro **MAX_TAIL_CALL_CNT** (not accessible to user space), - * which is currently set to 32. + * which is currently set to 33. * * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_tail_call)(void *ctx, void *prog_array_map, __u32 index) = (void *) 12; +static long (* const bpf_tail_call)(void *ctx, void *prog_array_map, __u32 index) = (void *) 12; /* * bpf_clone_redirect @@ -344,13 +350,16 @@ static long (*bpf_tail_call)(void *ctx, void *prog_array_map, __u32 index) = (vo * direct packet access. * * Returns - * 0 on success, or a negative error in case of failure. + * 0 on success, or a negative error in case of failure. Positive + * error indicates a potential drop or congestion in the target + * device. The particular positive error codes are not defined. */ -static long (*bpf_clone_redirect)(struct __sk_buff *skb, __u32 ifindex, __u64 flags) = (void *) 13; +static long (* const bpf_clone_redirect)(struct __sk_buff *skb, __u32 ifindex, __u64 flags) = (void *) 13; /* * bpf_get_current_pid_tgid * + * Get the current pid and tgid. * * Returns * A 64-bit integer containing the current tgid and pid, and @@ -358,17 +367,18 @@ static long (*bpf_clone_redirect)(struct __sk_buff *skb, __u32 ifindex, __u64 fl * *current_task*\ **->tgid << 32 \|** * *current_task*\ **->pid**. */ -static __u64 (*bpf_get_current_pid_tgid)(void) = (void *) 14; +static __u64 (* const bpf_get_current_pid_tgid)(void) = (void *) 14; /* * bpf_get_current_uid_gid * + * Get the current uid and gid. * * Returns * A 64-bit integer containing the current GID and UID, and * created as such: *current_gid* **<< 32 \|** *current_uid*. */ -static __u64 (*bpf_get_current_uid_gid)(void) = (void *) 15; +static __u64 (* const bpf_get_current_uid_gid)(void) = (void *) 15; /* * bpf_get_current_comm @@ -383,7 +393,7 @@ static __u64 (*bpf_get_current_uid_gid)(void) = (void *) 15; * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_get_current_comm)(void *buf, __u32 size_of_buf) = (void *) 16; +static long (* const bpf_get_current_comm)(void *buf, __u32 size_of_buf) = (void *) 16; /* * bpf_get_cgroup_classid @@ -413,7 +423,7 @@ static long (*bpf_get_current_comm)(void *buf, __u32 size_of_buf) = (void *) 16; * Returns * The classid, or 0 for the default unconfigured classid. */ -static __u32 (*bpf_get_cgroup_classid)(struct __sk_buff *skb) = (void *) 17; +static __u32 (* const bpf_get_cgroup_classid)(struct __sk_buff *skb) = (void *) 17; /* * bpf_skb_vlan_push @@ -433,7 +443,7 @@ static __u32 (*bpf_get_cgroup_classid)(struct __sk_buff *skb) = (void *) 17; * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_vlan_push)(struct __sk_buff *skb, __be16 vlan_proto, __u16 vlan_tci) = (void *) 18; +static long (* const bpf_skb_vlan_push)(struct __sk_buff *skb, __be16 vlan_proto, __u16 vlan_tci) = (void *) 18; /* * bpf_skb_vlan_pop @@ -449,7 +459,7 @@ static long (*bpf_skb_vlan_push)(struct __sk_buff *skb, __be16 vlan_proto, __u16 * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_vlan_pop)(struct __sk_buff *skb) = (void *) 19; +static long (* const bpf_skb_vlan_pop)(struct __sk_buff *skb) = (void *) 19; /* * bpf_skb_get_tunnel_key @@ -504,7 +514,7 @@ static long (*bpf_skb_vlan_pop)(struct __sk_buff *skb) = (void *) 19; * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_get_tunnel_key)(struct __sk_buff *skb, struct bpf_tunnel_key *key, __u32 size, __u64 flags) = (void *) 20; +static long (* const bpf_skb_get_tunnel_key)(struct __sk_buff *skb, struct bpf_tunnel_key *key, __u32 size, __u64 flags) = (void *) 20; /* * bpf_skb_set_tunnel_key @@ -529,6 +539,9 @@ static long (*bpf_skb_get_tunnel_key)(struct __sk_buff *skb, struct bpf_tunnel_k * sending the packet. This flag was added for GRE * encapsulation, but might be used with other protocols * as well in the future. + * **BPF_F_NO_TUNNEL_KEY** + * Add a flag to tunnel metadata indicating that no tunnel + * key should be set in the resulting tunnel header. * * Here is a typical usage on the transmit path: * @@ -545,7 +558,7 @@ static long (*bpf_skb_get_tunnel_key)(struct __sk_buff *skb, struct bpf_tunnel_k * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_set_tunnel_key)(struct __sk_buff *skb, struct bpf_tunnel_key *key, __u32 size, __u64 flags) = (void *) 21; +static long (* const bpf_skb_set_tunnel_key)(struct __sk_buff *skb, struct bpf_tunnel_key *key, __u32 size, __u64 flags) = (void *) 21; /* * bpf_perf_event_read @@ -578,7 +591,7 @@ static long (*bpf_skb_set_tunnel_key)(struct __sk_buff *skb, struct bpf_tunnel_k * The value of the perf event counter read from the map, or a * negative error code in case of failure. */ -static __u64 (*bpf_perf_event_read)(void *map, __u64 flags) = (void *) 22; +static __u64 (* const bpf_perf_event_read)(void *map, __u64 flags) = (void *) 22; /* * bpf_redirect @@ -605,7 +618,7 @@ static __u64 (*bpf_perf_event_read)(void *map, __u64 flags) = (void *) 22; * are **TC_ACT_REDIRECT** on success or **TC_ACT_SHOT** on * error. */ -static long (*bpf_redirect)(__u32 ifindex, __u64 flags) = (void *) 23; +static long (* const bpf_redirect)(__u32 ifindex, __u64 flags) = (void *) 23; /* * bpf_get_route_realm @@ -633,7 +646,7 @@ static long (*bpf_redirect)(__u32 ifindex, __u64 flags) = (void *) 23; * The realm of the route for the packet associated to *skb*, or 0 * if none was found. */ -static __u32 (*bpf_get_route_realm)(struct __sk_buff *skb) = (void *) 24; +static __u32 (* const bpf_get_route_realm)(struct __sk_buff *skb) = (void *) 24; /* * bpf_perf_event_output @@ -682,7 +695,7 @@ static __u32 (*bpf_get_route_realm)(struct __sk_buff *skb) = (void *) 24; * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_perf_event_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 25; +static long (* const bpf_perf_event_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 25; /* * bpf_skb_load_bytes @@ -703,7 +716,7 @@ static long (*bpf_perf_event_output)(void *ctx, void *map, __u64 flags, void *da * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_load_bytes)(const void *skb, __u32 offset, void *to, __u32 len) = (void *) 26; +static long (* const bpf_skb_load_bytes)(const void *skb, __u32 offset, void *to, __u32 len) = (void *) 26; /* * bpf_get_stackid @@ -749,7 +762,7 @@ static long (*bpf_skb_load_bytes)(const void *skb, __u32 offset, void *to, __u32 * The positive or null stack id on success, or a negative error * in case of failure. */ -static long (*bpf_get_stackid)(void *ctx, void *map, __u64 flags) = (void *) 27; +static long (* const bpf_get_stackid)(void *ctx, void *map, __u64 flags) = (void *) 27; /* * bpf_csum_diff @@ -780,7 +793,7 @@ static long (*bpf_get_stackid)(void *ctx, void *map, __u64 flags) = (void *) 27; * The checksum result, or a negative error code in case of * failure. */ -static __s64 (*bpf_csum_diff)(__be32 *from, __u32 from_size, __be32 *to, __u32 to_size, __wsum seed) = (void *) 28; +static __s64 (* const bpf_csum_diff)(__be32 *from, __u32 from_size, __be32 *to, __u32 to_size, __wsum seed) = (void *) 28; /* * bpf_skb_get_tunnel_opt @@ -802,7 +815,7 @@ static __s64 (*bpf_csum_diff)(__be32 *from, __u32 from_size, __be32 *to, __u32 t * Returns * The size of the option data retrieved. */ -static long (*bpf_skb_get_tunnel_opt)(struct __sk_buff *skb, void *opt, __u32 size) = (void *) 29; +static long (* const bpf_skb_get_tunnel_opt)(struct __sk_buff *skb, void *opt, __u32 size) = (void *) 29; /* * bpf_skb_set_tunnel_opt @@ -816,7 +829,7 @@ static long (*bpf_skb_get_tunnel_opt)(struct __sk_buff *skb, void *opt, __u32 si * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_set_tunnel_opt)(struct __sk_buff *skb, void *opt, __u32 size) = (void *) 30; +static long (* const bpf_skb_set_tunnel_opt)(struct __sk_buff *skb, void *opt, __u32 size) = (void *) 30; /* * bpf_skb_change_proto @@ -847,7 +860,7 @@ static long (*bpf_skb_set_tunnel_opt)(struct __sk_buff *skb, void *opt, __u32 si * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_change_proto)(struct __sk_buff *skb, __be16 proto, __u64 flags) = (void *) 31; +static long (* const bpf_skb_change_proto)(struct __sk_buff *skb, __be16 proto, __u64 flags) = (void *) 31; /* * bpf_skb_change_type @@ -878,7 +891,7 @@ static long (*bpf_skb_change_proto)(struct __sk_buff *skb, __be16 proto, __u64 f * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_change_type)(struct __sk_buff *skb, __u32 type) = (void *) 32; +static long (* const bpf_skb_change_type)(struct __sk_buff *skb, __u32 type) = (void *) 32; /* * bpf_skb_under_cgroup @@ -893,7 +906,7 @@ static long (*bpf_skb_change_type)(struct __sk_buff *skb, __u32 type) = (void *) * * 1, if the *skb* succeeded the cgroup2 descendant test. * * A negative error code, if an error occurred. */ -static long (*bpf_skb_under_cgroup)(struct __sk_buff *skb, void *map, __u32 index) = (void *) 33; +static long (* const bpf_skb_under_cgroup)(struct __sk_buff *skb, void *map, __u32 index) = (void *) 33; /* * bpf_get_hash_recalc @@ -913,16 +926,17 @@ static long (*bpf_skb_under_cgroup)(struct __sk_buff *skb, void *map, __u32 inde * Returns * The 32-bit hash. */ -static __u32 (*bpf_get_hash_recalc)(struct __sk_buff *skb) = (void *) 34; +static __u32 (* const bpf_get_hash_recalc)(struct __sk_buff *skb) = (void *) 34; /* * bpf_get_current_task * + * Get the current task. * * Returns * A pointer to the current task struct. */ -static __u64 (*bpf_get_current_task)(void) = (void *) 35; +static __u64 (* const bpf_get_current_task)(void) = (void *) 35; /* * bpf_probe_write_user @@ -945,7 +959,7 @@ static __u64 (*bpf_get_current_task)(void) = (void *) 35; * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_probe_write_user)(void *dst, const void *src, __u32 len) = (void *) 36; +static long (* const bpf_probe_write_user)(void *dst, const void *src, __u32 len) = (void *) 36; /* * bpf_current_task_under_cgroup @@ -957,11 +971,11 @@ static long (*bpf_probe_write_user)(void *dst, const void *src, __u32 len) = (vo * Returns * The return value depends on the result of the test, and can be: * - * * 0, if current task belongs to the cgroup2. - * * 1, if current task does not belong to the cgroup2. + * * 1, if current task belongs to the cgroup2. + * * 0, if current task does not belong to the cgroup2. * * A negative error code, if an error occurred. */ -static long (*bpf_current_task_under_cgroup)(void *map, __u32 index) = (void *) 37; +static long (* const bpf_current_task_under_cgroup)(void *map, __u32 index) = (void *) 37; /* * bpf_skb_change_tail @@ -989,7 +1003,7 @@ static long (*bpf_current_task_under_cgroup)(void *map, __u32 index) = (void *) * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_change_tail)(struct __sk_buff *skb, __u32 len, __u64 flags) = (void *) 38; +static long (* const bpf_skb_change_tail)(struct __sk_buff *skb, __u32 len, __u64 flags) = (void *) 38; /* * bpf_skb_pull_data @@ -997,7 +1011,8 @@ static long (*bpf_skb_change_tail)(struct __sk_buff *skb, __u32 len, __u64 flags * Pull in non-linear data in case the *skb* is non-linear and not * all of *len* are part of the linear section. Make *len* bytes * from *skb* readable and writable. If a zero value is passed for - * *len*, then the whole length of the *skb* is pulled. + * *len*, then all bytes in the linear part of *skb* will be made + * readable and writable. * * This helper is only needed for reading and writing with direct * packet access. @@ -1029,7 +1044,7 @@ static long (*bpf_skb_change_tail)(struct __sk_buff *skb, __u32 len, __u64 flags * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_pull_data)(struct __sk_buff *skb, __u32 len) = (void *) 39; +static long (* const bpf_skb_pull_data)(struct __sk_buff *skb, __u32 len) = (void *) 39; /* * bpf_csum_update @@ -1045,7 +1060,7 @@ static long (*bpf_skb_pull_data)(struct __sk_buff *skb, __u32 len) = (void *) 39 * The checksum on success, or a negative error code in case of * failure. */ -static __s64 (*bpf_csum_update)(struct __sk_buff *skb, __wsum csum) = (void *) 40; +static __s64 (* const bpf_csum_update)(struct __sk_buff *skb, __wsum csum) = (void *) 40; /* * bpf_set_hash_invalid @@ -1056,8 +1071,10 @@ static __s64 (*bpf_csum_update)(struct __sk_buff *skb, __wsum csum) = (void *) 4 * recalculation the next time the kernel tries to access this * hash or when the **bpf_get_hash_recalc**\ () helper is called. * + * Returns + * void. */ -static void (*bpf_set_hash_invalid)(struct __sk_buff *skb) = (void *) 41; +static void (* const bpf_set_hash_invalid)(struct __sk_buff *skb) = (void *) 41; /* * bpf_get_numa_node_id @@ -1072,7 +1089,7 @@ static void (*bpf_set_hash_invalid)(struct __sk_buff *skb) = (void *) 41; * Returns * The id of current NUMA node. */ -static long (*bpf_get_numa_node_id)(void) = (void *) 42; +static long (* const bpf_get_numa_node_id)(void) = (void *) 42; /* * bpf_skb_change_head @@ -1097,7 +1114,7 @@ static long (*bpf_get_numa_node_id)(void) = (void *) 42; * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_change_head)(struct __sk_buff *skb, __u32 len, __u64 flags) = (void *) 43; +static long (* const bpf_skb_change_head)(struct __sk_buff *skb, __u32 len, __u64 flags) = (void *) 43; /* * bpf_xdp_adjust_head @@ -1116,7 +1133,7 @@ static long (*bpf_skb_change_head)(struct __sk_buff *skb, __u32 len, __u64 flags * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_xdp_adjust_head)(struct xdp_md *xdp_md, int delta) = (void *) 44; +static long (* const bpf_xdp_adjust_head)(struct xdp_md *xdp_md, int delta) = (void *) 44; /* * bpf_probe_read_str @@ -1133,7 +1150,7 @@ static long (*bpf_xdp_adjust_head)(struct xdp_md *xdp_md, int delta) = (void *) * including the trailing NUL character. On error, a negative * value. */ -static long (*bpf_probe_read_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 45; +static long (* const bpf_probe_read_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 45; /* * bpf_get_socket_cookie @@ -1150,11 +1167,12 @@ static long (*bpf_probe_read_str)(void *dst, __u32 size, const void *unsafe_ptr) * A 8-byte long unique number on success, or 0 if the socket * field is missing inside *skb*. */ -static __u64 (*bpf_get_socket_cookie)(void *ctx) = (void *) 46; +static __u64 (* const bpf_get_socket_cookie)(void *ctx) = (void *) 46; /* * bpf_get_socket_uid * + * Get the owner UID of the socked associated to *skb*. * * Returns * The owner UID of the socket associated to *skb*. If the socket @@ -1163,7 +1181,7 @@ static __u64 (*bpf_get_socket_cookie)(void *ctx) = (void *) 46; * is returned (note that **overflowuid** might also be the actual * UID value for the socket). */ -static __u32 (*bpf_get_socket_uid)(struct __sk_buff *skb) = (void *) 47; +static __u32 (* const bpf_get_socket_uid)(struct __sk_buff *skb) = (void *) 47; /* * bpf_set_hash @@ -1174,7 +1192,7 @@ static __u32 (*bpf_get_socket_uid)(struct __sk_buff *skb) = (void *) 47; * Returns * 0 */ -static long (*bpf_set_hash)(struct __sk_buff *skb, __u32 hash) = (void *) 48; +static long (* const bpf_set_hash)(struct __sk_buff *skb, __u32 hash) = (void *) 48; /* * bpf_setsockopt @@ -1188,8 +1206,8 @@ static long (*bpf_set_hash)(struct __sk_buff *skb, __u32 hash) = (void *) 48; * *bpf_socket* should be one of the following: * * * **struct bpf_sock_ops** for **BPF_PROG_TYPE_SOCK_OPS**. - * * **struct bpf_sock_addr** for **BPF_CGROUP_INET4_CONNECT** - * and **BPF_CGROUP_INET6_CONNECT**. + * * **struct bpf_sock_addr** for **BPF_CGROUP_INET4_CONNECT**, + * **BPF_CGROUP_INET6_CONNECT** and **BPF_CGROUP_UNIX_CONNECT**. * * This helper actually implements a subset of **setsockopt()**. * It supports the following *level*\ s: @@ -1197,19 +1215,24 @@ static long (*bpf_set_hash)(struct __sk_buff *skb, __u32 hash) = (void *) 48; * * **SOL_SOCKET**, which supports the following *optname*\ s: * **SO_RCVBUF**, **SO_SNDBUF**, **SO_MAX_PACING_RATE**, * **SO_PRIORITY**, **SO_RCVLOWAT**, **SO_MARK**, - * **SO_BINDTODEVICE**, **SO_KEEPALIVE**. + * **SO_BINDTODEVICE**, **SO_KEEPALIVE**, **SO_REUSEADDR**, + * **SO_REUSEPORT**, **SO_BINDTOIFINDEX**, **SO_TXREHASH**. * * **IPPROTO_TCP**, which supports the following *optname*\ s: * **TCP_CONGESTION**, **TCP_BPF_IW**, * **TCP_BPF_SNDCWND_CLAMP**, **TCP_SAVE_SYN**, * **TCP_KEEPIDLE**, **TCP_KEEPINTVL**, **TCP_KEEPCNT**, - * **TCP_SYNCNT**, **TCP_USER_TIMEOUT**, **TCP_NOTSENT_LOWAT**. + * **TCP_SYNCNT**, **TCP_USER_TIMEOUT**, **TCP_NOTSENT_LOWAT**, + * **TCP_NODELAY**, **TCP_MAXSEG**, **TCP_WINDOW_CLAMP**, + * **TCP_THIN_LINEAR_TIMEOUTS**, **TCP_BPF_DELACK_MAX**, + * **TCP_BPF_RTO_MIN**. * * **IPPROTO_IP**, which supports *optname* **IP_TOS**. - * * **IPPROTO_IPV6**, which supports *optname* **IPV6_TCLASS**. + * * **IPPROTO_IPV6**, which supports the following *optname*\ s: + * **IPV6_TCLASS**, **IPV6_AUTOFLOWLABEL**. * * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_setsockopt)(void *bpf_socket, int level, int optname, void *optval, int optlen) = (void *) 49; +static long (* const bpf_setsockopt)(void *bpf_socket, int level, int optname, void *optval, int optlen) = (void *) 49; /* * bpf_skb_adjust_room @@ -1227,10 +1250,12 @@ static long (*bpf_setsockopt)(void *bpf_socket, int level, int optname, void *op * There are two supported modes at this time: * * * **BPF_ADJ_ROOM_MAC**: Adjust room at the mac layer - * (room space is added or removed below the layer 2 header). + * (room space is added or removed between the layer 2 and + * layer 3 headers). * * * **BPF_ADJ_ROOM_NET**: Adjust room at the network layer - * (room space is added or removed below the layer 3 header). + * (room space is added or removed between the layer 3 and + * layer 4 headers). * * The following flags are supported at this time: * @@ -1254,6 +1279,11 @@ static long (*bpf_setsockopt)(void *bpf_socket, int level, int optname, void *op * Use with BPF_F_ADJ_ROOM_ENCAP_L2 flag to further specify the * L2 type as Ethernet. * + * * **BPF_F_ADJ_ROOM_DECAP_L3_IPV4**, + * **BPF_F_ADJ_ROOM_DECAP_L3_IPV6**: + * Indicate the new IP header version after decapsulating the outer + * IP header. Used when the inner and outer IP versions are different. + * * A call to this helper is susceptible to change the underlying * packet buffer. Therefore, at load time, all checks on pointers * previously done by the verifier are invalidated and must be @@ -1263,7 +1293,7 @@ static long (*bpf_setsockopt)(void *bpf_socket, int level, int optname, void *op * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_adjust_room)(struct __sk_buff *skb, __s32 len_diff, __u32 mode, __u64 flags) = (void *) 50; +static long (* const bpf_skb_adjust_room)(struct __sk_buff *skb, __s32 len_diff, __u32 mode, __u64 flags) = (void *) 50; /* * bpf_redirect_map @@ -1292,7 +1322,7 @@ static long (*bpf_skb_adjust_room)(struct __sk_buff *skb, __s32 len_diff, __u32 * **XDP_REDIRECT** on success, or the value of the two lower bits * of the *flags* argument on error. */ -static long (*bpf_redirect_map)(void *map, __u32 key, __u64 flags) = (void *) 51; +static long (* const bpf_redirect_map)(void *map, __u64 key, __u64 flags) = (void *) 51; /* * bpf_sk_redirect_map @@ -1307,7 +1337,7 @@ static long (*bpf_redirect_map)(void *map, __u32 key, __u64 flags) = (void *) 51 * Returns * **SK_PASS** on success, or **SK_DROP** on error. */ -static long (*bpf_sk_redirect_map)(struct __sk_buff *skb, void *map, __u32 key, __u64 flags) = (void *) 52; +static long (* const bpf_sk_redirect_map)(struct __sk_buff *skb, void *map, __u32 key, __u64 flags) = (void *) 52; /* * bpf_sock_map_update @@ -1330,7 +1360,7 @@ static long (*bpf_sk_redirect_map)(struct __sk_buff *skb, void *map, __u32 key, * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_sock_map_update)(struct bpf_sock_ops *skops, void *map, void *key, __u64 flags) = (void *) 53; +static long (* const bpf_sock_map_update)(struct bpf_sock_ops *skops, void *map, void *key, __u64 flags) = (void *) 53; /* * bpf_xdp_adjust_meta @@ -1363,7 +1393,7 @@ static long (*bpf_sock_map_update)(struct bpf_sock_ops *skops, void *map, void * * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_xdp_adjust_meta)(struct xdp_md *xdp_md, int delta) = (void *) 54; +static long (* const bpf_xdp_adjust_meta)(struct xdp_md *xdp_md, int delta) = (void *) 54; /* * bpf_perf_event_read_value @@ -1417,12 +1447,12 @@ static long (*bpf_xdp_adjust_meta)(struct xdp_md *xdp_md, int delta) = (void *) * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_perf_event_read_value)(void *map, __u64 flags, struct bpf_perf_event_value *buf, __u32 buf_size) = (void *) 55; +static long (* const bpf_perf_event_read_value)(void *map, __u64 flags, struct bpf_perf_event_value *buf, __u32 buf_size) = (void *) 55; /* * bpf_perf_prog_read_value * - * For en eBPF program attached to a perf event, retrieve the + * For an eBPF program attached to a perf event, retrieve the * value of the event counter associated to *ctx* and store it in * the structure pointed by *buf* and of size *buf_size*. Enabled * and running times are also stored in the structure (see @@ -1432,7 +1462,7 @@ static long (*bpf_perf_event_read_value)(void *map, __u64 flags, struct bpf_perf * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_perf_prog_read_value)(struct bpf_perf_event_data *ctx, struct bpf_perf_event_value *buf, __u32 buf_size) = (void *) 56; +static long (* const bpf_perf_prog_read_value)(struct bpf_perf_event_data *ctx, struct bpf_perf_event_value *buf, __u32 buf_size) = (void *) 56; /* * bpf_getsockopt @@ -1447,21 +1477,19 @@ static long (*bpf_perf_prog_read_value)(struct bpf_perf_event_data *ctx, struct * *bpf_socket* should be one of the following: * * * **struct bpf_sock_ops** for **BPF_PROG_TYPE_SOCK_OPS**. - * * **struct bpf_sock_addr** for **BPF_CGROUP_INET4_CONNECT** - * and **BPF_CGROUP_INET6_CONNECT**. + * * **struct bpf_sock_addr** for **BPF_CGROUP_INET4_CONNECT**, + * **BPF_CGROUP_INET6_CONNECT** and **BPF_CGROUP_UNIX_CONNECT**. * * This helper actually implements a subset of **getsockopt()**. - * It supports the following *level*\ s: - * - * * **IPPROTO_TCP**, which supports *optname* - * **TCP_CONGESTION**. - * * **IPPROTO_IP**, which supports *optname* **IP_TOS**. - * * **IPPROTO_IPV6**, which supports *optname* **IPV6_TCLASS**. + * It supports the same set of *optname*\ s that is supported by + * the **bpf_setsockopt**\ () helper. The exceptions are + * **TCP_BPF_*** is **bpf_setsockopt**\ () only and + * **TCP_SAVED_SYN** is **bpf_getsockopt**\ () only. * * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_getsockopt)(void *bpf_socket, int level, int optname, void *optval, int optlen) = (void *) 57; +static long (* const bpf_getsockopt)(void *bpf_socket, int level, int optname, void *optval, int optlen) = (void *) 57; /* * bpf_override_return @@ -1490,7 +1518,7 @@ static long (*bpf_getsockopt)(void *bpf_socket, int level, int optname, void *op * Returns * 0 */ -static long (*bpf_override_return)(struct pt_regs *regs, __u64 rc) = (void *) 58; +static long (* const bpf_override_return)(struct pt_regs *regs, __u64 rc) = (void *) 58; /* * bpf_sock_ops_cb_flags_set @@ -1538,7 +1566,7 @@ static long (*bpf_override_return)(struct pt_regs *regs, __u64 rc) = (void *) 58 * be set is returned (which comes down to 0 if all bits were set * as required). */ -static long (*bpf_sock_ops_cb_flags_set)(struct bpf_sock_ops *bpf_sock, int argval) = (void *) 59; +static long (* const bpf_sock_ops_cb_flags_set)(struct bpf_sock_ops *bpf_sock, int argval) = (void *) 59; /* * bpf_msg_redirect_map @@ -1556,7 +1584,7 @@ static long (*bpf_sock_ops_cb_flags_set)(struct bpf_sock_ops *bpf_sock, int argv * Returns * **SK_PASS** on success, or **SK_DROP** on error. */ -static long (*bpf_msg_redirect_map)(struct sk_msg_md *msg, void *map, __u32 key, __u64 flags) = (void *) 60; +static long (* const bpf_msg_redirect_map)(struct sk_msg_md *msg, void *map, __u32 key, __u64 flags) = (void *) 60; /* * bpf_msg_apply_bytes @@ -1594,7 +1622,7 @@ static long (*bpf_msg_redirect_map)(struct sk_msg_md *msg, void *map, __u32 key, * Returns * 0 */ -static long (*bpf_msg_apply_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void *) 61; +static long (* const bpf_msg_apply_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void *) 61; /* * bpf_msg_cork_bytes @@ -1616,7 +1644,7 @@ static long (*bpf_msg_apply_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void * * Returns * 0 */ -static long (*bpf_msg_cork_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void *) 62; +static long (* const bpf_msg_cork_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void *) 62; /* * bpf_msg_pull_data @@ -1651,7 +1679,7 @@ static long (*bpf_msg_cork_bytes)(struct sk_msg_md *msg, __u32 bytes) = (void *) * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_msg_pull_data)(struct sk_msg_md *msg, __u32 start, __u32 end, __u64 flags) = (void *) 63; +static long (* const bpf_msg_pull_data)(struct sk_msg_md *msg, __u32 start, __u32 end, __u64 flags) = (void *) 63; /* * bpf_bind @@ -1673,7 +1701,7 @@ static long (*bpf_msg_pull_data)(struct sk_msg_md *msg, __u32 start, __u32 end, * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_bind)(struct bpf_sock_addr *ctx, struct sockaddr *addr, int addr_len) = (void *) 64; +static long (* const bpf_bind)(struct bpf_sock_addr *ctx, struct sockaddr *addr, int addr_len) = (void *) 64; /* * bpf_xdp_adjust_tail @@ -1691,7 +1719,7 @@ static long (*bpf_bind)(struct bpf_sock_addr *ctx, struct sockaddr *addr, int ad * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_xdp_adjust_tail)(struct xdp_md *xdp_md, int delta) = (void *) 65; +static long (* const bpf_xdp_adjust_tail)(struct xdp_md *xdp_md, int delta) = (void *) 65; /* * bpf_skb_get_xfrm_state @@ -1711,7 +1739,7 @@ static long (*bpf_xdp_adjust_tail)(struct xdp_md *xdp_md, int delta) = (void *) * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_get_xfrm_state)(struct __sk_buff *skb, __u32 index, struct bpf_xfrm_state *xfrm_state, __u32 size, __u64 flags) = (void *) 66; +static long (* const bpf_skb_get_xfrm_state)(struct __sk_buff *skb, __u32 index, struct bpf_xfrm_state *xfrm_state, __u32 size, __u64 flags) = (void *) 66; /* * bpf_get_stack @@ -1730,8 +1758,18 @@ static long (*bpf_skb_get_xfrm_state)(struct __sk_buff *skb, __u32 index, struct * **BPF_F_USER_STACK** * Collect a user space stack instead of a kernel stack. * **BPF_F_USER_BUILD_ID** - * Collect buildid+offset instead of ips for user stack, - * only valid if **BPF_F_USER_STACK** is also specified. + * Collect (build_id, file_offset) instead of ips for user + * stack, only valid if **BPF_F_USER_STACK** is also + * specified. + * + * *file_offset* is an offset relative to the beginning + * of the executable or shared object file backing the vma + * which the *ip* falls in. It is *not* an offset relative + * to that object's base address. Accordingly, it must be + * adjusted by adding (sh_addr - sh_offset), where + * sh_{addr,offset} correspond to the executable section + * containing *file_offset* in the object, for comparisons + * to symbols' st_value to be valid. * * **bpf_get_stack**\ () can collect up to * **PERF_MAX_STACK_DEPTH** both kernel and user frames, subject @@ -1745,10 +1783,10 @@ static long (*bpf_skb_get_xfrm_state)(struct __sk_buff *skb, __u32 index, struct * # sysctl kernel.perf_event_max_stack= * * Returns - * A non-negative value equal to or less than *size* on success, - * or a negative error in case of failure. + * The non-negative copied *buf* length equal to or less than + * *size* on success, or a negative error in case of failure. */ -static long (*bpf_get_stack)(void *ctx, void *buf, __u32 size, __u64 flags) = (void *) 67; +static long (* const bpf_get_stack)(void *ctx, void *buf, __u32 size, __u64 flags) = (void *) 67; /* * bpf_skb_load_bytes_relative @@ -1774,7 +1812,7 @@ static long (*bpf_get_stack)(void *ctx, void *buf, __u32 size, __u64 flags) = (v * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_load_bytes_relative)(const void *skb, __u32 offset, void *to, __u32 len, __u32 start_header) = (void *) 68; +static long (* const bpf_skb_load_bytes_relative)(const void *skb, __u32 offset, void *to, __u32 len, __u32 start_header) = (void *) 68; /* * bpf_fib_lookup @@ -1796,9 +1834,23 @@ static long (*bpf_skb_load_bytes_relative)(const void *skb, __u32 offset, void * * **BPF_FIB_LOOKUP_DIRECT** * Do a direct table lookup vs full lookup using FIB * rules. + * **BPF_FIB_LOOKUP_TBID** + * Used with BPF_FIB_LOOKUP_DIRECT. + * Use the routing table ID present in *params*->tbid + * for the fib lookup. * **BPF_FIB_LOOKUP_OUTPUT** * Perform lookup from an egress perspective (default is * ingress). + * **BPF_FIB_LOOKUP_SKIP_NEIGH** + * Skip the neighbour table lookup. *params*->dmac + * and *params*->smac will not be set as output. A common + * use case is to call **bpf_redirect_neigh**\ () after + * doing **bpf_fib_lookup**\ (). + * **BPF_FIB_LOOKUP_SRC** + * Derive and set source IP addr in *params*->ipv{4,6}_src + * for the nexthop. If the src addr cannot be derived, + * **BPF_FIB_LKUP_RET_NO_SRC_ADDR** is returned. In this + * case, *params*->dmac and *params*->smac are not set either. * * *ctx* is either **struct xdp_md** for XDP programs or * **struct sk_buff** tc cls_act programs. @@ -1812,7 +1864,7 @@ static long (*bpf_skb_load_bytes_relative)(const void *skb, __u32 offset, void * * If lookup fails with BPF_FIB_LKUP_RET_FRAG_NEEDED, then the MTU * was exceeded and output params->mtu_result contains the MTU. */ -static long (*bpf_fib_lookup)(void *ctx, struct bpf_fib_lookup *params, int plen, __u32 flags) = (void *) 69; +static long (* const bpf_fib_lookup)(void *ctx, struct bpf_fib_lookup *params, int plen, __u32 flags) = (void *) 69; /* * bpf_sock_hash_update @@ -1835,7 +1887,7 @@ static long (*bpf_fib_lookup)(void *ctx, struct bpf_fib_lookup *params, int plen * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_sock_hash_update)(struct bpf_sock_ops *skops, void *map, void *key, __u64 flags) = (void *) 70; +static long (* const bpf_sock_hash_update)(struct bpf_sock_ops *skops, void *map, void *key, __u64 flags) = (void *) 70; /* * bpf_msg_redirect_hash @@ -1853,7 +1905,7 @@ static long (*bpf_sock_hash_update)(struct bpf_sock_ops *skops, void *map, void * Returns * **SK_PASS** on success, or **SK_DROP** on error. */ -static long (*bpf_msg_redirect_hash)(struct sk_msg_md *msg, void *map, void *key, __u64 flags) = (void *) 71; +static long (* const bpf_msg_redirect_hash)(struct sk_msg_md *msg, void *map, void *key, __u64 flags) = (void *) 71; /* * bpf_sk_redirect_hash @@ -1871,7 +1923,7 @@ static long (*bpf_msg_redirect_hash)(struct sk_msg_md *msg, void *map, void *key * Returns * **SK_PASS** on success, or **SK_DROP** on error. */ -static long (*bpf_sk_redirect_hash)(struct __sk_buff *skb, void *map, void *key, __u64 flags) = (void *) 72; +static long (* const bpf_sk_redirect_hash)(struct __sk_buff *skb, void *map, void *key, __u64 flags) = (void *) 72; /* * bpf_lwt_push_encap @@ -1912,7 +1964,7 @@ static long (*bpf_sk_redirect_hash)(struct __sk_buff *skb, void *map, void *key, * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_lwt_push_encap)(struct __sk_buff *skb, __u32 type, void *hdr, __u32 len) = (void *) 73; +static long (* const bpf_lwt_push_encap)(struct __sk_buff *skb, __u32 type, void *hdr, __u32 len) = (void *) 73; /* * bpf_lwt_seg6_store_bytes @@ -1931,7 +1983,7 @@ static long (*bpf_lwt_push_encap)(struct __sk_buff *skb, __u32 type, void *hdr, * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_lwt_seg6_store_bytes)(struct __sk_buff *skb, __u32 offset, const void *from, __u32 len) = (void *) 74; +static long (* const bpf_lwt_seg6_store_bytes)(struct __sk_buff *skb, __u32 offset, const void *from, __u32 len) = (void *) 74; /* * bpf_lwt_seg6_adjust_srh @@ -1951,7 +2003,7 @@ static long (*bpf_lwt_seg6_store_bytes)(struct __sk_buff *skb, __u32 offset, con * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_lwt_seg6_adjust_srh)(struct __sk_buff *skb, __u32 offset, __s32 delta) = (void *) 75; +static long (* const bpf_lwt_seg6_adjust_srh)(struct __sk_buff *skb, __u32 offset, __s32 delta) = (void *) 75; /* * bpf_lwt_seg6_action @@ -1984,7 +2036,7 @@ static long (*bpf_lwt_seg6_adjust_srh)(struct __sk_buff *skb, __u32 offset, __s3 * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_lwt_seg6_action)(struct __sk_buff *skb, __u32 action, void *param, __u32 param_len) = (void *) 76; +static long (* const bpf_lwt_seg6_action)(struct __sk_buff *skb, __u32 action, void *param, __u32 param_len) = (void *) 76; /* * bpf_rc_repeat @@ -2007,7 +2059,7 @@ static long (*bpf_lwt_seg6_action)(struct __sk_buff *skb, __u32 action, void *pa * Returns * 0 */ -static long (*bpf_rc_repeat)(void *ctx) = (void *) 77; +static long (* const bpf_rc_repeat)(void *ctx) = (void *) 77; /* * bpf_rc_keydown @@ -2037,7 +2089,7 @@ static long (*bpf_rc_repeat)(void *ctx) = (void *) 77; * Returns * 0 */ -static long (*bpf_rc_keydown)(void *ctx, __u32 protocol, __u64 scancode, __u32 toggle) = (void *) 78; +static long (* const bpf_rc_keydown)(void *ctx, __u32 protocol, __u64 scancode, __u32 toggle) = (void *) 78; /* * bpf_skb_cgroup_id @@ -2057,17 +2109,19 @@ static long (*bpf_rc_keydown)(void *ctx, __u32 protocol, __u64 scancode, __u32 t * Returns * The id is returned or 0 in case the id could not be retrieved. */ -static __u64 (*bpf_skb_cgroup_id)(struct __sk_buff *skb) = (void *) 79; +static __u64 (* const bpf_skb_cgroup_id)(struct __sk_buff *skb) = (void *) 79; /* * bpf_get_current_cgroup_id * + * Get the current cgroup id based on the cgroup within which + * the current task is running. * * Returns * A 64-bit integer containing the current cgroup id based * on the cgroup within which the current task is running. */ -static __u64 (*bpf_get_current_cgroup_id)(void) = (void *) 80; +static __u64 (* const bpf_get_current_cgroup_id)(void) = (void *) 80; /* * bpf_get_local_storage @@ -2089,7 +2143,7 @@ static __u64 (*bpf_get_current_cgroup_id)(void) = (void *) 80; * Returns * A pointer to the local storage area. */ -static void *(*bpf_get_local_storage)(void *map, __u64 flags) = (void *) 81; +static void *(* const bpf_get_local_storage)(void *map, __u64 flags) = (void *) 81; /* * bpf_sk_select_reuseport @@ -2102,7 +2156,7 @@ static void *(*bpf_get_local_storage)(void *map, __u64 flags) = (void *) 81; * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_sk_select_reuseport)(struct sk_reuseport_md *reuse, void *map, void *key, __u64 flags) = (void *) 82; +static long (* const bpf_sk_select_reuseport)(struct sk_reuseport_md *reuse, void *map, void *key, __u64 flags) = (void *) 82; /* * bpf_skb_ancestor_cgroup_id @@ -2124,7 +2178,7 @@ static long (*bpf_sk_select_reuseport)(struct sk_reuseport_md *reuse, void *map, * Returns * The id is returned or 0 in case the id could not be retrieved. */ -static __u64 (*bpf_skb_ancestor_cgroup_id)(struct __sk_buff *skb, int ancestor_level) = (void *) 83; +static __u64 (* const bpf_skb_ancestor_cgroup_id)(struct __sk_buff *skb, int ancestor_level) = (void *) 83; /* * bpf_sk_lookup_tcp @@ -2165,7 +2219,7 @@ static __u64 (*bpf_skb_ancestor_cgroup_id)(struct __sk_buff *skb, int ancestor_l * result is from *reuse*\ **->socks**\ [] using the hash of the * tuple. */ -static struct bpf_sock *(*bpf_sk_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 84; +static struct bpf_sock *(* const bpf_sk_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 84; /* * bpf_sk_lookup_udp @@ -2206,7 +2260,7 @@ static struct bpf_sock *(*bpf_sk_lookup_tcp)(void *ctx, struct bpf_sock_tuple *t * result is from *reuse*\ **->socks**\ [] using the hash of the * tuple. */ -static struct bpf_sock *(*bpf_sk_lookup_udp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 85; +static struct bpf_sock *(* const bpf_sk_lookup_udp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 85; /* * bpf_sk_release @@ -2218,7 +2272,7 @@ static struct bpf_sock *(*bpf_sk_lookup_udp)(void *ctx, struct bpf_sock_tuple *t * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_sk_release)(void *sock) = (void *) 86; +static long (* const bpf_sk_release)(void *sock) = (void *) 86; /* * bpf_map_push_elem @@ -2232,7 +2286,7 @@ static long (*bpf_sk_release)(void *sock) = (void *) 86; * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_map_push_elem)(void *map, const void *value, __u64 flags) = (void *) 87; +static long (* const bpf_map_push_elem)(void *map, const void *value, __u64 flags) = (void *) 87; /* * bpf_map_pop_elem @@ -2242,7 +2296,7 @@ static long (*bpf_map_push_elem)(void *map, const void *value, __u64 flags) = (v * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_map_pop_elem)(void *map, void *value) = (void *) 88; +static long (* const bpf_map_pop_elem)(void *map, void *value) = (void *) 88; /* * bpf_map_peek_elem @@ -2252,7 +2306,7 @@ static long (*bpf_map_pop_elem)(void *map, void *value) = (void *) 88; * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_map_peek_elem)(void *map, void *value) = (void *) 89; +static long (* const bpf_map_peek_elem)(void *map, void *value) = (void *) 89; /* * bpf_msg_push_data @@ -2272,7 +2326,7 @@ static long (*bpf_map_peek_elem)(void *map, void *value) = (void *) 89; * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_msg_push_data)(struct sk_msg_md *msg, __u32 start, __u32 len, __u64 flags) = (void *) 90; +static long (* const bpf_msg_push_data)(struct sk_msg_md *msg, __u32 start, __u32 len, __u64 flags) = (void *) 90; /* * bpf_msg_pop_data @@ -2288,7 +2342,7 @@ static long (*bpf_msg_push_data)(struct sk_msg_md *msg, __u32 start, __u32 len, * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_msg_pop_data)(struct sk_msg_md *msg, __u32 start, __u32 len, __u64 flags) = (void *) 91; +static long (* const bpf_msg_pop_data)(struct sk_msg_md *msg, __u32 start, __u32 len, __u64 flags) = (void *) 91; /* * bpf_rc_pointer_rel @@ -2306,7 +2360,7 @@ static long (*bpf_msg_pop_data)(struct sk_msg_md *msg, __u32 start, __u32 len, _ * Returns * 0 */ -static long (*bpf_rc_pointer_rel)(void *ctx, __s32 rel_x, __s32 rel_y) = (void *) 92; +static long (* const bpf_rc_pointer_rel)(void *ctx, __s32 rel_x, __s32 rel_y) = (void *) 92; /* * bpf_spin_lock @@ -2358,7 +2412,7 @@ static long (*bpf_rc_pointer_rel)(void *ctx, __s32 rel_x, __s32 rel_y) = (void * * Returns * 0 */ -static long (*bpf_spin_lock)(struct bpf_spin_lock *lock) = (void *) 93; +static long (* const bpf_spin_lock)(struct bpf_spin_lock *lock) = (void *) 93; /* * bpf_spin_unlock @@ -2369,7 +2423,7 @@ static long (*bpf_spin_lock)(struct bpf_spin_lock *lock) = (void *) 93; * Returns * 0 */ -static long (*bpf_spin_unlock)(struct bpf_spin_lock *lock) = (void *) 94; +static long (* const bpf_spin_unlock)(struct bpf_spin_lock *lock) = (void *) 94; /* * bpf_sk_fullsock @@ -2381,7 +2435,7 @@ static long (*bpf_spin_unlock)(struct bpf_spin_lock *lock) = (void *) 94; * A **struct bpf_sock** pointer on success, or **NULL** in * case of failure. */ -static struct bpf_sock *(*bpf_sk_fullsock)(struct bpf_sock *sk) = (void *) 95; +static struct bpf_sock *(* const bpf_sk_fullsock)(struct bpf_sock *sk) = (void *) 95; /* * bpf_tcp_sock @@ -2393,7 +2447,7 @@ static struct bpf_sock *(*bpf_sk_fullsock)(struct bpf_sock *sk) = (void *) 95; * A **struct bpf_tcp_sock** pointer on success, or **NULL** in * case of failure. */ -static struct bpf_tcp_sock *(*bpf_tcp_sock)(struct bpf_sock *sk) = (void *) 96; +static struct bpf_tcp_sock *(* const bpf_tcp_sock)(struct bpf_sock *sk) = (void *) 96; /* * bpf_skb_ecn_set_ce @@ -2407,7 +2461,7 @@ static struct bpf_tcp_sock *(*bpf_tcp_sock)(struct bpf_sock *sk) = (void *) 96; * 1 if the **CE** flag is set (either by the current helper call * or because it was already present), 0 if it is not set. */ -static long (*bpf_skb_ecn_set_ce)(struct __sk_buff *skb) = (void *) 97; +static long (* const bpf_skb_ecn_set_ce)(struct __sk_buff *skb) = (void *) 97; /* * bpf_get_listener_sock @@ -2419,7 +2473,7 @@ static long (*bpf_skb_ecn_set_ce)(struct __sk_buff *skb) = (void *) 97; * A **struct bpf_sock** pointer on success, or **NULL** in * case of failure. */ -static struct bpf_sock *(*bpf_get_listener_sock)(struct bpf_sock *sk) = (void *) 98; +static struct bpf_sock *(* const bpf_get_listener_sock)(struct bpf_sock *sk) = (void *) 98; /* * bpf_skc_lookup_tcp @@ -2442,7 +2496,7 @@ static struct bpf_sock *(*bpf_get_listener_sock)(struct bpf_sock *sk) = (void *) * result is from *reuse*\ **->socks**\ [] using the hash of the * tuple. */ -static struct bpf_sock *(*bpf_skc_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 99; +static struct bpf_sock *(* const bpf_skc_lookup_tcp)(void *ctx, struct bpf_sock_tuple *tuple, __u32 tuple_size, __u64 netns, __u64 flags) = (void *) 99; /* * bpf_tcp_check_syncookie @@ -2452,16 +2506,17 @@ static struct bpf_sock *(*bpf_skc_lookup_tcp)(void *ctx, struct bpf_sock_tuple * * * *iph* points to the start of the IPv4 or IPv6 header, while * *iph_len* contains **sizeof**\ (**struct iphdr**) or - * **sizeof**\ (**struct ip6hdr**). + * **sizeof**\ (**struct ipv6hdr**). * * *th* points to the start of the TCP header, while *th_len* - * contains **sizeof**\ (**struct tcphdr**). + * contains the length of the TCP header (at least + * **sizeof**\ (**struct tcphdr**)). * * Returns * 0 if *iph* and *th* are a valid SYN cookie ACK, or a negative * error otherwise. */ -static long (*bpf_tcp_check_syncookie)(void *sk, void *iph, __u32 iph_len, struct tcphdr *th, __u32 th_len) = (void *) 100; +static long (* const bpf_tcp_check_syncookie)(void *sk, void *iph, __u32 iph_len, struct tcphdr *th, __u32 th_len) = (void *) 100; /* * bpf_sysctl_get_name @@ -2481,7 +2536,7 @@ static long (*bpf_tcp_check_syncookie)(void *sk, void *iph, __u32 iph_len, struc * **-E2BIG** if the buffer wasn't big enough (*buf* will contain * truncated name in this case). */ -static long (*bpf_sysctl_get_name)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len, __u64 flags) = (void *) 101; +static long (* const bpf_sysctl_get_name)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len, __u64 flags) = (void *) 101; /* * bpf_sysctl_get_current_value @@ -2504,7 +2559,7 @@ static long (*bpf_sysctl_get_name)(struct bpf_sysctl *ctx, char *buf, unsigned l * **-EINVAL** if current value was unavailable, e.g. because * sysctl is uninitialized and read returns -EIO for it. */ -static long (*bpf_sysctl_get_current_value)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len) = (void *) 102; +static long (* const bpf_sysctl_get_current_value)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len) = (void *) 102; /* * bpf_sysctl_get_new_value @@ -2525,7 +2580,7 @@ static long (*bpf_sysctl_get_current_value)(struct bpf_sysctl *ctx, char *buf, u * * **-EINVAL** if sysctl is being read. */ -static long (*bpf_sysctl_get_new_value)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len) = (void *) 103; +static long (* const bpf_sysctl_get_new_value)(struct bpf_sysctl *ctx, char *buf, unsigned long buf_len) = (void *) 103; /* * bpf_sysctl_set_new_value @@ -2546,7 +2601,7 @@ static long (*bpf_sysctl_get_new_value)(struct bpf_sysctl *ctx, char *buf, unsig * * **-EINVAL** if sysctl is being read. */ -static long (*bpf_sysctl_set_new_value)(struct bpf_sysctl *ctx, const char *buf, unsigned long buf_len) = (void *) 104; +static long (* const bpf_sysctl_set_new_value)(struct bpf_sysctl *ctx, const char *buf, unsigned long buf_len) = (void *) 104; /* * bpf_strtol @@ -2574,7 +2629,7 @@ static long (*bpf_sysctl_set_new_value)(struct bpf_sysctl *ctx, const char *buf, * * **-ERANGE** if resulting value was out of range. */ -static long (*bpf_strtol)(const char *buf, unsigned long buf_len, __u64 flags, long *res) = (void *) 105; +static long (* const bpf_strtol)(const char *buf, unsigned long buf_len, __u64 flags, long *res) = (void *) 105; /* * bpf_strtoul @@ -2601,7 +2656,7 @@ static long (*bpf_strtol)(const char *buf, unsigned long buf_len, __u64 flags, l * * **-ERANGE** if resulting value was out of range. */ -static long (*bpf_strtoul)(const char *buf, unsigned long buf_len, __u64 flags, unsigned long *res) = (void *) 106; +static long (* const bpf_strtoul)(const char *buf, unsigned long buf_len, __u64 flags, unsigned long *res) = (void *) 106; /* * bpf_sk_storage_get @@ -2636,7 +2691,7 @@ static long (*bpf_strtoul)(const char *buf, unsigned long buf_len, __u64 flags, * **NULL** if not found or there was an error in adding * a new bpf-local-storage. */ -static void *(*bpf_sk_storage_get)(void *map, void *sk, void *value, __u64 flags) = (void *) 107; +static void *(* const bpf_sk_storage_get)(void *map, void *sk, void *value, __u64 flags) = (void *) 107; /* * bpf_sk_storage_delete @@ -2649,7 +2704,7 @@ static void *(*bpf_sk_storage_get)(void *map, void *sk, void *value, __u64 flags * **-ENOENT** if the bpf-local-storage cannot be found. * **-EINVAL** if sk is not a fullsock (e.g. a request_sock). */ -static long (*bpf_sk_storage_delete)(void *map, void *sk) = (void *) 108; +static long (* const bpf_sk_storage_delete)(void *map, void *sk) = (void *) 108; /* * bpf_send_signal @@ -2668,7 +2723,7 @@ static long (*bpf_sk_storage_delete)(void *map, void *sk) = (void *) 108; * * **-EAGAIN** if bpf program can try again. */ -static long (*bpf_send_signal)(__u32 sig) = (void *) 109; +static long (* const bpf_send_signal)(__u32 sig) = (void *) 109; /* * bpf_tcp_gen_syncookie @@ -2678,10 +2733,11 @@ static long (*bpf_send_signal)(__u32 sig) = (void *) 109; * * *iph* points to the start of the IPv4 or IPv6 header, while * *iph_len* contains **sizeof**\ (**struct iphdr**) or - * **sizeof**\ (**struct ip6hdr**). + * **sizeof**\ (**struct ipv6hdr**). * * *th* points to the start of the TCP header, while *th_len* - * contains the length of the TCP header. + * contains the length of the TCP header with options (at least + * **sizeof**\ (**struct tcphdr**)). * * Returns * On success, lower 32 bits hold the generated SYN cookie in @@ -2698,7 +2754,7 @@ static long (*bpf_send_signal)(__u32 sig) = (void *) 109; * * **-EPROTONOSUPPORT** IP packet version is not 4 or 6 */ -static __s64 (*bpf_tcp_gen_syncookie)(void *sk, void *iph, __u32 iph_len, struct tcphdr *th, __u32 th_len) = (void *) 110; +static __s64 (* const bpf_tcp_gen_syncookie)(void *sk, void *iph, __u32 iph_len, struct tcphdr *th, __u32 th_len) = (void *) 110; /* * bpf_skb_output @@ -2726,7 +2782,7 @@ static __s64 (*bpf_tcp_gen_syncookie)(void *sk, void *iph, __u32 iph_len, struct * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_skb_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 111; +static long (* const bpf_skb_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 111; /* * bpf_probe_read_user @@ -2737,7 +2793,7 @@ static long (*bpf_skb_output)(void *ctx, void *map, __u64 flags, void *data, __u * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_probe_read_user)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 112; +static long (* const bpf_probe_read_user)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 112; /* * bpf_probe_read_kernel @@ -2748,7 +2804,7 @@ static long (*bpf_probe_read_user)(void *dst, __u32 size, const void *unsafe_ptr * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_probe_read_kernel)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 113; +static long (* const bpf_probe_read_kernel)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 113; /* * bpf_probe_read_user_str @@ -2796,7 +2852,7 @@ static long (*bpf_probe_read_kernel)(void *dst, __u32 size, const void *unsafe_p * including the trailing NUL character. On error, a negative * value. */ -static long (*bpf_probe_read_user_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 114; +static long (* const bpf_probe_read_user_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 114; /* * bpf_probe_read_kernel_str @@ -2808,7 +2864,7 @@ static long (*bpf_probe_read_user_str)(void *dst, __u32 size, const void *unsafe * On success, the strictly positive length of the string, including * the trailing NUL character. On error, a negative value. */ -static long (*bpf_probe_read_kernel_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 115; +static long (* const bpf_probe_read_kernel_str)(void *dst, __u32 size, const void *unsafe_ptr) = (void *) 115; /* * bpf_tcp_send_ack @@ -2819,7 +2875,7 @@ static long (*bpf_probe_read_kernel_str)(void *dst, __u32 size, const void *unsa * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_tcp_send_ack)(void *tp, __u32 rcv_nxt) = (void *) 116; +static long (* const bpf_tcp_send_ack)(void *tp, __u32 rcv_nxt) = (void *) 116; /* * bpf_send_signal_thread @@ -2837,7 +2893,7 @@ static long (*bpf_tcp_send_ack)(void *tp, __u32 rcv_nxt) = (void *) 116; * * **-EAGAIN** if bpf program can try again. */ -static long (*bpf_send_signal_thread)(__u32 sig) = (void *) 117; +static long (* const bpf_send_signal_thread)(__u32 sig) = (void *) 117; /* * bpf_jiffies64 @@ -2847,7 +2903,7 @@ static long (*bpf_send_signal_thread)(__u32 sig) = (void *) 117; * Returns * The 64 bit jiffies */ -static __u64 (*bpf_jiffies64)(void) = (void *) 118; +static __u64 (* const bpf_jiffies64)(void) = (void *) 118; /* * bpf_read_branch_records @@ -2870,7 +2926,7 @@ static __u64 (*bpf_jiffies64)(void) = (void *) 118; * * **-ENOENT** if architecture does not support branch records. */ -static long (*bpf_read_branch_records)(struct bpf_perf_event_data *ctx, void *buf, __u32 size, __u64 flags) = (void *) 119; +static long (* const bpf_read_branch_records)(struct bpf_perf_event_data *ctx, void *buf, __u32 size, __u64 flags) = (void *) 119; /* * bpf_get_ns_current_pid_tgid @@ -2886,7 +2942,7 @@ static long (*bpf_read_branch_records)(struct bpf_perf_event_data *ctx, void *bu * * **-ENOENT** if pidns does not exists for the current task. */ -static long (*bpf_get_ns_current_pid_tgid)(__u64 dev, __u64 ino, struct bpf_pidns_info *nsdata, __u32 size) = (void *) 120; +static long (* const bpf_get_ns_current_pid_tgid)(__u64 dev, __u64 ino, struct bpf_pidns_info *nsdata, __u32 size) = (void *) 120; /* * bpf_xdp_output @@ -2914,7 +2970,7 @@ static long (*bpf_get_ns_current_pid_tgid)(__u64 dev, __u64 ino, struct bpf_pidn * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_xdp_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 121; +static long (* const bpf_xdp_output)(void *ctx, void *map, __u64 flags, void *data, __u64 size) = (void *) 121; /* * bpf_get_netns_cookie @@ -2931,7 +2987,7 @@ static long (*bpf_xdp_output)(void *ctx, void *map, __u64 flags, void *data, __u * Returns * A 8-byte long opaque number. */ -static __u64 (*bpf_get_netns_cookie)(void *ctx) = (void *) 122; +static __u64 (* const bpf_get_netns_cookie)(void *ctx) = (void *) 122; /* * bpf_get_current_ancestor_cgroup_id @@ -2953,7 +3009,7 @@ static __u64 (*bpf_get_netns_cookie)(void *ctx) = (void *) 122; * Returns * The id is returned or 0 in case the id could not be retrieved. */ -static __u64 (*bpf_get_current_ancestor_cgroup_id)(int ancestor_level) = (void *) 123; +static __u64 (* const bpf_get_current_ancestor_cgroup_id)(int ancestor_level) = (void *) 123; /* * bpf_sk_assign @@ -2984,11 +3040,8 @@ static __u64 (*bpf_get_current_ancestor_cgroup_id)(int ancestor_level) = (void * * * **-EOPNOTSUPP** if the operation is not supported, for example * a call from outside of TC ingress. - * - * **-ESOCKTNOSUPPORT** if the socket type is not supported - * (reuseport). */ -static long (*bpf_sk_assign)(void *ctx, void *sk, __u64 flags) = (void *) 124; +static long (* const bpf_sk_assign)(void *ctx, void *sk, __u64 flags) = (void *) 124; /* * bpf_ktime_get_boot_ns @@ -3000,7 +3053,7 @@ static long (*bpf_sk_assign)(void *ctx, void *sk, __u64 flags) = (void *) 124; * Returns * Current *ktime*. */ -static __u64 (*bpf_ktime_get_boot_ns)(void) = (void *) 125; +static __u64 (* const bpf_ktime_get_boot_ns)(void) = (void *) 125; /* * bpf_seq_printf @@ -3012,7 +3065,7 @@ static __u64 (*bpf_ktime_get_boot_ns)(void) = (void *) 125; * arguments. The *data* are a **u64** array and corresponding format string * values are stored in the array. For strings and pointers where pointees * are accessed, only the pointer values are stored in the *data* array. - * The *data_len* is the size of *data* in bytes. + * The *data_len* is the size of *data* in bytes - must be a multiple of 8. * * Formats **%s**, **%p{i,I}{4,6}** requires to read kernel memory. * Reading kernel memory may fail due to either invalid address or @@ -3033,7 +3086,7 @@ static __u64 (*bpf_ktime_get_boot_ns)(void) = (void *) 125; * * **-EOVERFLOW** if an overflow happened: The same object will be tried again. */ -static long (*bpf_seq_printf)(struct seq_file *m, const char *fmt, __u32 fmt_size, const void *data, __u32 data_len) = (void *) 126; +static long (* const bpf_seq_printf)(struct seq_file *m, const char *fmt, __u32 fmt_size, const void *data, __u32 data_len) = (void *) 126; /* * bpf_seq_write @@ -3047,7 +3100,7 @@ static long (*bpf_seq_printf)(struct seq_file *m, const char *fmt, __u32 fmt_siz * * **-EOVERFLOW** if an overflow happened: The same object will be tried again. */ -static long (*bpf_seq_write)(struct seq_file *m, const void *data, __u32 len) = (void *) 127; +static long (* const bpf_seq_write)(struct seq_file *m, const void *data, __u32 len) = (void *) 127; /* * bpf_sk_cgroup_id @@ -3065,7 +3118,7 @@ static long (*bpf_seq_write)(struct seq_file *m, const void *data, __u32 len) = * Returns * The id is returned or 0 in case the id could not be retrieved. */ -static __u64 (*bpf_sk_cgroup_id)(void *sk) = (void *) 128; +static __u64 (* const bpf_sk_cgroup_id)(void *sk) = (void *) 128; /* * bpf_sk_ancestor_cgroup_id @@ -3087,7 +3140,7 @@ static __u64 (*bpf_sk_cgroup_id)(void *sk) = (void *) 128; * Returns * The id is returned or 0 in case the id could not be retrieved. */ -static __u64 (*bpf_sk_ancestor_cgroup_id)(void *sk, int ancestor_level) = (void *) 129; +static __u64 (* const bpf_sk_ancestor_cgroup_id)(void *sk, int ancestor_level) = (void *) 129; /* * bpf_ringbuf_output @@ -3108,7 +3161,7 @@ static __u64 (*bpf_sk_ancestor_cgroup_id)(void *sk, int ancestor_level) = (void * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_ringbuf_output)(void *ringbuf, void *data, __u64 size, __u64 flags) = (void *) 130; +static long (* const bpf_ringbuf_output)(void *ringbuf, void *data, __u64 size, __u64 flags) = (void *) 130; /* * bpf_ringbuf_reserve @@ -3120,7 +3173,7 @@ static long (*bpf_ringbuf_output)(void *ringbuf, void *data, __u64 size, __u64 f * Valid pointer with *size* bytes of memory available; NULL, * otherwise. */ -static void *(*bpf_ringbuf_reserve)(void *ringbuf, __u64 size, __u64 flags) = (void *) 131; +static void *(* const bpf_ringbuf_reserve)(void *ringbuf, __u64 size, __u64 flags) = (void *) 131; /* * bpf_ringbuf_submit @@ -3138,7 +3191,7 @@ static void *(*bpf_ringbuf_reserve)(void *ringbuf, __u64 size, __u64 flags) = (v * Returns * Nothing. Always succeeds. */ -static void (*bpf_ringbuf_submit)(void *data, __u64 flags) = (void *) 132; +static void (* const bpf_ringbuf_submit)(void *data, __u64 flags) = (void *) 132; /* * bpf_ringbuf_discard @@ -3156,7 +3209,7 @@ static void (*bpf_ringbuf_submit)(void *data, __u64 flags) = (void *) 132; * Returns * Nothing. Always succeeds. */ -static void (*bpf_ringbuf_discard)(void *data, __u64 flags) = (void *) 133; +static void (* const bpf_ringbuf_discard)(void *data, __u64 flags) = (void *) 133; /* * bpf_ringbuf_query @@ -3177,7 +3230,7 @@ static void (*bpf_ringbuf_discard)(void *data, __u64 flags) = (void *) 133; * Returns * Requested value, or 0, if *flags* are not recognized. */ -static __u64 (*bpf_ringbuf_query)(void *ringbuf, __u64 flags) = (void *) 134; +static __u64 (* const bpf_ringbuf_query)(void *ringbuf, __u64 flags) = (void *) 134; /* * bpf_csum_level @@ -3213,7 +3266,7 @@ static __u64 (*bpf_ringbuf_query)(void *ringbuf, __u64 flags) = (void *) 134; * is returned or the error code -EACCES in case the skb is not * subject to CHECKSUM_UNNECESSARY. */ -static long (*bpf_csum_level)(struct __sk_buff *skb, __u64 level) = (void *) 135; +static long (* const bpf_csum_level)(struct __sk_buff *skb, __u64 level) = (void *) 135; /* * bpf_skc_to_tcp6_sock @@ -3223,7 +3276,7 @@ static long (*bpf_csum_level)(struct __sk_buff *skb, __u64 level) = (void *) 135 * Returns * *sk* if casting is valid, or **NULL** otherwise. */ -static struct tcp6_sock *(*bpf_skc_to_tcp6_sock)(void *sk) = (void *) 136; +static struct tcp6_sock *(* const bpf_skc_to_tcp6_sock)(void *sk) = (void *) 136; /* * bpf_skc_to_tcp_sock @@ -3233,7 +3286,7 @@ static struct tcp6_sock *(*bpf_skc_to_tcp6_sock)(void *sk) = (void *) 136; * Returns * *sk* if casting is valid, or **NULL** otherwise. */ -static struct tcp_sock *(*bpf_skc_to_tcp_sock)(void *sk) = (void *) 137; +static struct tcp_sock *(* const bpf_skc_to_tcp_sock)(void *sk) = (void *) 137; /* * bpf_skc_to_tcp_timewait_sock @@ -3243,7 +3296,7 @@ static struct tcp_sock *(*bpf_skc_to_tcp_sock)(void *sk) = (void *) 137; * Returns * *sk* if casting is valid, or **NULL** otherwise. */ -static struct tcp_timewait_sock *(*bpf_skc_to_tcp_timewait_sock)(void *sk) = (void *) 138; +static struct tcp_timewait_sock *(* const bpf_skc_to_tcp_timewait_sock)(void *sk) = (void *) 138; /* * bpf_skc_to_tcp_request_sock @@ -3253,7 +3306,7 @@ static struct tcp_timewait_sock *(*bpf_skc_to_tcp_timewait_sock)(void *sk) = (vo * Returns * *sk* if casting is valid, or **NULL** otherwise. */ -static struct tcp_request_sock *(*bpf_skc_to_tcp_request_sock)(void *sk) = (void *) 139; +static struct tcp_request_sock *(* const bpf_skc_to_tcp_request_sock)(void *sk) = (void *) 139; /* * bpf_skc_to_udp6_sock @@ -3263,12 +3316,14 @@ static struct tcp_request_sock *(*bpf_skc_to_tcp_request_sock)(void *sk) = (void * Returns * *sk* if casting is valid, or **NULL** otherwise. */ -static struct udp6_sock *(*bpf_skc_to_udp6_sock)(void *sk) = (void *) 140; +static struct udp6_sock *(* const bpf_skc_to_udp6_sock)(void *sk) = (void *) 140; /* * bpf_get_task_stack * * Return a user or a kernel stack in bpf program provided buffer. + * Note: the user stack will only be populated if the *task* is + * the current task; all other tasks will return -EOPNOTSUPP. * To achieve this, the helper needs *task*, which is a valid * pointer to **struct task_struct**. To store the stacktrace, the * bpf program provides *buf* with a nonnegative *size*. @@ -3280,6 +3335,7 @@ static struct udp6_sock *(*bpf_skc_to_udp6_sock)(void *sk) = (void *) 140; * * **BPF_F_USER_STACK** * Collect a user space stack instead of a kernel stack. + * The *task* must be the current task. * **BPF_F_USER_BUILD_ID** * Collect buildid+offset instead of ips for user stack, * only valid if **BPF_F_USER_STACK** is also specified. @@ -3296,10 +3352,10 @@ static struct udp6_sock *(*bpf_skc_to_udp6_sock)(void *sk) = (void *) 140; * # sysctl kernel.perf_event_max_stack= * * Returns - * A non-negative value equal to or less than *size* on success, - * or a negative error in case of failure. + * The non-negative copied *buf* length equal to or less than + * *size* on success, or a negative error in case of failure. */ -static long (*bpf_get_task_stack)(struct task_struct *task, void *buf, __u32 size, __u64 flags) = (void *) 141; +static long (* const bpf_get_task_stack)(struct task_struct *task, void *buf, __u32 size, __u64 flags) = (void *) 141; /* * bpf_load_hdr_opt @@ -3366,7 +3422,7 @@ static long (*bpf_get_task_stack)(struct task_struct *task, void *buf, __u32 siz * **-EPERM** if the helper cannot be used under the current * *skops*\ **->op**. */ -static long (*bpf_load_hdr_opt)(struct bpf_sock_ops *skops, void *searchby_res, __u32 len, __u64 flags) = (void *) 142; +static long (* const bpf_load_hdr_opt)(struct bpf_sock_ops *skops, void *searchby_res, __u32 len, __u64 flags) = (void *) 142; /* * bpf_store_hdr_opt @@ -3398,12 +3454,12 @@ static long (*bpf_load_hdr_opt)(struct bpf_sock_ops *skops, void *searchby_res, * * **-EEXIST** if the option already exists. * - * **-EFAULT** on failrue to parse the existing header options. + * **-EFAULT** on failure to parse the existing header options. * * **-EPERM** if the helper cannot be used under the current * *skops*\ **->op**. */ -static long (*bpf_store_hdr_opt)(struct bpf_sock_ops *skops, const void *from, __u32 len, __u64 flags) = (void *) 143; +static long (* const bpf_store_hdr_opt)(struct bpf_sock_ops *skops, const void *from, __u32 len, __u64 flags) = (void *) 143; /* * bpf_reserve_hdr_opt @@ -3429,7 +3485,7 @@ static long (*bpf_store_hdr_opt)(struct bpf_sock_ops *skops, const void *from, _ * **-EPERM** if the helper cannot be used under the current * *skops*\ **->op**. */ -static long (*bpf_reserve_hdr_opt)(struct bpf_sock_ops *skops, __u32 len, __u64 flags) = (void *) 144; +static long (* const bpf_reserve_hdr_opt)(struct bpf_sock_ops *skops, __u32 len, __u64 flags) = (void *) 144; /* * bpf_inode_storage_get @@ -3461,7 +3517,7 @@ static long (*bpf_reserve_hdr_opt)(struct bpf_sock_ops *skops, __u32 len, __u64 * **NULL** if not found or there was an error in adding * a new bpf_local_storage. */ -static void *(*bpf_inode_storage_get)(void *map, void *inode, void *value, __u64 flags) = (void *) 145; +static void *(* const bpf_inode_storage_get)(void *map, void *inode, void *value, __u64 flags) = (void *) 145; /* * bpf_inode_storage_delete @@ -3473,7 +3529,7 @@ static void *(*bpf_inode_storage_get)(void *map, void *inode, void *value, __u64 * * **-ENOENT** if the bpf_local_storage cannot be found. */ -static int (*bpf_inode_storage_delete)(void *map, void *inode) = (void *) 146; +static int (* const bpf_inode_storage_delete)(void *map, void *inode) = (void *) 146; /* * bpf_d_path @@ -3489,7 +3545,7 @@ static int (*bpf_inode_storage_delete)(void *map, void *inode) = (void *) 146; * including the trailing NUL character. On error, a negative * value. */ -static long (*bpf_d_path)(struct path *path, char *buf, __u32 sz) = (void *) 147; +static long (* const bpf_d_path)(struct path *path, char *buf, __u32 sz) = (void *) 147; /* * bpf_copy_from_user @@ -3500,7 +3556,7 @@ static long (*bpf_d_path)(struct path *path, char *buf, __u32 sz) = (void *) 147 * Returns * 0 on success, or a negative error in case of failure. */ -static long (*bpf_copy_from_user)(void *dst, __u32 size, const void *user_ptr) = (void *) 148; +static long (* const bpf_copy_from_user)(void *dst, __u32 size, const void *user_ptr) = (void *) 148; /* * bpf_snprintf_btf @@ -3540,7 +3596,7 @@ static long (*bpf_copy_from_user)(void *dst, __u32 size, const void *user_ptr) = * written if output had to be truncated due to string size), * or a negative error in cases of failure. */ -static long (*bpf_snprintf_btf)(char *str, __u32 str_size, struct btf_ptr *ptr, __u32 btf_ptr_size, __u64 flags) = (void *) 149; +static long (* const bpf_snprintf_btf)(char *str, __u32 str_size, struct btf_ptr *ptr, __u32 btf_ptr_size, __u64 flags) = (void *) 149; /* * bpf_seq_printf_btf @@ -3552,7 +3608,7 @@ static long (*bpf_snprintf_btf)(char *str, __u32 str_size, struct btf_ptr *ptr, * Returns * 0 on success or a negative error in case of failure. */ -static long (*bpf_seq_printf_btf)(struct seq_file *m, struct btf_ptr *ptr, __u32 ptr_size, __u64 flags) = (void *) 150; +static long (* const bpf_seq_printf_btf)(struct seq_file *m, struct btf_ptr *ptr, __u32 ptr_size, __u64 flags) = (void *) 150; /* * bpf_skb_cgroup_classid @@ -3565,7 +3621,7 @@ static long (*bpf_seq_printf_btf)(struct seq_file *m, struct btf_ptr *ptr, __u32 * Returns * The id is returned or 0 in case the id could not be retrieved. */ -static __u64 (*bpf_skb_cgroup_classid)(struct __sk_buff *skb) = (void *) 151; +static __u64 (* const bpf_skb_cgroup_classid)(struct __sk_buff *skb) = (void *) 151; /* * bpf_redirect_neigh @@ -3590,7 +3646,7 @@ static __u64 (*bpf_skb_cgroup_classid)(struct __sk_buff *skb) = (void *) 151; * The helper returns **TC_ACT_REDIRECT** on success or * **TC_ACT_SHOT** on error. */ -static long (*bpf_redirect_neigh)(__u32 ifindex, struct bpf_redir_neigh *params, int plen, __u64 flags) = (void *) 152; +static long (* const bpf_redirect_neigh)(__u32 ifindex, struct bpf_redir_neigh *params, int plen, __u64 flags) = (void *) 152; /* * bpf_per_cpu_ptr @@ -3611,7 +3667,7 @@ static long (*bpf_redirect_neigh)(__u32 ifindex, struct bpf_redir_neigh *params, * A pointer pointing to the kernel percpu variable on *cpu*, or * NULL, if *cpu* is invalid. */ -static void *(*bpf_per_cpu_ptr)(const void *percpu_ptr, __u32 cpu) = (void *) 153; +static void *(* const bpf_per_cpu_ptr)(const void *percpu_ptr, __u32 cpu) = (void *) 153; /* * bpf_this_cpu_ptr @@ -3627,7 +3683,7 @@ static void *(*bpf_per_cpu_ptr)(const void *percpu_ptr, __u32 cpu) = (void *) 15 * Returns * A pointer pointing to the kernel percpu variable on this cpu. */ -static void *(*bpf_this_cpu_ptr)(const void *percpu_ptr) = (void *) 154; +static void *(* const bpf_this_cpu_ptr)(const void *percpu_ptr) = (void *) 154; /* * bpf_redirect_peer @@ -3639,15 +3695,15 @@ static void *(*bpf_this_cpu_ptr)(const void *percpu_ptr) = (void *) 154; * going through the CPU's backlog queue. * * The *flags* argument is reserved and must be 0. The helper is - * currently only supported for tc BPF program types at the ingress - * hook and for veth device types. The peer device must reside in a - * different network namespace. + * currently only supported for tc BPF program types at the + * ingress hook and for veth and netkit target device types. The + * peer device must reside in a different network namespace. * * Returns * The helper returns **TC_ACT_REDIRECT** on success or * **TC_ACT_SHOT** on error. */ -static long (*bpf_redirect_peer)(__u32 ifindex, __u64 flags) = (void *) 155; +static long (* const bpf_redirect_peer)(__u32 ifindex, __u64 flags) = (void *) 155; /* * bpf_task_storage_get @@ -3658,7 +3714,7 @@ static long (*bpf_redirect_peer)(__u32 ifindex, __u64 flags) = (void *) 155; * a *map* with *task* as the **key**. From this * perspective, the usage is not much different from * **bpf_map_lookup_elem**\ (*map*, **&**\ *task*) except this - * helper enforces the key must be an task_struct and the map must also + * helper enforces the key must be a task_struct and the map must also * be a **BPF_MAP_TYPE_TASK_STORAGE**. * * Underneath, the value is stored locally at *task* instead of @@ -3679,7 +3735,7 @@ static long (*bpf_redirect_peer)(__u32 ifindex, __u64 flags) = (void *) 155; * **NULL** if not found or there was an error in adding * a new bpf_local_storage. */ -static void *(*bpf_task_storage_get)(void *map, struct task_struct *task, void *value, __u64 flags) = (void *) 156; +static void *(* const bpf_task_storage_get)(void *map, struct task_struct *task, void *value, __u64 flags) = (void *) 156; /* * bpf_task_storage_delete @@ -3691,7 +3747,7 @@ static void *(*bpf_task_storage_get)(void *map, struct task_struct *task, void * * * **-ENOENT** if the bpf_local_storage cannot be found. */ -static long (*bpf_task_storage_delete)(void *map, struct task_struct *task) = (void *) 157; +static long (* const bpf_task_storage_delete)(void *map, struct task_struct *task) = (void *) 157; /* * bpf_get_current_task_btf @@ -3703,7 +3759,7 @@ static long (*bpf_task_storage_delete)(void *map, struct task_struct *task) = (v * Returns * Pointer to the current task. */ -static struct task_struct *(*bpf_get_current_task_btf)(void) = (void *) 158; +static struct task_struct *(* const bpf_get_current_task_btf)(void) = (void *) 158; /* * bpf_bprm_opts_set @@ -3717,7 +3773,7 @@ static struct task_struct *(*bpf_get_current_task_btf)(void) = (void *) 158; * Returns * **-EINVAL** if invalid *flags* are passed, zero otherwise. */ -static long (*bpf_bprm_opts_set)(struct linux_binprm *bprm, __u64 flags) = (void *) 159; +static long (* const bpf_bprm_opts_set)(struct linux_binprm *bprm, __u64 flags) = (void *) 159; /* * bpf_ktime_get_coarse_ns @@ -3731,12 +3787,12 @@ static long (*bpf_bprm_opts_set)(struct linux_binprm *bprm, __u64 flags) = (void * Returns * Current *ktime*. */ -static __u64 (*bpf_ktime_get_coarse_ns)(void) = (void *) 160; +static __u64 (* const bpf_ktime_get_coarse_ns)(void) = (void *) 160; /* * bpf_ima_inode_hash * - * Returns the stored IMA hash of the *inode* (if it's avaialable). + * Returns the stored IMA hash of the *inode* (if it's available). * If the hash is larger than *size*, then only *size* * bytes will be copied to *dst* * @@ -3745,7 +3801,7 @@ static __u64 (*bpf_ktime_get_coarse_ns)(void) = (void *) 160; * **-EOPNOTSUP** if IMA is disabled or **-EINVAL** if * invalid arguments are passed. */ -static long (*bpf_ima_inode_hash)(struct inode *inode, void *dst, __u32 size) = (void *) 161; +static long (* const bpf_ima_inode_hash)(struct inode *inode, void *dst, __u32 size) = (void *) 161; /* * bpf_sock_from_file @@ -3757,7 +3813,7 @@ static long (*bpf_ima_inode_hash)(struct inode *inode, void *dst, __u32 size) = * A pointer to a struct socket on success or NULL if the file is * not a socket. */ -static struct socket *(*bpf_sock_from_file)(struct file *file) = (void *) 162; +static struct socket *(* const bpf_sock_from_file)(struct file *file) = (void *) 162; /* * bpf_check_mtu @@ -3768,12 +3824,12 @@ static struct socket *(*bpf_sock_from_file)(struct file *file) = (void *) 162; * * The argument *len_diff* can be used for querying with a planned * size change. This allows to check MTU prior to changing packet - * ctx. Providing an *len_diff* adjustment that is larger than the + * ctx. Providing a *len_diff* adjustment that is larger than the * actual packet size (resulting in negative packet size) will in - * principle not exceed the MTU, why it is not considered a - * failure. Other BPF-helpers are needed for performing the - * planned size change, why the responsability for catch a negative - * packet size belong in those helpers. + * principle not exceed the MTU, which is why it is not considered + * a failure. Other BPF helpers are needed for performing the + * planned size change; therefore the responsibility for catching + * a negative packet size belongs in those helpers. * * Specifying *ifindex* zero means the MTU check is performed * against the current net device. This is practical if this isn't @@ -3828,7 +3884,7 @@ static struct socket *(*bpf_sock_from_file)(struct file *file) = (void *) 162; * * **BPF_MTU_CHK_RET_FRAG_NEEDED** * * **BPF_MTU_CHK_RET_SEGS_TOOBIG** */ -static long (*bpf_check_mtu)(void *ctx, __u32 ifindex, __u32 *mtu_len, __s32 len_diff, __u64 flags) = (void *) 163; +static long (* const bpf_check_mtu)(void *ctx, __u32 ifindex, __u32 *mtu_len, __s32 len_diff, __u64 flags) = (void *) 163; /* * bpf_for_each_map_elem @@ -3861,7 +3917,7 @@ static long (*bpf_check_mtu)(void *ctx, __u32 ifindex, __u32 *mtu_len, __s32 len * The number of traversed map elements for success, **-EINVAL** for * invalid **flags**. */ -static long (*bpf_for_each_map_elem)(void *map, void *callback_fn, void *callback_ctx, __u64 flags) = (void *) 164; +static long (* const bpf_for_each_map_elem)(void *map, void *callback_fn, void *callback_ctx, __u64 flags) = (void *) 164; /* * bpf_snprintf @@ -3873,7 +3929,8 @@ static long (*bpf_for_each_map_elem)(void *map, void *callback_fn, void *callbac * Each format specifier in **fmt** corresponds to one u64 element * in the **data** array. For strings and pointers where pointees * are accessed, only the pointer values are stored in the *data* - * array. The *data_len* is the size of *data* in bytes. + * array. The *data_len* is the size of *data* in bytes - must be + * a multiple of 8. * * Formats **%s** and **%p{i,I}{4,6}** require to read kernel * memory. Reading kernel memory may fail due to either invalid @@ -3892,7 +3949,7 @@ static long (*bpf_for_each_map_elem)(void *map, void *callback_fn, void *callbac * * Or **-EBUSY** if the per-CPU memory copy buffer is busy. */ -static long (*bpf_snprintf)(char *str, __u32 str_size, const char *fmt, __u64 *data, __u32 data_len) = (void *) 165; +static long (* const bpf_snprintf)(char *str, __u32 str_size, const char *fmt, __u64 *data, __u32 data_len) = (void *) 165; /* * bpf_sys_bpf @@ -3902,7 +3959,7 @@ static long (*bpf_snprintf)(char *str, __u32 str_size, const char *fmt, __u64 *d * Returns * A syscall result. */ -static long (*bpf_sys_bpf)(__u32 cmd, void *attr, __u32 attr_size) = (void *) 166; +static long (* const bpf_sys_bpf)(__u32 cmd, void *attr, __u32 attr_size) = (void *) 166; /* * bpf_btf_find_by_name_kind @@ -3912,7 +3969,7 @@ static long (*bpf_sys_bpf)(__u32 cmd, void *attr, __u32 attr_size) = (void *) 16 * Returns * Returns btf_id and btf_obj_fd in lower and upper 32 bits. */ -static long (*bpf_btf_find_by_name_kind)(char *name, int name_sz, __u32 kind, int flags) = (void *) 167; +static long (* const bpf_btf_find_by_name_kind)(char *name, int name_sz, __u32 kind, int flags) = (void *) 167; /* * bpf_sys_close @@ -3922,7 +3979,7 @@ static long (*bpf_btf_find_by_name_kind)(char *name, int name_sz, __u32 kind, in * Returns * A syscall result. */ -static long (*bpf_sys_close)(__u32 fd) = (void *) 168; +static long (* const bpf_sys_close)(__u32 fd) = (void *) 168; /* * bpf_timer_init @@ -3943,7 +4000,7 @@ static long (*bpf_sys_close)(__u32 fd) = (void *) 168; * or pin such map in bpffs. When map is unpinned or file descriptor is * closed all timers in the map will be cancelled and freed. */ -static long (*bpf_timer_init)(struct bpf_timer *timer, void *map, __u64 flags) = (void *) 169; +static long (* const bpf_timer_init)(struct bpf_timer *timer, void *map, __u64 flags) = (void *) 169; /* * bpf_timer_set_callback @@ -3958,7 +4015,7 @@ static long (*bpf_timer_init)(struct bpf_timer *timer, void *map, __u64 flags) = * or pin such map in bpffs. When map is unpinned or file descriptor is * closed all timers in the map will be cancelled and freed. */ -static long (*bpf_timer_set_callback)(struct bpf_timer *timer, void *callback_fn) = (void *) 170; +static long (* const bpf_timer_set_callback)(struct bpf_timer *timer, void *callback_fn) = (void *) 170; /* * bpf_timer_start @@ -3982,13 +4039,21 @@ static long (*bpf_timer_set_callback)(struct bpf_timer *timer, void *callback_fn * different maps if key/value layout matches across maps. * Every bpf_timer_set_callback() can have different callback_fn. * + * *flags* can be one of: + * + * **BPF_F_TIMER_ABS** + * Start the timer in absolute expire value instead of the + * default relative one. + * **BPF_F_TIMER_CPU_PIN** + * Timer will be pinned to the CPU of the caller. + * * * Returns * 0 on success. * **-EINVAL** if *timer* was not initialized with bpf_timer_init() earlier * or invalid *flags* are passed. */ -static long (*bpf_timer_start)(struct bpf_timer *timer, __u64 nsecs, __u64 flags) = (void *) 171; +static long (* const bpf_timer_start)(struct bpf_timer *timer, __u64 nsecs, __u64 flags) = (void *) 171; /* * bpf_timer_cancel @@ -4002,17 +4067,23 @@ static long (*bpf_timer_start)(struct bpf_timer *timer, __u64 nsecs, __u64 flags * **-EDEADLK** if callback_fn tried to call bpf_timer_cancel() on its * own timer which would have led to a deadlock otherwise. */ -static long (*bpf_timer_cancel)(struct bpf_timer *timer) = (void *) 172; +static long (* const bpf_timer_cancel)(struct bpf_timer *timer) = (void *) 172; /* * bpf_get_func_ip * * Get address of the traced function (for tracing and kprobe programs). * + * When called for kprobe program attached as uprobe it returns + * probe address for both entry and return uprobe. + * + * * Returns - * Address of the traced function. + * Address of the traced function for kprobe. + * 0 for kprobes placed within the function (not at the entry). + * Address of the probe for uprobe and return uprobe. */ -static __u64 (*bpf_get_func_ip)(void *ctx) = (void *) 173; +static __u64 (* const bpf_get_func_ip)(void *ctx) = (void *) 173; /* * bpf_get_attach_cookie @@ -4031,7 +4102,7 @@ static __u64 (*bpf_get_func_ip)(void *ctx) = (void *) 173; * Value specified by user at BPF link creation/attachment time * or 0, if it was not specified. */ -static __u64 (*bpf_get_attach_cookie)(void *ctx) = (void *) 174; +static __u64 (* const bpf_get_attach_cookie)(void *ctx) = (void *) 174; /* * bpf_task_pt_regs @@ -4041,6 +4112,655 @@ static __u64 (*bpf_get_attach_cookie)(void *ctx) = (void *) 174; * Returns * A pointer to struct pt_regs. */ -static long (*bpf_task_pt_regs)(struct task_struct *task) = (void *) 175; +static long (* const bpf_task_pt_regs)(struct task_struct *task) = (void *) 175; + +/* + * bpf_get_branch_snapshot + * + * Get branch trace from hardware engines like Intel LBR. The + * hardware engine is stopped shortly after the helper is + * called. Therefore, the user need to filter branch entries + * based on the actual use case. To capture branch trace + * before the trigger point of the BPF program, the helper + * should be called at the beginning of the BPF program. + * + * The data is stored as struct perf_branch_entry into output + * buffer *entries*. *size* is the size of *entries* in bytes. + * *flags* is reserved for now and must be zero. + * + * + * Returns + * On success, number of bytes written to *buf*. On error, a + * negative value. + * + * **-EINVAL** if *flags* is not zero. + * + * **-ENOENT** if architecture does not support branch records. + */ +static long (* const bpf_get_branch_snapshot)(void *entries, __u32 size, __u64 flags) = (void *) 176; + +/* + * bpf_trace_vprintk + * + * Behaves like **bpf_trace_printk**\ () helper, but takes an array of u64 + * to format and can handle more format args as a result. + * + * Arguments are to be used as in **bpf_seq_printf**\ () helper. + * + * Returns + * The number of bytes written to the buffer, or a negative error + * in case of failure. + */ +static long (* const bpf_trace_vprintk)(const char *fmt, __u32 fmt_size, const void *data, __u32 data_len) = (void *) 177; + +/* + * bpf_skc_to_unix_sock + * + * Dynamically cast a *sk* pointer to a *unix_sock* pointer. + * + * Returns + * *sk* if casting is valid, or **NULL** otherwise. + */ +static struct unix_sock *(* const bpf_skc_to_unix_sock)(void *sk) = (void *) 178; + +/* + * bpf_kallsyms_lookup_name + * + * Get the address of a kernel symbol, returned in *res*. *res* is + * set to 0 if the symbol is not found. + * + * Returns + * On success, zero. On error, a negative value. + * + * **-EINVAL** if *flags* is not zero. + * + * **-EINVAL** if string *name* is not the same size as *name_sz*. + * + * **-ENOENT** if symbol is not found. + * + * **-EPERM** if caller does not have permission to obtain kernel address. + */ +static long (* const bpf_kallsyms_lookup_name)(const char *name, int name_sz, int flags, __u64 *res) = (void *) 179; + +/* + * bpf_find_vma + * + * Find vma of *task* that contains *addr*, call *callback_fn* + * function with *task*, *vma*, and *callback_ctx*. + * The *callback_fn* should be a static function and + * the *callback_ctx* should be a pointer to the stack. + * The *flags* is used to control certain aspects of the helper. + * Currently, the *flags* must be 0. + * + * The expected callback signature is + * + * long (\*callback_fn)(struct task_struct \*task, struct vm_area_struct \*vma, void \*callback_ctx); + * + * + * Returns + * 0 on success. + * **-ENOENT** if *task->mm* is NULL, or no vma contains *addr*. + * **-EBUSY** if failed to try lock mmap_lock. + * **-EINVAL** for invalid **flags**. + */ +static long (* const bpf_find_vma)(struct task_struct *task, __u64 addr, void *callback_fn, void *callback_ctx, __u64 flags) = (void *) 180; + +/* + * bpf_loop + * + * For **nr_loops**, call **callback_fn** function + * with **callback_ctx** as the context parameter. + * The **callback_fn** should be a static function and + * the **callback_ctx** should be a pointer to the stack. + * The **flags** is used to control certain aspects of the helper. + * Currently, the **flags** must be 0. Currently, nr_loops is + * limited to 1 << 23 (~8 million) loops. + * + * long (\*callback_fn)(u32 index, void \*ctx); + * + * where **index** is the current index in the loop. The index + * is zero-indexed. + * + * If **callback_fn** returns 0, the helper will continue to the next + * loop. If return value is 1, the helper will skip the rest of + * the loops and return. Other return values are not used now, + * and will be rejected by the verifier. + * + * + * Returns + * The number of loops performed, **-EINVAL** for invalid **flags**, + * **-E2BIG** if **nr_loops** exceeds the maximum number of loops. + */ +static long (* const bpf_loop)(__u32 nr_loops, void *callback_fn, void *callback_ctx, __u64 flags) = (void *) 181; + +/* + * bpf_strncmp + * + * Do strncmp() between **s1** and **s2**. **s1** doesn't need + * to be null-terminated and **s1_sz** is the maximum storage + * size of **s1**. **s2** must be a read-only string. + * + * Returns + * An integer less than, equal to, or greater than zero + * if the first **s1_sz** bytes of **s1** is found to be + * less than, to match, or be greater than **s2**. + */ +static long (* const bpf_strncmp)(const char *s1, __u32 s1_sz, const char *s2) = (void *) 182; + +/* + * bpf_get_func_arg + * + * Get **n**-th argument register (zero based) of the traced function (for tracing programs) + * returned in **value**. + * + * + * Returns + * 0 on success. + * **-EINVAL** if n >= argument register count of traced function. + */ +static long (* const bpf_get_func_arg)(void *ctx, __u32 n, __u64 *value) = (void *) 183; + +/* + * bpf_get_func_ret + * + * Get return value of the traced function (for tracing programs) + * in **value**. + * + * + * Returns + * 0 on success. + * **-EOPNOTSUPP** for tracing programs other than BPF_TRACE_FEXIT or BPF_MODIFY_RETURN. + */ +static long (* const bpf_get_func_ret)(void *ctx, __u64 *value) = (void *) 184; + +/* + * bpf_get_func_arg_cnt + * + * Get number of registers of the traced function (for tracing programs) where + * function arguments are stored in these registers. + * + * + * Returns + * The number of argument registers of the traced function. + */ +static long (* const bpf_get_func_arg_cnt)(void *ctx) = (void *) 185; + +/* + * bpf_get_retval + * + * Get the BPF program's return value that will be returned to the upper layers. + * + * This helper is currently supported by cgroup programs and only by the hooks + * where BPF program's return value is returned to the userspace via errno. + * + * Returns + * The BPF program's return value. + */ +static int (* const bpf_get_retval)(void) = (void *) 186; + +/* + * bpf_set_retval + * + * Set the BPF program's return value that will be returned to the upper layers. + * + * This helper is currently supported by cgroup programs and only by the hooks + * where BPF program's return value is returned to the userspace via errno. + * + * Note that there is the following corner case where the program exports an error + * via bpf_set_retval but signals success via 'return 1': + * + * bpf_set_retval(-EPERM); + * return 1; + * + * In this case, the BPF program's return value will use helper's -EPERM. This + * still holds true for cgroup/bind{4,6} which supports extra 'return 3' success case. + * + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static int (* const bpf_set_retval)(int retval) = (void *) 187; + +/* + * bpf_xdp_get_buff_len + * + * Get the total size of a given xdp buff (linear and paged area) + * + * Returns + * The total size of a given xdp buffer. + */ +static __u64 (* const bpf_xdp_get_buff_len)(struct xdp_md *xdp_md) = (void *) 188; + +/* + * bpf_xdp_load_bytes + * + * This helper is provided as an easy way to load data from a + * xdp buffer. It can be used to load *len* bytes from *offset* from + * the frame associated to *xdp_md*, into the buffer pointed by + * *buf*. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (* const bpf_xdp_load_bytes)(struct xdp_md *xdp_md, __u32 offset, void *buf, __u32 len) = (void *) 189; + +/* + * bpf_xdp_store_bytes + * + * Store *len* bytes from buffer *buf* into the frame + * associated to *xdp_md*, at *offset*. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (* const bpf_xdp_store_bytes)(struct xdp_md *xdp_md, __u32 offset, void *buf, __u32 len) = (void *) 190; + +/* + * bpf_copy_from_user_task + * + * Read *size* bytes from user space address *user_ptr* in *tsk*'s + * address space, and stores the data in *dst*. *flags* is not + * used yet and is provided for future extensibility. This helper + * can only be used by sleepable programs. + * + * Returns + * 0 on success, or a negative error in case of failure. On error + * *dst* buffer is zeroed out. + */ +static long (* const bpf_copy_from_user_task)(void *dst, __u32 size, const void *user_ptr, struct task_struct *tsk, __u64 flags) = (void *) 191; + +/* + * bpf_skb_set_tstamp + * + * Change the __sk_buff->tstamp_type to *tstamp_type* + * and set *tstamp* to the __sk_buff->tstamp together. + * + * If there is no need to change the __sk_buff->tstamp_type, + * the tstamp value can be directly written to __sk_buff->tstamp + * instead. + * + * BPF_SKB_TSTAMP_DELIVERY_MONO is the only tstamp that + * will be kept during bpf_redirect_*(). A non zero + * *tstamp* must be used with the BPF_SKB_TSTAMP_DELIVERY_MONO + * *tstamp_type*. + * + * A BPF_SKB_TSTAMP_UNSPEC *tstamp_type* can only be used + * with a zero *tstamp*. + * + * Only IPv4 and IPv6 skb->protocol are supported. + * + * This function is most useful when it needs to set a + * mono delivery time to __sk_buff->tstamp and then + * bpf_redirect_*() to the egress of an iface. For example, + * changing the (rcv) timestamp in __sk_buff->tstamp at + * ingress to a mono delivery time and then bpf_redirect_*() + * to sch_fq@phy-dev. + * + * Returns + * 0 on success. + * **-EINVAL** for invalid input + * **-EOPNOTSUPP** for unsupported protocol + */ +static long (* const bpf_skb_set_tstamp)(struct __sk_buff *skb, __u64 tstamp, __u32 tstamp_type) = (void *) 192; + +/* + * bpf_ima_file_hash + * + * Returns a calculated IMA hash of the *file*. + * If the hash is larger than *size*, then only *size* + * bytes will be copied to *dst* + * + * Returns + * The **hash_algo** is returned on success, + * **-EOPNOTSUP** if the hash calculation failed or **-EINVAL** if + * invalid arguments are passed. + */ +static long (* const bpf_ima_file_hash)(struct file *file, void *dst, __u32 size) = (void *) 193; + +/* + * bpf_kptr_xchg + * + * Exchange kptr at pointer *map_value* with *ptr*, and return the + * old value. *ptr* can be NULL, otherwise it must be a referenced + * pointer which will be released when this helper is called. + * + * Returns + * The old value of kptr (which can be NULL). The returned pointer + * if not NULL, is a reference which must be released using its + * corresponding release function, or moved into a BPF map before + * program exit. + */ +static void *(* const bpf_kptr_xchg)(void *map_value, void *ptr) = (void *) 194; + +/* + * bpf_map_lookup_percpu_elem + * + * Perform a lookup in *percpu map* for an entry associated to + * *key* on *cpu*. + * + * Returns + * Map value associated to *key* on *cpu*, or **NULL** if no entry + * was found or *cpu* is invalid. + */ +static void *(* const bpf_map_lookup_percpu_elem)(void *map, const void *key, __u32 cpu) = (void *) 195; + +/* + * bpf_skc_to_mptcp_sock + * + * Dynamically cast a *sk* pointer to a *mptcp_sock* pointer. + * + * Returns + * *sk* if casting is valid, or **NULL** otherwise. + */ +static struct mptcp_sock *(* const bpf_skc_to_mptcp_sock)(void *sk) = (void *) 196; + +/* + * bpf_dynptr_from_mem + * + * Get a dynptr to local memory *data*. + * + * *data* must be a ptr to a map value. + * The maximum *size* supported is DYNPTR_MAX_SIZE. + * *flags* is currently unused. + * + * Returns + * 0 on success, -E2BIG if the size exceeds DYNPTR_MAX_SIZE, + * -EINVAL if flags is not 0. + */ +static long (* const bpf_dynptr_from_mem)(void *data, __u32 size, __u64 flags, struct bpf_dynptr *ptr) = (void *) 197; + +/* + * bpf_ringbuf_reserve_dynptr + * + * Reserve *size* bytes of payload in a ring buffer *ringbuf* + * through the dynptr interface. *flags* must be 0. + * + * Please note that a corresponding bpf_ringbuf_submit_dynptr or + * bpf_ringbuf_discard_dynptr must be called on *ptr*, even if the + * reservation fails. This is enforced by the verifier. + * + * Returns + * 0 on success, or a negative error in case of failure. + */ +static long (* const bpf_ringbuf_reserve_dynptr)(void *ringbuf, __u32 size, __u64 flags, struct bpf_dynptr *ptr) = (void *) 198; + +/* + * bpf_ringbuf_submit_dynptr + * + * Submit reserved ring buffer sample, pointed to by *data*, + * through the dynptr interface. This is a no-op if the dynptr is + * invalid/null. + * + * For more information on *flags*, please see + * 'bpf_ringbuf_submit'. + * + * Returns + * Nothing. Always succeeds. + */ +static void (* const bpf_ringbuf_submit_dynptr)(struct bpf_dynptr *ptr, __u64 flags) = (void *) 199; + +/* + * bpf_ringbuf_discard_dynptr + * + * Discard reserved ring buffer sample through the dynptr + * interface. This is a no-op if the dynptr is invalid/null. + * + * For more information on *flags*, please see + * 'bpf_ringbuf_discard'. + * + * Returns + * Nothing. Always succeeds. + */ +static void (* const bpf_ringbuf_discard_dynptr)(struct bpf_dynptr *ptr, __u64 flags) = (void *) 200; + +/* + * bpf_dynptr_read + * + * Read *len* bytes from *src* into *dst*, starting from *offset* + * into *src*. + * *flags* is currently unused. + * + * Returns + * 0 on success, -E2BIG if *offset* + *len* exceeds the length + * of *src*'s data, -EINVAL if *src* is an invalid dynptr or if + * *flags* is not 0. + */ +static long (* const bpf_dynptr_read)(void *dst, __u32 len, const struct bpf_dynptr *src, __u32 offset, __u64 flags) = (void *) 201; + +/* + * bpf_dynptr_write + * + * Write *len* bytes from *src* into *dst*, starting from *offset* + * into *dst*. + * + * *flags* must be 0 except for skb-type dynptrs. + * + * For skb-type dynptrs: + * * All data slices of the dynptr are automatically + * invalidated after **bpf_dynptr_write**\ (). This is + * because writing may pull the skb and change the + * underlying packet buffer. + * + * * For *flags*, please see the flags accepted by + * **bpf_skb_store_bytes**\ (). + * + * Returns + * 0 on success, -E2BIG if *offset* + *len* exceeds the length + * of *dst*'s data, -EINVAL if *dst* is an invalid dynptr or if *dst* + * is a read-only dynptr or if *flags* is not correct. For skb-type dynptrs, + * other errors correspond to errors returned by **bpf_skb_store_bytes**\ (). + */ +static long (* const bpf_dynptr_write)(const struct bpf_dynptr *dst, __u32 offset, void *src, __u32 len, __u64 flags) = (void *) 202; + +/* + * bpf_dynptr_data + * + * Get a pointer to the underlying dynptr data. + * + * *len* must be a statically known value. The returned data slice + * is invalidated whenever the dynptr is invalidated. + * + * skb and xdp type dynptrs may not use bpf_dynptr_data. They should + * instead use bpf_dynptr_slice and bpf_dynptr_slice_rdwr. + * + * Returns + * Pointer to the underlying dynptr data, NULL if the dynptr is + * read-only, if the dynptr is invalid, or if the offset and length + * is out of bounds. + */ +static void *(* const bpf_dynptr_data)(const struct bpf_dynptr *ptr, __u32 offset, __u32 len) = (void *) 203; + +/* + * bpf_tcp_raw_gen_syncookie_ipv4 + * + * Try to issue a SYN cookie for the packet with corresponding + * IPv4/TCP headers, *iph* and *th*, without depending on a + * listening socket. + * + * *iph* points to the IPv4 header. + * + * *th* points to the start of the TCP header, while *th_len* + * contains the length of the TCP header (at least + * **sizeof**\ (**struct tcphdr**)). + * + * Returns + * On success, lower 32 bits hold the generated SYN cookie in + * followed by 16 bits which hold the MSS value for that cookie, + * and the top 16 bits are unused. + * + * On failure, the returned value is one of the following: + * + * **-EINVAL** if *th_len* is invalid. + */ +static __s64 (* const bpf_tcp_raw_gen_syncookie_ipv4)(struct iphdr *iph, struct tcphdr *th, __u32 th_len) = (void *) 204; + +/* + * bpf_tcp_raw_gen_syncookie_ipv6 + * + * Try to issue a SYN cookie for the packet with corresponding + * IPv6/TCP headers, *iph* and *th*, without depending on a + * listening socket. + * + * *iph* points to the IPv6 header. + * + * *th* points to the start of the TCP header, while *th_len* + * contains the length of the TCP header (at least + * **sizeof**\ (**struct tcphdr**)). + * + * Returns + * On success, lower 32 bits hold the generated SYN cookie in + * followed by 16 bits which hold the MSS value for that cookie, + * and the top 16 bits are unused. + * + * On failure, the returned value is one of the following: + * + * **-EINVAL** if *th_len* is invalid. + * + * **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin. + */ +static __s64 (* const bpf_tcp_raw_gen_syncookie_ipv6)(struct ipv6hdr *iph, struct tcphdr *th, __u32 th_len) = (void *) 205; + +/* + * bpf_tcp_raw_check_syncookie_ipv4 + * + * Check whether *iph* and *th* contain a valid SYN cookie ACK + * without depending on a listening socket. + * + * *iph* points to the IPv4 header. + * + * *th* points to the TCP header. + * + * Returns + * 0 if *iph* and *th* are a valid SYN cookie ACK. + * + * On failure, the returned value is one of the following: + * + * **-EACCES** if the SYN cookie is not valid. + */ +static long (* const bpf_tcp_raw_check_syncookie_ipv4)(struct iphdr *iph, struct tcphdr *th) = (void *) 206; + +/* + * bpf_tcp_raw_check_syncookie_ipv6 + * + * Check whether *iph* and *th* contain a valid SYN cookie ACK + * without depending on a listening socket. + * + * *iph* points to the IPv6 header. + * + * *th* points to the TCP header. + * + * Returns + * 0 if *iph* and *th* are a valid SYN cookie ACK. + * + * On failure, the returned value is one of the following: + * + * **-EACCES** if the SYN cookie is not valid. + * + * **-EPROTONOSUPPORT** if CONFIG_IPV6 is not builtin. + */ +static long (* const bpf_tcp_raw_check_syncookie_ipv6)(struct ipv6hdr *iph, struct tcphdr *th) = (void *) 207; + +/* + * bpf_ktime_get_tai_ns + * + * A nonsettable system-wide clock derived from wall-clock time but + * ignoring leap seconds. This clock does not experience + * discontinuities and backwards jumps caused by NTP inserting leap + * seconds as CLOCK_REALTIME does. + * + * See: **clock_gettime**\ (**CLOCK_TAI**) + * + * Returns + * Current *ktime*. + */ +static __u64 (* const bpf_ktime_get_tai_ns)(void) = (void *) 208; + +/* + * bpf_user_ringbuf_drain + * + * Drain samples from the specified user ring buffer, and invoke + * the provided callback for each such sample: + * + * long (\*callback_fn)(const struct bpf_dynptr \*dynptr, void \*ctx); + * + * If **callback_fn** returns 0, the helper will continue to try + * and drain the next sample, up to a maximum of + * BPF_MAX_USER_RINGBUF_SAMPLES samples. If the return value is 1, + * the helper will skip the rest of the samples and return. Other + * return values are not used now, and will be rejected by the + * verifier. + * + * Returns + * The number of drained samples if no error was encountered while + * draining samples, or 0 if no samples were present in the ring + * buffer. If a user-space producer was epoll-waiting on this map, + * and at least one sample was drained, they will receive an event + * notification notifying them of available space in the ring + * buffer. If the BPF_RB_NO_WAKEUP flag is passed to this + * function, no wakeup notification will be sent. If the + * BPF_RB_FORCE_WAKEUP flag is passed, a wakeup notification will + * be sent even if no sample was drained. + * + * On failure, the returned value is one of the following: + * + * **-EBUSY** if the ring buffer is contended, and another calling + * context was concurrently draining the ring buffer. + * + * **-EINVAL** if user-space is not properly tracking the ring + * buffer due to the producer position not being aligned to 8 + * bytes, a sample not being aligned to 8 bytes, or the producer + * position not matching the advertised length of a sample. + * + * **-E2BIG** if user-space has tried to publish a sample which is + * larger than the size of the ring buffer, or which cannot fit + * within a struct bpf_dynptr. + */ +static long (* const bpf_user_ringbuf_drain)(void *map, void *callback_fn, void *ctx, __u64 flags) = (void *) 209; + +/* + * bpf_cgrp_storage_get + * + * Get a bpf_local_storage from the *cgroup*. + * + * Logically, it could be thought of as getting the value from + * a *map* with *cgroup* as the **key**. From this + * perspective, the usage is not much different from + * **bpf_map_lookup_elem**\ (*map*, **&**\ *cgroup*) except this + * helper enforces the key must be a cgroup struct and the map must also + * be a **BPF_MAP_TYPE_CGRP_STORAGE**. + * + * In reality, the local-storage value is embedded directly inside of the + * *cgroup* object itself, rather than being located in the + * **BPF_MAP_TYPE_CGRP_STORAGE** map. When the local-storage value is + * queried for some *map* on a *cgroup* object, the kernel will perform an + * O(n) iteration over all of the live local-storage values for that + * *cgroup* object until the local-storage value for the *map* is found. + * + * An optional *flags* (**BPF_LOCAL_STORAGE_GET_F_CREATE**) can be + * used such that a new bpf_local_storage will be + * created if one does not exist. *value* can be used + * together with **BPF_LOCAL_STORAGE_GET_F_CREATE** to specify + * the initial value of a bpf_local_storage. If *value* is + * **NULL**, the new bpf_local_storage will be zero initialized. + * + * Returns + * A bpf_local_storage pointer is returned on success. + * + * **NULL** if not found or there was an error in adding + * a new bpf_local_storage. + */ +static void *(* const bpf_cgrp_storage_get)(void *map, struct cgroup *cgroup, void *value, __u64 flags) = (void *) 210; + +/* + * bpf_cgrp_storage_delete + * + * Delete a bpf_local_storage from a *cgroup*. + * + * Returns + * 0 on success. + * + * **-ENOENT** if the bpf_local_storage cannot be found. + */ +static long (* const bpf_cgrp_storage_delete)(void *map, struct cgroup *cgroup) = (void *) 211; diff --git a/internal/include/libbpf/bpf_helpers.h b/internal/include/libbpf/bpf_helpers.h index b9987c3ef..cd17f6d07 100644 --- a/internal/include/libbpf/bpf_helpers.h +++ b/internal/include/libbpf/bpf_helpers.h @@ -13,14 +13,7 @@ #define __uint(name, val) int (*name)[val] #define __type(name, val) typeof(val) *name #define __array(name, val) typeof(val) *name[] - -/* Helper macro to print out debug messages */ -#define bpf_printk(fmt, ...) \ -({ \ - char ____fmt[] = fmt; \ - bpf_trace_printk(____fmt, sizeof(____fmt), \ - ##__VA_ARGS__); \ -}) +#define __ulong(name, val) enum { ___bpf_concat(__unique_value, __COUNTER__) = val } name /* * Helper macro to place programs, maps, license in @@ -30,12 +23,25 @@ * To allow use of SEC() with externs (e.g., for extern .maps declarations), * make sure __attribute__((unused)) doesn't trigger compilation warning. */ +#if __GNUC__ && !__clang__ + +/* + * Pragma macros are broken on GCC + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=55578 + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=90400 + */ +#define SEC(name) __attribute__((section(name), used)) + +#else + #define SEC(name) \ _Pragma("GCC diagnostic push") \ _Pragma("GCC diagnostic ignored \"-Wignored-attributes\"") \ __attribute__((section(name), used)) \ _Pragma("GCC diagnostic pop") \ +#endif + /* Avoid 'linux/stddef.h' definition of '__always_inline'. */ #undef __always_inline #define __always_inline inline __attribute__((always_inline)) @@ -72,15 +78,44 @@ /* * Helper macros to manipulate data structures */ -#ifndef offsetof -#define offsetof(TYPE, MEMBER) ((unsigned long)&((TYPE *)0)->MEMBER) -#endif -#ifndef container_of + +/* offsetof() definition that uses __builtin_offset() might not preserve field + * offset CO-RE relocation properly, so force-redefine offsetof() using + * old-school approach which works with CO-RE correctly + */ +#undef offsetof +#define offsetof(type, member) ((unsigned long)&((type *)0)->member) + +/* redefined container_of() to ensure we use the above offsetof() macro */ +#undef container_of #define container_of(ptr, type, member) \ ({ \ void *__mptr = (void *)(ptr); \ ((type *)(__mptr - offsetof(type, member))); \ }) + +/* + * Compiler (optimization) barrier. + */ +#ifndef barrier +#define barrier() asm volatile("" ::: "memory") +#endif + +/* Variable-specific compiler (optimization) barrier. It's a no-op which makes + * compiler believe that there is some black box modification of a given + * variable and thus prevents compiler from making extra assumption about its + * value and potential simplifications and optimizations on this variable. + * + * E.g., compiler might often delay or even omit 32-bit to 64-bit casting of + * a variable, making some code patterns unverifiable. Putting barrier_var() + * in place will ensure that cast is performed before the barrier_var() + * invocation, because compiler has to pessimistically assume that embedded + * asm section might perform some extra operations on that variable. + * + * This is a variable-specific variant of more global barrier(). + */ +#ifndef barrier_var +#define barrier_var(var) asm volatile("" : "+r"(var)) #endif /* @@ -131,18 +166,6 @@ bpf_tail_call_static(void *ctx, const void *map, const __u32 slot) } #endif -/* - * Helper structure used by eBPF C program - * to describe BPF map attributes to libbpf loader - */ -struct bpf_map_def { - unsigned int type; - unsigned int key_size; - unsigned int value_size; - unsigned int max_entries; - unsigned int map_flags; -}; - enum libbpf_pin_type { LIBBPF_PIN_NONE, /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ @@ -157,6 +180,20 @@ enum libbpf_tristate { #define __kconfig __attribute__((section(".kconfig"))) #define __ksym __attribute__((section(".ksyms"))) +#define __kptr_untrusted __attribute__((btf_type_tag("kptr_untrusted"))) +#define __kptr __attribute__((btf_type_tag("kptr"))) +#define __percpu_kptr __attribute__((btf_type_tag("percpu_kptr"))) + +#define bpf_ksym_exists(sym) ({ \ + _Static_assert(!__builtin_constant_p(!!sym), #sym " should be marked as __weak"); \ + !!sym; \ +}) + +#define __arg_ctx __attribute__((btf_decl_tag("arg:ctx"))) +#define __arg_nonnull __attribute((btf_decl_tag("arg:nonnull"))) +#define __arg_nullable __attribute((btf_decl_tag("arg:nullable"))) +#define __arg_trusted __attribute((btf_decl_tag("arg:trusted"))) +#define __arg_arena __attribute((btf_decl_tag("arg:arena"))) #ifndef ___bpf_concat #define ___bpf_concat(a, b) a ## b @@ -224,4 +261,150 @@ enum libbpf_tristate { ___param, sizeof(___param)); \ }) +#ifdef BPF_NO_GLOBAL_DATA +#define BPF_PRINTK_FMT_MOD +#else +#define BPF_PRINTK_FMT_MOD static const +#endif + +#define __bpf_printk(fmt, ...) \ +({ \ + BPF_PRINTK_FMT_MOD char ____fmt[] = fmt; \ + bpf_trace_printk(____fmt, sizeof(____fmt), \ + ##__VA_ARGS__); \ +}) + +/* + * __bpf_vprintk wraps the bpf_trace_vprintk helper with variadic arguments + * instead of an array of u64. + */ +#define __bpf_vprintk(fmt, args...) \ +({ \ + static const char ___fmt[] = fmt; \ + unsigned long long ___param[___bpf_narg(args)]; \ + \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ + ___bpf_fill(___param, args); \ + _Pragma("GCC diagnostic pop") \ + \ + bpf_trace_vprintk(___fmt, sizeof(___fmt), \ + ___param, sizeof(___param)); \ +}) + +/* Use __bpf_printk when bpf_printk call has 3 or fewer fmt args + * Otherwise use __bpf_vprintk + */ +#define ___bpf_pick_printk(...) \ + ___bpf_nth(_, ##__VA_ARGS__, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, \ + __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, __bpf_vprintk, \ + __bpf_vprintk, __bpf_vprintk, __bpf_printk /*3*/, __bpf_printk /*2*/,\ + __bpf_printk /*1*/, __bpf_printk /*0*/) + +/* Helper macro to print out debug messages */ +#define bpf_printk(fmt, args...) ___bpf_pick_printk(args)(fmt, ##args) + +struct bpf_iter_num; + +extern int bpf_iter_num_new(struct bpf_iter_num *it, int start, int end) __weak __ksym; +extern int *bpf_iter_num_next(struct bpf_iter_num *it) __weak __ksym; +extern void bpf_iter_num_destroy(struct bpf_iter_num *it) __weak __ksym; + +#ifndef bpf_for_each +/* bpf_for_each(iter_type, cur_elem, args...) provides generic construct for + * using BPF open-coded iterators without having to write mundane explicit + * low-level loop logic. Instead, it provides for()-like generic construct + * that can be used pretty naturally. E.g., for some hypothetical cgroup + * iterator, you'd write: + * + * struct cgroup *cg, *parent_cg = <...>; + * + * bpf_for_each(cgroup, cg, parent_cg, CG_ITER_CHILDREN) { + * bpf_printk("Child cgroup id = %d", cg->cgroup_id); + * if (cg->cgroup_id == 123) + * break; + * } + * + * I.e., it looks almost like high-level for each loop in other languages, + * supports continue/break, and is verifiable by BPF verifier. + * + * For iterating integers, the difference betwen bpf_for_each(num, i, N, M) + * and bpf_for(i, N, M) is in that bpf_for() provides additional proof to + * verifier that i is in [N, M) range, and in bpf_for_each() case i is `int + * *`, not just `int`. So for integers bpf_for() is more convenient. + * + * Note: this macro relies on C99 feature of allowing to declare variables + * inside for() loop, bound to for() loop lifetime. It also utilizes GCC + * extension: __attribute__((cleanup())), supported by both GCC and + * Clang. + */ +#define bpf_for_each(type, cur, args...) for ( \ + /* initialize and define destructor */ \ + struct bpf_iter_##type ___it __attribute__((aligned(8), /* enforce, just in case */, \ + cleanup(bpf_iter_##type##_destroy))), \ + /* ___p pointer is just to call bpf_iter_##type##_new() *once* to init ___it */ \ + *___p __attribute__((unused)) = ( \ + bpf_iter_##type##_new(&___it, ##args), \ + /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ + /* for bpf_iter_##type##_destroy() when used from cleanup() attribute */ \ + (void)bpf_iter_##type##_destroy, (void *)0); \ + /* iteration and termination check */ \ + (((cur) = bpf_iter_##type##_next(&___it))); \ +) +#endif /* bpf_for_each */ + +#ifndef bpf_for +/* bpf_for(i, start, end) implements a for()-like looping construct that sets + * provided integer variable *i* to values starting from *start* through, + * but not including, *end*. It also proves to BPF verifier that *i* belongs + * to range [start, end), so this can be used for accessing arrays without + * extra checks. + * + * Note: *start* and *end* are assumed to be expressions with no side effects + * and whose values do not change throughout bpf_for() loop execution. They do + * not have to be statically known or constant, though. + * + * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() + * loop bound variables and cleanup attribute, supported by GCC and Clang. + */ +#define bpf_for(i, start, end) for ( \ + /* initialize and define destructor */ \ + struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ + cleanup(bpf_iter_num_destroy))), \ + /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ + *___p __attribute__((unused)) = ( \ + bpf_iter_num_new(&___it, (start), (end)), \ + /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ + /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ + (void)bpf_iter_num_destroy, (void *)0); \ + ({ \ + /* iteration step */ \ + int *___t = bpf_iter_num_next(&___it); \ + /* termination and bounds check */ \ + (___t && ((i) = *___t, (i) >= (start) && (i) < (end))); \ + }); \ +) +#endif /* bpf_for */ + +#ifndef bpf_repeat +/* bpf_repeat(N) performs N iterations without exposing iteration number + * + * Note: similarly to bpf_for_each(), it relies on C99 feature of declaring for() + * loop bound variables and cleanup attribute, supported by GCC and Clang. + */ +#define bpf_repeat(N) for ( \ + /* initialize and define destructor */ \ + struct bpf_iter_num ___it __attribute__((aligned(8), /* enforce, just in case */ \ + cleanup(bpf_iter_num_destroy))), \ + /* ___p pointer is necessary to call bpf_iter_num_new() *once* to init ___it */ \ + *___p __attribute__((unused)) = ( \ + bpf_iter_num_new(&___it, 0, (N)), \ + /* this is a workaround for Clang bug: it currently doesn't emit BTF */ \ + /* for bpf_iter_num_destroy() when used from cleanup() attribute */ \ + (void)bpf_iter_num_destroy, (void *)0); \ + bpf_iter_num_next(&___it); \ + /* nothing here */ \ +) +#endif /* bpf_repeat */ + #endif diff --git a/internal/include/libbpf/bpf_tracing.h b/internal/include/libbpf/bpf_tracing.h index 46a57a02f..1c13f8e88 100644 --- a/internal/include/libbpf/bpf_tracing.h +++ b/internal/include/libbpf/bpf_tracing.h @@ -2,6 +2,8 @@ #ifndef __BPF_TRACING_H__ #define __BPF_TRACING_H__ +#include "bpf_helpers.h" + /* Scan the ARCH passed in from ARCH env variable (see Makefile) */ #if defined(__TARGET_ARCH_x86) #define bpf_target_x86 @@ -27,6 +29,12 @@ #elif defined(__TARGET_ARCH_riscv) #define bpf_target_riscv #define bpf_target_defined +#elif defined(__TARGET_ARCH_arc) + #define bpf_target_arc + #define bpf_target_defined +#elif defined(__TARGET_ARCH_loongarch) + #define bpf_target_loongarch + #define bpf_target_defined #else /* Fall back to what the compiler says */ @@ -54,6 +62,12 @@ #elif defined(__riscv) && __riscv_xlen == 64 #define bpf_target_riscv #define bpf_target_defined +#elif defined(__arc__) + #define bpf_target_arc + #define bpf_target_defined +#elif defined(__loongarch__) + #define bpf_target_loongarch + #define bpf_target_defined #endif /* no compiler target */ #endif @@ -64,308 +78,496 @@ #if defined(bpf_target_x86) +/* + * https://en.wikipedia.org/wiki/X86_calling_conventions#System_V_AMD64_ABI + */ + #if defined(__KERNEL__) || defined(__VMLINUX_H__) -#define GO_PARAM1(x) ((x)->ax) -#define GO_PARAM2(x) ((x)->bx) -#define GO_PARAM3(x) ((x)->cx) -#define GO_PARAM4(x) ((x)->di) -#define GO_PARAM5(x) ((x)->si) -#define GO_PARAM6(x) ((x)->r8) -#define GO_PARAM7(x) ((x)->r9) -#define GO_PARAM8(x) ((x)->r10) -#define GO_PARAM9(x) ((x)->r11) -#define GOROUTINE(x) ((x)->r14) - -#define PT_REGS_PARM1(x) ((x)->di) -#define PT_REGS_PARM2(x) ((x)->si) -#define PT_REGS_PARM3(x) ((x)->dx) -#define PT_REGS_PARM4(x) ((x)->cx) -#define PT_REGS_PARM5(x) ((x)->r8) -#define PT_REGS_RET(x) ((x)->sp) -#define PT_REGS_FP(x) ((x)->bp) -#define PT_REGS_RC(x) ((x)->ax) -#define PT_REGS_SP(x) ((x)->sp) -#define PT_REGS_IP(x) ((x)->ip) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), di) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), si) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), dx) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), cx) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), r8) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), sp) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), bp) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), ax) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), sp) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), ip) +#define __PT_PARM1_REG di +#define __PT_PARM2_REG si +#define __PT_PARM3_REG dx +#define __PT_PARM4_REG cx +#define __PT_PARM5_REG r8 +#define __PT_PARM6_REG r9 +/* + * Syscall uses r10 for PARM4. See arch/x86/entry/entry_64.S:entry_SYSCALL_64 + * comments in Linux sources. And refer to syscall(2) manpage. + */ +#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG +#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG +#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG +#define __PT_PARM4_SYSCALL_REG r10 +#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG +#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG + +#define __PT_RET_REG sp +#define __PT_FP_REG bp +#define __PT_RC_REG ax +#define __PT_SP_REG sp +#define __PT_IP_REG ip #else #ifdef __i386__ -/* i386 kernel is built with -mregparm=3 */ -#define PT_REGS_PARM1(x) ((x)->eax) -#define PT_REGS_PARM2(x) ((x)->edx) -#define PT_REGS_PARM3(x) ((x)->ecx) -#define PT_REGS_PARM4(x) 0 -#define PT_REGS_PARM5(x) 0 -#define PT_REGS_RET(x) ((x)->esp) -#define PT_REGS_FP(x) ((x)->ebp) -#define PT_REGS_RC(x) ((x)->eax) -#define PT_REGS_SP(x) ((x)->esp) -#define PT_REGS_IP(x) ((x)->eip) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), eax) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), edx) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), ecx) -#define PT_REGS_PARM4_CORE(x) 0 -#define PT_REGS_PARM5_CORE(x) 0 -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), esp) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), ebp) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), eax) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), esp) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), eip) -#else +/* i386 kernel is built with -mregparm=3 */ +#define __PT_PARM1_REG eax +#define __PT_PARM2_REG edx +#define __PT_PARM3_REG ecx +/* i386 syscall ABI is very different, refer to syscall(2) manpage */ +#define __PT_PARM1_SYSCALL_REG ebx +#define __PT_PARM2_SYSCALL_REG ecx +#define __PT_PARM3_SYSCALL_REG edx +#define __PT_PARM4_SYSCALL_REG esi +#define __PT_PARM5_SYSCALL_REG edi +#define __PT_PARM6_SYSCALL_REG ebp + +#define __PT_RET_REG esp +#define __PT_FP_REG ebp +#define __PT_RC_REG eax +#define __PT_SP_REG esp +#define __PT_IP_REG eip + +#else /* __i386__ */ + +#define __PT_PARM1_REG rdi +#define __PT_PARM2_REG rsi +#define __PT_PARM3_REG rdx +#define __PT_PARM4_REG rcx +#define __PT_PARM5_REG r8 +#define __PT_PARM6_REG r9 + +#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG +#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG +#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG +#define __PT_PARM4_SYSCALL_REG r10 +#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG +#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG + +#define __PT_RET_REG rsp +#define __PT_FP_REG rbp +#define __PT_RC_REG rax +#define __PT_SP_REG rsp +#define __PT_IP_REG rip + +#endif /* __i386__ */ + +#endif /* __KERNEL__ || __VMLINUX_H__ */ -#define PT_REGS_PARM1(x) ((x)->rdi) -#define PT_REGS_PARM2(x) ((x)->rsi) -#define PT_REGS_PARM3(x) ((x)->rdx) -#define PT_REGS_PARM4(x) ((x)->rcx) -#define PT_REGS_PARM5(x) ((x)->r8) -#define PT_REGS_RET(x) ((x)->rsp) -#define PT_REGS_FP(x) ((x)->rbp) -#define PT_REGS_RC(x) ((x)->rax) -#define PT_REGS_SP(x) ((x)->rsp) -#define PT_REGS_IP(x) ((x)->rip) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), rdi) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), rsi) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), rdx) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), rcx) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), r8) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), rsp) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), rbp) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), rax) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), rsp) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), rip) +#elif defined(bpf_target_s390) -#endif -#endif +/* + * https://github.com/IBM/s390x-abi/releases/download/v1.6/lzsabi_s390x.pdf + */ -#elif defined(bpf_target_s390) +struct pt_regs___s390 { + unsigned long orig_gpr2; +}; /* s390 provides user_pt_regs instead of struct pt_regs to userspace */ -struct pt_regs; -#define PT_REGS_S390 const volatile user_pt_regs -#define PT_REGS_PARM1(x) (((PT_REGS_S390 *)(x))->gprs[2]) -#define PT_REGS_PARM2(x) (((PT_REGS_S390 *)(x))->gprs[3]) -#define PT_REGS_PARM3(x) (((PT_REGS_S390 *)(x))->gprs[4]) -#define PT_REGS_PARM4(x) (((PT_REGS_S390 *)(x))->gprs[5]) -#define PT_REGS_PARM5(x) (((PT_REGS_S390 *)(x))->gprs[6]) -#define PT_REGS_RET(x) (((PT_REGS_S390 *)(x))->gprs[14]) -/* Works only with CONFIG_FRAME_POINTER */ -#define PT_REGS_FP(x) (((PT_REGS_S390 *)(x))->gprs[11]) -#define PT_REGS_RC(x) (((PT_REGS_S390 *)(x))->gprs[2]) -#define PT_REGS_SP(x) (((PT_REGS_S390 *)(x))->gprs[15]) -#define PT_REGS_IP(x) (((PT_REGS_S390 *)(x))->psw.addr) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[2]) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[3]) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[4]) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[5]) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[6]) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[14]) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[11]) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[2]) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), gprs[15]) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((PT_REGS_S390 *)(x), psw.addr) +#define __PT_REGS_CAST(x) ((const user_pt_regs *)(x)) +#define __PT_PARM1_REG gprs[2] +#define __PT_PARM2_REG gprs[3] +#define __PT_PARM3_REG gprs[4] +#define __PT_PARM4_REG gprs[5] +#define __PT_PARM5_REG gprs[6] + +#define __PT_PARM1_SYSCALL_REG orig_gpr2 +#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG +#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG +#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG +#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG +#define __PT_PARM6_SYSCALL_REG gprs[7] +#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x) +#define PT_REGS_PARM1_CORE_SYSCALL(x) \ + BPF_CORE_READ((const struct pt_regs___s390 *)(x), __PT_PARM1_SYSCALL_REG) + +#define __PT_RET_REG gprs[14] +#define __PT_FP_REG gprs[11] /* Works only with CONFIG_FRAME_POINTER */ +#define __PT_RC_REG gprs[2] +#define __PT_SP_REG gprs[15] +#define __PT_IP_REG psw.addr #elif defined(bpf_target_arm) -#define PT_REGS_PARM1(x) ((x)->uregs[0]) -#define PT_REGS_PARM2(x) ((x)->uregs[1]) -#define PT_REGS_PARM3(x) ((x)->uregs[2]) -#define PT_REGS_PARM4(x) ((x)->uregs[3]) -#define PT_REGS_PARM5(x) ((x)->uregs[4]) -#define PT_REGS_RET(x) ((x)->uregs[14]) -#define PT_REGS_FP(x) ((x)->uregs[11]) /* Works only with CONFIG_FRAME_POINTER */ -#define PT_REGS_RC(x) ((x)->uregs[0]) -#define PT_REGS_SP(x) ((x)->uregs[13]) -#define PT_REGS_IP(x) ((x)->uregs[12]) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), uregs[0]) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), uregs[1]) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), uregs[2]) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), uregs[3]) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), uregs[4]) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), uregs[14]) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), uregs[11]) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), uregs[0]) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), uregs[13]) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), uregs[12]) +/* + * https://github.com/ARM-software/abi-aa/blob/main/aapcs32/aapcs32.rst#machine-registers + */ + +#define __PT_PARM1_REG uregs[0] +#define __PT_PARM2_REG uregs[1] +#define __PT_PARM3_REG uregs[2] +#define __PT_PARM4_REG uregs[3] + +#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG +#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG +#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG +#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG +#define __PT_PARM5_SYSCALL_REG uregs[4] +#define __PT_PARM6_SYSCALL_REG uregs[5] +#define __PT_PARM7_SYSCALL_REG uregs[6] + +#define __PT_RET_REG uregs[14] +#define __PT_FP_REG uregs[11] /* Works only with CONFIG_FRAME_POINTER */ +#define __PT_RC_REG uregs[0] +#define __PT_SP_REG uregs[13] +#define __PT_IP_REG uregs[12] #elif defined(bpf_target_arm64) +/* + * https://github.com/ARM-software/abi-aa/blob/main/aapcs64/aapcs64.rst#machine-registers + */ + +struct pt_regs___arm64 { + unsigned long orig_x0; +}; + /* arm64 provides struct user_pt_regs instead of struct pt_regs to userspace */ -struct pt_regs; -#define PT_REGS_ARM64 const volatile struct user_pt_regs - -#define GO_PARAM1(x) (((PT_REGS_ARM64 *)(x))->regs[0]) -#define GO_PARAM2(x) (((PT_REGS_ARM64 *)(x))->regs[1]) -#define GO_PARAM3(x) (((PT_REGS_ARM64 *)(x))->regs[2]) -#define GO_PARAM4(x) (((PT_REGS_ARM64 *)(x))->regs[3]) -#define GO_PARAM5(x) (((PT_REGS_ARM64 *)(x))->regs[4]) -#define GO_PARAM6(x) (((PT_REGS_ARM64 *)(x))->regs[5]) -#define GO_PARAM7(x) (((PT_REGS_ARM64 *)(x))->regs[6]) -#define GO_PARAM8(x) (((PT_REGS_ARM64 *)(x))->regs[7]) -#define GO_PARAM9(x) (((PT_REGS_ARM64 *)(x))->regs[8]) -#define GOROUTINE(x) (((PT_REGS_ARM64 *)(x))->regs[28]) - -#define PT_REGS_PARM1(x) (((PT_REGS_ARM64 *)(x))->regs[0]) -#define PT_REGS_PARM2(x) (((PT_REGS_ARM64 *)(x))->regs[1]) -#define PT_REGS_PARM3(x) (((PT_REGS_ARM64 *)(x))->regs[2]) -#define PT_REGS_PARM4(x) (((PT_REGS_ARM64 *)(x))->regs[3]) -#define PT_REGS_PARM5(x) (((PT_REGS_ARM64 *)(x))->regs[4]) -#define PT_REGS_RET(x) (((PT_REGS_ARM64 *)(x))->regs[30]) -/* Works only with CONFIG_FRAME_POINTER */ -#define PT_REGS_FP(x) (((PT_REGS_ARM64 *)(x))->regs[29]) -#define PT_REGS_RC(x) (((PT_REGS_ARM64 *)(x))->regs[0]) -#define PT_REGS_SP(x) (((PT_REGS_ARM64 *)(x))->sp) -#define PT_REGS_IP(x) (((PT_REGS_ARM64 *)(x))->pc) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[0]) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[1]) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[2]) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[3]) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[4]) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[30]) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[29]) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), regs[0]) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), sp) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((PT_REGS_ARM64 *)(x), pc) +#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x)) +#define __PT_PARM1_REG regs[0] +#define __PT_PARM2_REG regs[1] +#define __PT_PARM3_REG regs[2] +#define __PT_PARM4_REG regs[3] +#define __PT_PARM5_REG regs[4] +#define __PT_PARM6_REG regs[5] +#define __PT_PARM7_REG regs[6] +#define __PT_PARM8_REG regs[7] + +#define __PT_PARM1_SYSCALL_REG orig_x0 +#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG +#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG +#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG +#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG +#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG +#define PT_REGS_PARM1_SYSCALL(x) PT_REGS_PARM1_CORE_SYSCALL(x) +#define PT_REGS_PARM1_CORE_SYSCALL(x) \ + BPF_CORE_READ((const struct pt_regs___arm64 *)(x), __PT_PARM1_SYSCALL_REG) + +#define __PT_RET_REG regs[30] +#define __PT_FP_REG regs[29] /* Works only with CONFIG_FRAME_POINTER */ +#define __PT_RC_REG regs[0] +#define __PT_SP_REG sp +#define __PT_IP_REG pc #elif defined(bpf_target_mips) -#define PT_REGS_PARM1(x) ((x)->regs[4]) -#define PT_REGS_PARM2(x) ((x)->regs[5]) -#define PT_REGS_PARM3(x) ((x)->regs[6]) -#define PT_REGS_PARM4(x) ((x)->regs[7]) -#define PT_REGS_PARM5(x) ((x)->regs[8]) -#define PT_REGS_RET(x) ((x)->regs[31]) -#define PT_REGS_FP(x) ((x)->regs[30]) /* Works only with CONFIG_FRAME_POINTER */ -#define PT_REGS_RC(x) ((x)->regs[2]) -#define PT_REGS_SP(x) ((x)->regs[29]) -#define PT_REGS_IP(x) ((x)->cp0_epc) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), regs[4]) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), regs[5]) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), regs[6]) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), regs[7]) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), regs[8]) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), regs[31]) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((x), regs[30]) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), regs[2]) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), regs[29]) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), cp0_epc) +/* + * N64 ABI is assumed right now. + * https://en.wikipedia.org/wiki/MIPS_architecture#Calling_conventions + */ + +#define __PT_PARM1_REG regs[4] +#define __PT_PARM2_REG regs[5] +#define __PT_PARM3_REG regs[6] +#define __PT_PARM4_REG regs[7] +#define __PT_PARM5_REG regs[8] +#define __PT_PARM6_REG regs[9] +#define __PT_PARM7_REG regs[10] +#define __PT_PARM8_REG regs[11] + +#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG +#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG +#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG +#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG +#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG /* only N32/N64 */ +#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG /* only N32/N64 */ + +#define __PT_RET_REG regs[31] +#define __PT_FP_REG regs[30] /* Works only with CONFIG_FRAME_POINTER */ +#define __PT_RC_REG regs[2] +#define __PT_SP_REG regs[29] +#define __PT_IP_REG cp0_epc #elif defined(bpf_target_powerpc) -#define PT_REGS_PARM1(x) ((x)->gpr[3]) -#define PT_REGS_PARM2(x) ((x)->gpr[4]) -#define PT_REGS_PARM3(x) ((x)->gpr[5]) -#define PT_REGS_PARM4(x) ((x)->gpr[6]) -#define PT_REGS_PARM5(x) ((x)->gpr[7]) -#define PT_REGS_RC(x) ((x)->gpr[3]) -#define PT_REGS_SP(x) ((x)->sp) -#define PT_REGS_IP(x) ((x)->nip) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), gpr[3]) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), gpr[4]) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), gpr[5]) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), gpr[6]) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), gpr[7]) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), gpr[3]) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), sp) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), nip) +/* + * http://refspecs.linux-foundation.org/elf/elfspec_ppc.pdf (page 3-14, + * section "Function Calling Sequence") + */ + +#define __PT_PARM1_REG gpr[3] +#define __PT_PARM2_REG gpr[4] +#define __PT_PARM3_REG gpr[5] +#define __PT_PARM4_REG gpr[6] +#define __PT_PARM5_REG gpr[7] +#define __PT_PARM6_REG gpr[8] +#define __PT_PARM7_REG gpr[9] +#define __PT_PARM8_REG gpr[10] + +/* powerpc does not select ARCH_HAS_SYSCALL_WRAPPER. */ +#define PT_REGS_SYSCALL_REGS(ctx) ctx +#define __PT_PARM1_SYSCALL_REG orig_gpr3 +#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG +#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG +#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG +#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG +#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG +#if !defined(__arch64__) +#define __PT_PARM7_SYSCALL_REG __PT_PARM7_REG /* only powerpc (not powerpc64) */ +#endif + +#define __PT_RET_REG regs[31] +#define __PT_FP_REG __unsupported__ +#define __PT_RC_REG gpr[3] +#define __PT_SP_REG sp +#define __PT_IP_REG nip #elif defined(bpf_target_sparc) -#define PT_REGS_PARM1(x) ((x)->u_regs[UREG_I0]) -#define PT_REGS_PARM2(x) ((x)->u_regs[UREG_I1]) -#define PT_REGS_PARM3(x) ((x)->u_regs[UREG_I2]) -#define PT_REGS_PARM4(x) ((x)->u_regs[UREG_I3]) -#define PT_REGS_PARM5(x) ((x)->u_regs[UREG_I4]) -#define PT_REGS_RET(x) ((x)->u_regs[UREG_I7]) -#define PT_REGS_RC(x) ((x)->u_regs[UREG_I0]) -#define PT_REGS_SP(x) ((x)->u_regs[UREG_FP]) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I0]) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I1]) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I2]) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I3]) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I4]) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I7]) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((x), u_regs[UREG_I0]) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((x), u_regs[UREG_FP]) +/* + * https://en.wikipedia.org/wiki/Calling_convention#SPARC + */ +#define __PT_PARM1_REG u_regs[UREG_I0] +#define __PT_PARM2_REG u_regs[UREG_I1] +#define __PT_PARM3_REG u_regs[UREG_I2] +#define __PT_PARM4_REG u_regs[UREG_I3] +#define __PT_PARM5_REG u_regs[UREG_I4] +#define __PT_PARM6_REG u_regs[UREG_I5] + +#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG +#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG +#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG +#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG +#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG +#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG + +#define __PT_RET_REG u_regs[UREG_I7] +#define __PT_FP_REG __unsupported__ +#define __PT_RC_REG u_regs[UREG_I0] +#define __PT_SP_REG u_regs[UREG_FP] /* Should this also be a bpf_target check for the sparc case? */ #if defined(__arch64__) -#define PT_REGS_IP(x) ((x)->tpc) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), tpc) +#define __PT_IP_REG tpc #else -#define PT_REGS_IP(x) ((x)->pc) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((x), pc) +#define __PT_IP_REG pc #endif #elif defined(bpf_target_riscv) +/* + * https://github.com/riscv-non-isa/riscv-elf-psabi-doc/blob/master/riscv-cc.adoc#risc-v-calling-conventions + */ + +/* riscv provides struct user_regs_struct instead of struct pt_regs to userspace */ +#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x)) +#define __PT_PARM1_REG a0 +#define __PT_PARM2_REG a1 +#define __PT_PARM3_REG a2 +#define __PT_PARM4_REG a3 +#define __PT_PARM5_REG a4 +#define __PT_PARM6_REG a5 +#define __PT_PARM7_REG a6 +#define __PT_PARM8_REG a7 + +#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG +#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG +#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG +#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG +#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG +#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG + +#define __PT_RET_REG ra +#define __PT_FP_REG s0 +#define __PT_RC_REG a0 +#define __PT_SP_REG sp +#define __PT_IP_REG pc + +#elif defined(bpf_target_arc) + +/* + * Section "Function Calling Sequence" (page 24): + * https://raw.githubusercontent.com/wiki/foss-for-synopsys-dwc-arc-processors/toolchain/files/ARCv2_ABI.pdf + */ + +/* arc provides struct user_regs_struct instead of struct pt_regs to userspace */ +#define __PT_REGS_CAST(x) ((const struct user_regs_struct *)(x)) +#define __PT_PARM1_REG scratch.r0 +#define __PT_PARM2_REG scratch.r1 +#define __PT_PARM3_REG scratch.r2 +#define __PT_PARM4_REG scratch.r3 +#define __PT_PARM5_REG scratch.r4 +#define __PT_PARM6_REG scratch.r5 +#define __PT_PARM7_REG scratch.r6 +#define __PT_PARM8_REG scratch.r7 + +/* arc does not select ARCH_HAS_SYSCALL_WRAPPER. */ +#define PT_REGS_SYSCALL_REGS(ctx) ctx +#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG +#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG +#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG +#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG +#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG +#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG + +#define __PT_RET_REG scratch.blink +#define __PT_FP_REG scratch.fp +#define __PT_RC_REG scratch.r0 +#define __PT_SP_REG scratch.sp +#define __PT_IP_REG scratch.ret + +#elif defined(bpf_target_loongarch) + +/* + * https://docs.kernel.org/loongarch/introduction.html + * https://loongson.github.io/LoongArch-Documentation/LoongArch-ELF-ABI-EN.html + */ + +/* loongarch provides struct user_pt_regs instead of struct pt_regs to userspace */ +#define __PT_REGS_CAST(x) ((const struct user_pt_regs *)(x)) +#define __PT_PARM1_REG regs[4] +#define __PT_PARM2_REG regs[5] +#define __PT_PARM3_REG regs[6] +#define __PT_PARM4_REG regs[7] +#define __PT_PARM5_REG regs[8] +#define __PT_PARM6_REG regs[9] +#define __PT_PARM7_REG regs[10] +#define __PT_PARM8_REG regs[11] + +/* loongarch does not select ARCH_HAS_SYSCALL_WRAPPER. */ +#define PT_REGS_SYSCALL_REGS(ctx) ctx +#define __PT_PARM1_SYSCALL_REG __PT_PARM1_REG +#define __PT_PARM2_SYSCALL_REG __PT_PARM2_REG +#define __PT_PARM3_SYSCALL_REG __PT_PARM3_REG +#define __PT_PARM4_SYSCALL_REG __PT_PARM4_REG +#define __PT_PARM5_SYSCALL_REG __PT_PARM5_REG +#define __PT_PARM6_SYSCALL_REG __PT_PARM6_REG + +#define __PT_RET_REG regs[1] +#define __PT_FP_REG regs[22] +#define __PT_RC_REG regs[4] +#define __PT_SP_REG regs[3] +#define __PT_IP_REG csr_era + +#endif + +#if defined(bpf_target_defined) + struct pt_regs; -#define PT_REGS_RV const volatile struct user_regs_struct -#define PT_REGS_PARM1(x) (((PT_REGS_RV *)(x))->a0) -#define PT_REGS_PARM2(x) (((PT_REGS_RV *)(x))->a1) -#define PT_REGS_PARM3(x) (((PT_REGS_RV *)(x))->a2) -#define PT_REGS_PARM4(x) (((PT_REGS_RV *)(x))->a3) -#define PT_REGS_PARM5(x) (((PT_REGS_RV *)(x))->a4) -#define PT_REGS_RET(x) (((PT_REGS_RV *)(x))->ra) -#define PT_REGS_FP(x) (((PT_REGS_RV *)(x))->s5) -#define PT_REGS_RC(x) (((PT_REGS_RV *)(x))->a5) -#define PT_REGS_SP(x) (((PT_REGS_RV *)(x))->sp) -#define PT_REGS_IP(x) (((PT_REGS_RV *)(x))->epc) - -#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a0) -#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a1) -#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a2) -#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a3) -#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a4) -#define PT_REGS_RET_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), ra) -#define PT_REGS_FP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), fp) -#define PT_REGS_RC_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), a5) -#define PT_REGS_SP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), sp) -#define PT_REGS_IP_CORE(x) BPF_CORE_READ((PT_REGS_RV *)(x), epc) +/* allow some architectures to override `struct pt_regs` */ +#ifndef __PT_REGS_CAST +#define __PT_REGS_CAST(x) (x) +#endif + +/* + * Different architectures support different number of arguments passed + * through registers. i386 supports just 3, some arches support up to 8. + */ +#ifndef __PT_PARM4_REG +#define __PT_PARM4_REG __unsupported__ +#endif +#ifndef __PT_PARM5_REG +#define __PT_PARM5_REG __unsupported__ +#endif +#ifndef __PT_PARM6_REG +#define __PT_PARM6_REG __unsupported__ +#endif +#ifndef __PT_PARM7_REG +#define __PT_PARM7_REG __unsupported__ +#endif +#ifndef __PT_PARM8_REG +#define __PT_PARM8_REG __unsupported__ +#endif +/* + * Similarly, syscall-specific conventions might differ between function call + * conventions within each architecutre. All supported architectures pass + * either 6 or 7 syscall arguments in registers. + * + * See syscall(2) manpage for succinct table with information on each arch. + */ +#ifndef __PT_PARM7_SYSCALL_REG +#define __PT_PARM7_SYSCALL_REG __unsupported__ #endif +#define PT_REGS_PARM1(x) (__PT_REGS_CAST(x)->__PT_PARM1_REG) +#define PT_REGS_PARM2(x) (__PT_REGS_CAST(x)->__PT_PARM2_REG) +#define PT_REGS_PARM3(x) (__PT_REGS_CAST(x)->__PT_PARM3_REG) +#define PT_REGS_PARM4(x) (__PT_REGS_CAST(x)->__PT_PARM4_REG) +#define PT_REGS_PARM5(x) (__PT_REGS_CAST(x)->__PT_PARM5_REG) +#define PT_REGS_PARM6(x) (__PT_REGS_CAST(x)->__PT_PARM6_REG) +#define PT_REGS_PARM7(x) (__PT_REGS_CAST(x)->__PT_PARM7_REG) +#define PT_REGS_PARM8(x) (__PT_REGS_CAST(x)->__PT_PARM8_REG) +#define PT_REGS_RET(x) (__PT_REGS_CAST(x)->__PT_RET_REG) +#define PT_REGS_FP(x) (__PT_REGS_CAST(x)->__PT_FP_REG) +#define PT_REGS_RC(x) (__PT_REGS_CAST(x)->__PT_RC_REG) +#define PT_REGS_SP(x) (__PT_REGS_CAST(x)->__PT_SP_REG) +#define PT_REGS_IP(x) (__PT_REGS_CAST(x)->__PT_IP_REG) + +#define PT_REGS_PARM1_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_REG) +#define PT_REGS_PARM2_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_REG) +#define PT_REGS_PARM3_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_REG) +#define PT_REGS_PARM4_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_REG) +#define PT_REGS_PARM5_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_REG) +#define PT_REGS_PARM6_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_REG) +#define PT_REGS_PARM7_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_REG) +#define PT_REGS_PARM8_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM8_REG) +#define PT_REGS_RET_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RET_REG) +#define PT_REGS_FP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_FP_REG) +#define PT_REGS_RC_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_RC_REG) +#define PT_REGS_SP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_SP_REG) +#define PT_REGS_IP_CORE(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_IP_REG) + #if defined(bpf_target_powerpc) + #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; }) #define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP + #elif defined(bpf_target_sparc) + #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); }) #define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP -#elif defined(bpf_target_defined) + +#else + #define BPF_KPROBE_READ_RET_IP(ip, ctx) \ ({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); }) #define BPF_KRETPROBE_READ_RET_IP(ip, ctx) \ - ({ bpf_probe_read_kernel(&(ip), sizeof(ip), \ - (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) + ({ bpf_probe_read_kernel(&(ip), sizeof(ip), (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) + #endif -#if !defined(bpf_target_defined) +#ifndef PT_REGS_PARM1_SYSCALL +#define PT_REGS_PARM1_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM1_SYSCALL_REG) +#define PT_REGS_PARM1_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM1_SYSCALL_REG) +#endif +#ifndef PT_REGS_PARM2_SYSCALL +#define PT_REGS_PARM2_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM2_SYSCALL_REG) +#define PT_REGS_PARM2_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM2_SYSCALL_REG) +#endif +#ifndef PT_REGS_PARM3_SYSCALL +#define PT_REGS_PARM3_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM3_SYSCALL_REG) +#define PT_REGS_PARM3_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM3_SYSCALL_REG) +#endif +#ifndef PT_REGS_PARM4_SYSCALL +#define PT_REGS_PARM4_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM4_SYSCALL_REG) +#define PT_REGS_PARM4_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM4_SYSCALL_REG) +#endif +#ifndef PT_REGS_PARM5_SYSCALL +#define PT_REGS_PARM5_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM5_SYSCALL_REG) +#define PT_REGS_PARM5_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM5_SYSCALL_REG) +#endif +#ifndef PT_REGS_PARM6_SYSCALL +#define PT_REGS_PARM6_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM6_SYSCALL_REG) +#define PT_REGS_PARM6_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM6_SYSCALL_REG) +#endif +#ifndef PT_REGS_PARM7_SYSCALL +#define PT_REGS_PARM7_SYSCALL(x) (__PT_REGS_CAST(x)->__PT_PARM7_SYSCALL_REG) +#define PT_REGS_PARM7_CORE_SYSCALL(x) BPF_CORE_READ(__PT_REGS_CAST(x), __PT_PARM7_SYSCALL_REG) +#endif + +#else /* defined(bpf_target_defined) */ #define PT_REGS_PARM1(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define PT_REGS_PARM2(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define PT_REGS_PARM3(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define PT_REGS_PARM4(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define PT_REGS_PARM5(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM6(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM7(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM8(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define PT_REGS_RET(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define PT_REGS_FP(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define PT_REGS_RC(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) @@ -377,6 +579,9 @@ struct pt_regs; #define PT_REGS_PARM3_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define PT_REGS_PARM4_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define PT_REGS_PARM5_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM6_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM7_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM8_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define PT_REGS_RET_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define PT_REGS_FP_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define PT_REGS_RC_CORE(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) @@ -386,7 +591,33 @@ struct pt_regs; #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) #define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) -#endif /* !defined(bpf_target_defined) */ +#define PT_REGS_PARM1_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM2_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM3_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM4_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM5_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM6_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM7_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) + +#define PT_REGS_PARM1_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM2_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM3_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM4_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM5_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM6_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) +#define PT_REGS_PARM7_CORE_SYSCALL(x) ({ _Pragma(__BPF_TARGET_MISSING); 0l; }) + +#endif /* defined(bpf_target_defined) */ + +/* + * When invoked from a syscall handler kprobe, returns a pointer to a + * struct pt_regs containing syscall arguments and suitable for passing to + * PT_REGS_PARMn_SYSCALL() and PT_REGS_PARMn_CORE_SYSCALL(). + */ +#ifndef PT_REGS_SYSCALL_REGS +/* By default, assume that the arch selects ARCH_HAS_SYSCALL_WRAPPER. */ +#define PT_REGS_SYSCALL_REGS(ctx) ((struct pt_regs *)PT_REGS_PARM1(ctx)) +#endif #ifndef ___bpf_concat #define ___bpf_concat(a, b) a ## b @@ -398,25 +629,23 @@ struct pt_regs; #define ___bpf_nth(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _a, _b, _c, N, ...) N #endif #ifndef ___bpf_narg -#define ___bpf_narg(...) \ - ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) +#define ___bpf_narg(...) ___bpf_nth(_, ##__VA_ARGS__, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0) #endif -#define ___bpf_ctx_cast0() ctx -#define ___bpf_ctx_cast1(x) ___bpf_ctx_cast0(), (void *)ctx[0] -#define ___bpf_ctx_cast2(x, args...) ___bpf_ctx_cast1(args), (void *)ctx[1] -#define ___bpf_ctx_cast3(x, args...) ___bpf_ctx_cast2(args), (void *)ctx[2] -#define ___bpf_ctx_cast4(x, args...) ___bpf_ctx_cast3(args), (void *)ctx[3] -#define ___bpf_ctx_cast5(x, args...) ___bpf_ctx_cast4(args), (void *)ctx[4] -#define ___bpf_ctx_cast6(x, args...) ___bpf_ctx_cast5(args), (void *)ctx[5] -#define ___bpf_ctx_cast7(x, args...) ___bpf_ctx_cast6(args), (void *)ctx[6] -#define ___bpf_ctx_cast8(x, args...) ___bpf_ctx_cast7(args), (void *)ctx[7] -#define ___bpf_ctx_cast9(x, args...) ___bpf_ctx_cast8(args), (void *)ctx[8] +#define ___bpf_ctx_cast0() ctx +#define ___bpf_ctx_cast1(x) ___bpf_ctx_cast0(), (void *)ctx[0] +#define ___bpf_ctx_cast2(x, args...) ___bpf_ctx_cast1(args), (void *)ctx[1] +#define ___bpf_ctx_cast3(x, args...) ___bpf_ctx_cast2(args), (void *)ctx[2] +#define ___bpf_ctx_cast4(x, args...) ___bpf_ctx_cast3(args), (void *)ctx[3] +#define ___bpf_ctx_cast5(x, args...) ___bpf_ctx_cast4(args), (void *)ctx[4] +#define ___bpf_ctx_cast6(x, args...) ___bpf_ctx_cast5(args), (void *)ctx[5] +#define ___bpf_ctx_cast7(x, args...) ___bpf_ctx_cast6(args), (void *)ctx[6] +#define ___bpf_ctx_cast8(x, args...) ___bpf_ctx_cast7(args), (void *)ctx[7] +#define ___bpf_ctx_cast9(x, args...) ___bpf_ctx_cast8(args), (void *)ctx[8] #define ___bpf_ctx_cast10(x, args...) ___bpf_ctx_cast9(args), (void *)ctx[9] #define ___bpf_ctx_cast11(x, args...) ___bpf_ctx_cast10(args), (void *)ctx[10] #define ___bpf_ctx_cast12(x, args...) ___bpf_ctx_cast11(args), (void *)ctx[11] -#define ___bpf_ctx_cast(args...) \ - ___bpf_apply(___bpf_ctx_cast, ___bpf_narg(args))(args) +#define ___bpf_ctx_cast(args...) ___bpf_apply(___bpf_ctx_cast, ___bpf_narg(args))(args) /* * BPF_PROG is a convenience wrapper for generic tp_btf/fentry/fexit and @@ -435,7 +664,7 @@ struct pt_regs; */ #define BPF_PROG(name, args...) \ name(unsigned long long *ctx); \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(unsigned long long *ctx, ##args); \ typeof(name(0)) name(unsigned long long *ctx) \ { \ @@ -444,24 +673,128 @@ typeof(name(0)) name(unsigned long long *ctx) \ return ____##name(___bpf_ctx_cast(args)); \ _Pragma("GCC diagnostic pop") \ } \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(unsigned long long *ctx, ##args) +#ifndef ___bpf_nth2 +#define ___bpf_nth2(_, _1, _2, _3, _4, _5, _6, _7, _8, _9, _10, _11, _12, _13, \ + _14, _15, _16, _17, _18, _19, _20, _21, _22, _23, _24, N, ...) N +#endif +#ifndef ___bpf_narg2 +#define ___bpf_narg2(...) \ + ___bpf_nth2(_, ##__VA_ARGS__, 12, 12, 11, 11, 10, 10, 9, 9, 8, 8, 7, 7, \ + 6, 6, 5, 5, 4, 4, 3, 3, 2, 2, 1, 1, 0) +#endif + +#define ___bpf_treg_cnt(t) \ + __builtin_choose_expr(sizeof(t) == 1, 1, \ + __builtin_choose_expr(sizeof(t) == 2, 1, \ + __builtin_choose_expr(sizeof(t) == 4, 1, \ + __builtin_choose_expr(sizeof(t) == 8, 1, \ + __builtin_choose_expr(sizeof(t) == 16, 2, \ + (void)0))))) + +#define ___bpf_reg_cnt0() (0) +#define ___bpf_reg_cnt1(t, x) (___bpf_reg_cnt0() + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt2(t, x, args...) (___bpf_reg_cnt1(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt3(t, x, args...) (___bpf_reg_cnt2(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt4(t, x, args...) (___bpf_reg_cnt3(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt5(t, x, args...) (___bpf_reg_cnt4(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt6(t, x, args...) (___bpf_reg_cnt5(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt7(t, x, args...) (___bpf_reg_cnt6(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt8(t, x, args...) (___bpf_reg_cnt7(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt9(t, x, args...) (___bpf_reg_cnt8(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt10(t, x, args...) (___bpf_reg_cnt9(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt11(t, x, args...) (___bpf_reg_cnt10(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt12(t, x, args...) (___bpf_reg_cnt11(args) + ___bpf_treg_cnt(t)) +#define ___bpf_reg_cnt(args...) ___bpf_apply(___bpf_reg_cnt, ___bpf_narg2(args))(args) + +#define ___bpf_union_arg(t, x, n) \ + __builtin_choose_expr(sizeof(t) == 1, ({ union { __u8 z[1]; t x; } ___t = { .z = {ctx[n]}}; ___t.x; }), \ + __builtin_choose_expr(sizeof(t) == 2, ({ union { __u16 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \ + __builtin_choose_expr(sizeof(t) == 4, ({ union { __u32 z[1]; t x; } ___t = { .z = {ctx[n]} }; ___t.x; }), \ + __builtin_choose_expr(sizeof(t) == 8, ({ union { __u64 z[1]; t x; } ___t = {.z = {ctx[n]} }; ___t.x; }), \ + __builtin_choose_expr(sizeof(t) == 16, ({ union { __u64 z[2]; t x; } ___t = {.z = {ctx[n], ctx[n + 1]} }; ___t.x; }), \ + (void)0))))) + +#define ___bpf_ctx_arg0(n, args...) +#define ___bpf_ctx_arg1(n, t, x) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt1(t, x)) +#define ___bpf_ctx_arg2(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt2(t, x, args)) ___bpf_ctx_arg1(n, args) +#define ___bpf_ctx_arg3(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt3(t, x, args)) ___bpf_ctx_arg2(n, args) +#define ___bpf_ctx_arg4(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt4(t, x, args)) ___bpf_ctx_arg3(n, args) +#define ___bpf_ctx_arg5(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt5(t, x, args)) ___bpf_ctx_arg4(n, args) +#define ___bpf_ctx_arg6(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt6(t, x, args)) ___bpf_ctx_arg5(n, args) +#define ___bpf_ctx_arg7(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt7(t, x, args)) ___bpf_ctx_arg6(n, args) +#define ___bpf_ctx_arg8(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt8(t, x, args)) ___bpf_ctx_arg7(n, args) +#define ___bpf_ctx_arg9(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt9(t, x, args)) ___bpf_ctx_arg8(n, args) +#define ___bpf_ctx_arg10(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt10(t, x, args)) ___bpf_ctx_arg9(n, args) +#define ___bpf_ctx_arg11(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt11(t, x, args)) ___bpf_ctx_arg10(n, args) +#define ___bpf_ctx_arg12(n, t, x, args...) , ___bpf_union_arg(t, x, n - ___bpf_reg_cnt12(t, x, args)) ___bpf_ctx_arg11(n, args) +#define ___bpf_ctx_arg(args...) ___bpf_apply(___bpf_ctx_arg, ___bpf_narg2(args))(___bpf_reg_cnt(args), args) + +#define ___bpf_ctx_decl0() +#define ___bpf_ctx_decl1(t, x) , t x +#define ___bpf_ctx_decl2(t, x, args...) , t x ___bpf_ctx_decl1(args) +#define ___bpf_ctx_decl3(t, x, args...) , t x ___bpf_ctx_decl2(args) +#define ___bpf_ctx_decl4(t, x, args...) , t x ___bpf_ctx_decl3(args) +#define ___bpf_ctx_decl5(t, x, args...) , t x ___bpf_ctx_decl4(args) +#define ___bpf_ctx_decl6(t, x, args...) , t x ___bpf_ctx_decl5(args) +#define ___bpf_ctx_decl7(t, x, args...) , t x ___bpf_ctx_decl6(args) +#define ___bpf_ctx_decl8(t, x, args...) , t x ___bpf_ctx_decl7(args) +#define ___bpf_ctx_decl9(t, x, args...) , t x ___bpf_ctx_decl8(args) +#define ___bpf_ctx_decl10(t, x, args...) , t x ___bpf_ctx_decl9(args) +#define ___bpf_ctx_decl11(t, x, args...) , t x ___bpf_ctx_decl10(args) +#define ___bpf_ctx_decl12(t, x, args...) , t x ___bpf_ctx_decl11(args) +#define ___bpf_ctx_decl(args...) ___bpf_apply(___bpf_ctx_decl, ___bpf_narg2(args))(args) + +/* + * BPF_PROG2 is an enhanced version of BPF_PROG in order to handle struct + * arguments. Since each struct argument might take one or two u64 values + * in the trampoline stack, argument type size is needed to place proper number + * of u64 values for each argument. Therefore, BPF_PROG2 has different + * syntax from BPF_PROG. For example, for the following BPF_PROG syntax: + * + * int BPF_PROG(test2, int a, int b) { ... } + * + * the corresponding BPF_PROG2 syntax is: + * + * int BPF_PROG2(test2, int, a, int, b) { ... } + * + * where type and the corresponding argument name are separated by comma. + * + * Use BPF_PROG2 macro if one of the arguments might be a struct/union larger + * than 8 bytes: + * + * int BPF_PROG2(test_struct_arg, struct bpf_testmod_struct_arg_1, a, int, b, + * int, c, int, d, struct bpf_testmod_struct_arg_2, e, int, ret) + * { + * // access a, b, c, d, e, and ret directly + * ... + * } + */ +#define BPF_PROG2(name, args...) \ +name(unsigned long long *ctx); \ +static __always_inline typeof(name(0)) \ +____##name(unsigned long long *ctx ___bpf_ctx_decl(args)); \ +typeof(name(0)) name(unsigned long long *ctx) \ +{ \ + return ____##name(ctx ___bpf_ctx_arg(args)); \ +} \ +static __always_inline typeof(name(0)) \ +____##name(unsigned long long *ctx ___bpf_ctx_decl(args)) + struct pt_regs; -#define ___bpf_kprobe_args0() ctx -#define ___bpf_kprobe_args1(x) \ - ___bpf_kprobe_args0(), (void *)PT_REGS_PARM1(ctx) -#define ___bpf_kprobe_args2(x, args...) \ - ___bpf_kprobe_args1(args), (void *)PT_REGS_PARM2(ctx) -#define ___bpf_kprobe_args3(x, args...) \ - ___bpf_kprobe_args2(args), (void *)PT_REGS_PARM3(ctx) -#define ___bpf_kprobe_args4(x, args...) \ - ___bpf_kprobe_args3(args), (void *)PT_REGS_PARM4(ctx) -#define ___bpf_kprobe_args5(x, args...) \ - ___bpf_kprobe_args4(args), (void *)PT_REGS_PARM5(ctx) -#define ___bpf_kprobe_args(args...) \ - ___bpf_apply(___bpf_kprobe_args, ___bpf_narg(args))(args) +#define ___bpf_kprobe_args0() ctx +#define ___bpf_kprobe_args1(x) ___bpf_kprobe_args0(), (void *)PT_REGS_PARM1(ctx) +#define ___bpf_kprobe_args2(x, args...) ___bpf_kprobe_args1(args), (void *)PT_REGS_PARM2(ctx) +#define ___bpf_kprobe_args3(x, args...) ___bpf_kprobe_args2(args), (void *)PT_REGS_PARM3(ctx) +#define ___bpf_kprobe_args4(x, args...) ___bpf_kprobe_args3(args), (void *)PT_REGS_PARM4(ctx) +#define ___bpf_kprobe_args5(x, args...) ___bpf_kprobe_args4(args), (void *)PT_REGS_PARM5(ctx) +#define ___bpf_kprobe_args6(x, args...) ___bpf_kprobe_args5(args), (void *)PT_REGS_PARM6(ctx) +#define ___bpf_kprobe_args7(x, args...) ___bpf_kprobe_args6(args), (void *)PT_REGS_PARM7(ctx) +#define ___bpf_kprobe_args8(x, args...) ___bpf_kprobe_args7(args), (void *)PT_REGS_PARM8(ctx) +#define ___bpf_kprobe_args(args...) ___bpf_apply(___bpf_kprobe_args, ___bpf_narg(args))(args) /* * BPF_KPROBE serves the same purpose for kprobes as BPF_PROG for @@ -475,7 +808,7 @@ struct pt_regs; */ #define BPF_KPROBE(name, args...) \ name(struct pt_regs *ctx); \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args); \ typeof(name(0)) name(struct pt_regs *ctx) \ { \ @@ -484,14 +817,12 @@ typeof(name(0)) name(struct pt_regs *ctx) \ return ____##name(___bpf_kprobe_args(args)); \ _Pragma("GCC diagnostic pop") \ } \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args) -#define ___bpf_kretprobe_args0() ctx -#define ___bpf_kretprobe_args1(x) \ - ___bpf_kretprobe_args0(), (void *)PT_REGS_RC(ctx) -#define ___bpf_kretprobe_args(args...) \ - ___bpf_apply(___bpf_kretprobe_args, ___bpf_narg(args))(args) +#define ___bpf_kretprobe_args0() ctx +#define ___bpf_kretprobe_args1(x) ___bpf_kretprobe_args0(), (void *)PT_REGS_RC(ctx) +#define ___bpf_kretprobe_args(args...) ___bpf_apply(___bpf_kretprobe_args, ___bpf_narg(args))(args) /* * BPF_KRETPROBE is similar to BPF_KPROBE, except, it only provides optional @@ -501,7 +832,7 @@ ____##name(struct pt_regs *ctx, ##args) */ #define BPF_KRETPROBE(name, args...) \ name(struct pt_regs *ctx); \ -static __attribute__((always_inline)) typeof(name(0)) \ +static __always_inline typeof(name(0)) \ ____##name(struct pt_regs *ctx, ##args); \ typeof(name(0)) name(struct pt_regs *ctx) \ { \ @@ -512,4 +843,80 @@ typeof(name(0)) name(struct pt_regs *ctx) \ } \ static __always_inline typeof(name(0)) ____##name(struct pt_regs *ctx, ##args) +/* If kernel has CONFIG_ARCH_HAS_SYSCALL_WRAPPER, read pt_regs directly */ +#define ___bpf_syscall_args0() ctx +#define ___bpf_syscall_args1(x) ___bpf_syscall_args0(), (void *)PT_REGS_PARM1_SYSCALL(regs) +#define ___bpf_syscall_args2(x, args...) ___bpf_syscall_args1(args), (void *)PT_REGS_PARM2_SYSCALL(regs) +#define ___bpf_syscall_args3(x, args...) ___bpf_syscall_args2(args), (void *)PT_REGS_PARM3_SYSCALL(regs) +#define ___bpf_syscall_args4(x, args...) ___bpf_syscall_args3(args), (void *)PT_REGS_PARM4_SYSCALL(regs) +#define ___bpf_syscall_args5(x, args...) ___bpf_syscall_args4(args), (void *)PT_REGS_PARM5_SYSCALL(regs) +#define ___bpf_syscall_args6(x, args...) ___bpf_syscall_args5(args), (void *)PT_REGS_PARM6_SYSCALL(regs) +#define ___bpf_syscall_args7(x, args...) ___bpf_syscall_args6(args), (void *)PT_REGS_PARM7_SYSCALL(regs) +#define ___bpf_syscall_args(args...) ___bpf_apply(___bpf_syscall_args, ___bpf_narg(args))(args) + +/* If kernel doesn't have CONFIG_ARCH_HAS_SYSCALL_WRAPPER, we have to BPF_CORE_READ from pt_regs */ +#define ___bpf_syswrap_args0() ctx +#define ___bpf_syswrap_args1(x) ___bpf_syswrap_args0(), (void *)PT_REGS_PARM1_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args2(x, args...) ___bpf_syswrap_args1(args), (void *)PT_REGS_PARM2_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args3(x, args...) ___bpf_syswrap_args2(args), (void *)PT_REGS_PARM3_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args4(x, args...) ___bpf_syswrap_args3(args), (void *)PT_REGS_PARM4_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args5(x, args...) ___bpf_syswrap_args4(args), (void *)PT_REGS_PARM5_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args6(x, args...) ___bpf_syswrap_args5(args), (void *)PT_REGS_PARM6_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args7(x, args...) ___bpf_syswrap_args6(args), (void *)PT_REGS_PARM7_CORE_SYSCALL(regs) +#define ___bpf_syswrap_args(args...) ___bpf_apply(___bpf_syswrap_args, ___bpf_narg(args))(args) + +/* + * BPF_KSYSCALL is a variant of BPF_KPROBE, which is intended for + * tracing syscall functions, like __x64_sys_close. It hides the underlying + * platform-specific low-level way of getting syscall input arguments from + * struct pt_regs, and provides a familiar typed and named function arguments + * syntax and semantics of accessing syscall input parameters. + * + * Original struct pt_regs * context is preserved as 'ctx' argument. This might + * be necessary when using BPF helpers like bpf_perf_event_output(). + * + * At the moment BPF_KSYSCALL does not transparently handle all the calling + * convention quirks for the following syscalls: + * + * - mmap(): __ARCH_WANT_SYS_OLD_MMAP. + * - clone(): CONFIG_CLONE_BACKWARDS, CONFIG_CLONE_BACKWARDS2 and + * CONFIG_CLONE_BACKWARDS3. + * - socket-related syscalls: __ARCH_WANT_SYS_SOCKETCALL. + * - compat syscalls. + * + * This may or may not change in the future. User needs to take extra measures + * to handle such quirks explicitly, if necessary. + * + * This macro relies on BPF CO-RE support and virtual __kconfig externs. + */ +#define BPF_KSYSCALL(name, args...) \ +name(struct pt_regs *ctx); \ +extern _Bool LINUX_HAS_SYSCALL_WRAPPER __kconfig; \ +static __always_inline typeof(name(0)) \ +____##name(struct pt_regs *ctx, ##args); \ +typeof(name(0)) name(struct pt_regs *ctx) \ +{ \ + struct pt_regs *regs = LINUX_HAS_SYSCALL_WRAPPER \ + ? (struct pt_regs *)PT_REGS_PARM1(ctx) \ + : ctx; \ + _Pragma("GCC diagnostic push") \ + _Pragma("GCC diagnostic ignored \"-Wint-conversion\"") \ + if (LINUX_HAS_SYSCALL_WRAPPER) \ + return ____##name(___bpf_syswrap_args(args)); \ + else \ + return ____##name(___bpf_syscall_args(args)); \ + _Pragma("GCC diagnostic pop") \ +} \ +static __always_inline typeof(name(0)) \ +____##name(struct pt_regs *ctx, ##args) + +#define BPF_KPROBE_SYSCALL BPF_KSYSCALL + +/* BPF_UPROBE and BPF_URETPROBE are identical to BPF_KPROBE and BPF_KRETPROBE, + * but are named way less confusingly for SEC("uprobe") and SEC("uretprobe") + * use cases. + */ +#define BPF_UPROBE(name, args...) BPF_KPROBE(name, ##args) +#define BPF_URETPROBE(name, args...) BPF_KRETPROBE(name, ##args) + #endif diff --git a/internal/include/libbpf/btf.h b/internal/include/libbpf/btf.h deleted file mode 100644 index 4a711f990..000000000 --- a/internal/include/libbpf/btf.h +++ /dev/null @@ -1,403 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -/* Copyright (c) 2018 Facebook */ - -#ifndef __LIBBPF_BTF_H -#define __LIBBPF_BTF_H - -#include -#include -#include -#include - -#include "libbpf_common.h" - -#ifdef __cplusplus -extern "C" { -#endif - -#define BTF_ELF_SEC ".BTF" -#define BTF_EXT_ELF_SEC ".BTF.ext" -#define MAPS_ELF_SEC ".maps" - -struct btf; -struct btf_ext; -struct btf_type; - -struct bpf_object; - -enum btf_endianness { - BTF_LITTLE_ENDIAN = 0, - BTF_BIG_ENDIAN = 1, -}; - -LIBBPF_API void btf__free(struct btf *btf); - -LIBBPF_API struct btf *btf__new(const void *data, __u32 size); -LIBBPF_API struct btf *btf__new_split(const void *data, __u32 size, struct btf *base_btf); -LIBBPF_API struct btf *btf__new_empty(void); -LIBBPF_API struct btf *btf__new_empty_split(struct btf *base_btf); - -LIBBPF_API struct btf *btf__parse(const char *path, struct btf_ext **btf_ext); -LIBBPF_API struct btf *btf__parse_split(const char *path, struct btf *base_btf); -LIBBPF_API struct btf *btf__parse_elf(const char *path, struct btf_ext **btf_ext); -LIBBPF_API struct btf *btf__parse_elf_split(const char *path, struct btf *base_btf); -LIBBPF_API struct btf *btf__parse_raw(const char *path); -LIBBPF_API struct btf *btf__parse_raw_split(const char *path, struct btf *base_btf); - -LIBBPF_API struct btf *btf__load_vmlinux_btf(void); -LIBBPF_API struct btf *btf__load_module_btf(const char *module_name, struct btf *vmlinux_btf); -LIBBPF_API struct btf *libbpf_find_kernel_btf(void); - -LIBBPF_API struct btf *btf__load_from_kernel_by_id(__u32 id); -LIBBPF_API struct btf *btf__load_from_kernel_by_id_split(__u32 id, struct btf *base_btf); -LIBBPF_API int btf__get_from_id(__u32 id, struct btf **btf); - -LIBBPF_API int btf__finalize_data(struct bpf_object *obj, struct btf *btf); -LIBBPF_API int btf__load(struct btf *btf); -LIBBPF_API int btf__load_into_kernel(struct btf *btf); -LIBBPF_API __s32 btf__find_by_name(const struct btf *btf, - const char *type_name); -LIBBPF_API __s32 btf__find_by_name_kind(const struct btf *btf, - const char *type_name, __u32 kind); -LIBBPF_API __u32 btf__get_nr_types(const struct btf *btf); -LIBBPF_API const struct btf *btf__base_btf(const struct btf *btf); -LIBBPF_API const struct btf_type *btf__type_by_id(const struct btf *btf, - __u32 id); -LIBBPF_API size_t btf__pointer_size(const struct btf *btf); -LIBBPF_API int btf__set_pointer_size(struct btf *btf, size_t ptr_sz); -LIBBPF_API enum btf_endianness btf__endianness(const struct btf *btf); -LIBBPF_API int btf__set_endianness(struct btf *btf, enum btf_endianness endian); -LIBBPF_API __s64 btf__resolve_size(const struct btf *btf, __u32 type_id); -LIBBPF_API int btf__resolve_type(const struct btf *btf, __u32 type_id); -LIBBPF_API int btf__align_of(const struct btf *btf, __u32 id); -LIBBPF_API int btf__fd(const struct btf *btf); -LIBBPF_API void btf__set_fd(struct btf *btf, int fd); -LIBBPF_API const void *btf__get_raw_data(const struct btf *btf, __u32 *size); -LIBBPF_API const char *btf__name_by_offset(const struct btf *btf, __u32 offset); -LIBBPF_API const char *btf__str_by_offset(const struct btf *btf, __u32 offset); -LIBBPF_API int btf__get_map_kv_tids(const struct btf *btf, const char *map_name, - __u32 expected_key_size, - __u32 expected_value_size, - __u32 *key_type_id, __u32 *value_type_id); - -LIBBPF_API struct btf_ext *btf_ext__new(__u8 *data, __u32 size); -LIBBPF_API void btf_ext__free(struct btf_ext *btf_ext); -LIBBPF_API const void *btf_ext__get_raw_data(const struct btf_ext *btf_ext, - __u32 *size); -LIBBPF_API LIBBPF_DEPRECATED("btf_ext__reloc_func_info was never meant as a public API and has wrong assumptions embedded in it; it will be removed in the future libbpf versions") -int btf_ext__reloc_func_info(const struct btf *btf, - const struct btf_ext *btf_ext, - const char *sec_name, __u32 insns_cnt, - void **func_info, __u32 *cnt); -LIBBPF_API LIBBPF_DEPRECATED("btf_ext__reloc_line_info was never meant as a public API and has wrong assumptions embedded in it; it will be removed in the future libbpf versions") -int btf_ext__reloc_line_info(const struct btf *btf, - const struct btf_ext *btf_ext, - const char *sec_name, __u32 insns_cnt, - void **line_info, __u32 *cnt); -LIBBPF_API __u32 btf_ext__func_info_rec_size(const struct btf_ext *btf_ext); -LIBBPF_API __u32 btf_ext__line_info_rec_size(const struct btf_ext *btf_ext); - -LIBBPF_API int btf__find_str(struct btf *btf, const char *s); -LIBBPF_API int btf__add_str(struct btf *btf, const char *s); -LIBBPF_API int btf__add_type(struct btf *btf, const struct btf *src_btf, - const struct btf_type *src_type); - -LIBBPF_API int btf__add_int(struct btf *btf, const char *name, size_t byte_sz, int encoding); -LIBBPF_API int btf__add_float(struct btf *btf, const char *name, size_t byte_sz); -LIBBPF_API int btf__add_ptr(struct btf *btf, int ref_type_id); -LIBBPF_API int btf__add_array(struct btf *btf, - int index_type_id, int elem_type_id, __u32 nr_elems); -/* struct/union construction APIs */ -LIBBPF_API int btf__add_struct(struct btf *btf, const char *name, __u32 sz); -LIBBPF_API int btf__add_union(struct btf *btf, const char *name, __u32 sz); -LIBBPF_API int btf__add_field(struct btf *btf, const char *name, int field_type_id, - __u32 bit_offset, __u32 bit_size); - -/* enum construction APIs */ -LIBBPF_API int btf__add_enum(struct btf *btf, const char *name, __u32 bytes_sz); -LIBBPF_API int btf__add_enum_value(struct btf *btf, const char *name, __s64 value); - -enum btf_fwd_kind { - BTF_FWD_STRUCT = 0, - BTF_FWD_UNION = 1, - BTF_FWD_ENUM = 2, -}; - -LIBBPF_API int btf__add_fwd(struct btf *btf, const char *name, enum btf_fwd_kind fwd_kind); -LIBBPF_API int btf__add_typedef(struct btf *btf, const char *name, int ref_type_id); -LIBBPF_API int btf__add_volatile(struct btf *btf, int ref_type_id); -LIBBPF_API int btf__add_const(struct btf *btf, int ref_type_id); -LIBBPF_API int btf__add_restrict(struct btf *btf, int ref_type_id); - -/* func and func_proto construction APIs */ -LIBBPF_API int btf__add_func(struct btf *btf, const char *name, - enum btf_func_linkage linkage, int proto_type_id); -LIBBPF_API int btf__add_func_proto(struct btf *btf, int ret_type_id); -LIBBPF_API int btf__add_func_param(struct btf *btf, const char *name, int type_id); - -/* var & datasec construction APIs */ -LIBBPF_API int btf__add_var(struct btf *btf, const char *name, int linkage, int type_id); -LIBBPF_API int btf__add_datasec(struct btf *btf, const char *name, __u32 byte_sz); -LIBBPF_API int btf__add_datasec_var_info(struct btf *btf, int var_type_id, - __u32 offset, __u32 byte_sz); - -struct btf_dedup_opts { - unsigned int dedup_table_size; - bool dont_resolve_fwds; -}; - -LIBBPF_API int btf__dedup(struct btf *btf, struct btf_ext *btf_ext, - const struct btf_dedup_opts *opts); - -struct btf_dump; - -struct btf_dump_opts { - void *ctx; -}; - -typedef void (*btf_dump_printf_fn_t)(void *ctx, const char *fmt, va_list args); - -LIBBPF_API struct btf_dump *btf_dump__new(const struct btf *btf, - const struct btf_ext *btf_ext, - const struct btf_dump_opts *opts, - btf_dump_printf_fn_t printf_fn); -LIBBPF_API void btf_dump__free(struct btf_dump *d); - -LIBBPF_API int btf_dump__dump_type(struct btf_dump *d, __u32 id); - -struct btf_dump_emit_type_decl_opts { - /* size of this struct, for forward/backward compatiblity */ - size_t sz; - /* optional field name for type declaration, e.g.: - * - struct my_struct - * - void (*)(int) - * - char (*)[123] - */ - const char *field_name; - /* extra indentation level (in number of tabs) to emit for multi-line - * type declarations (e.g., anonymous struct); applies for lines - * starting from the second one (first line is assumed to have - * necessary indentation already - */ - int indent_level; - /* strip all the const/volatile/restrict mods */ - bool strip_mods; - size_t :0; -}; -#define btf_dump_emit_type_decl_opts__last_field strip_mods - -LIBBPF_API int -btf_dump__emit_type_decl(struct btf_dump *d, __u32 id, - const struct btf_dump_emit_type_decl_opts *opts); - - -struct btf_dump_type_data_opts { - /* size of this struct, for forward/backward compatibility */ - size_t sz; - const char *indent_str; - int indent_level; - /* below match "show" flags for bpf_show_snprintf() */ - bool compact; /* no newlines/indentation */ - bool skip_names; /* skip member/type names */ - bool emit_zeroes; /* show 0-valued fields */ - size_t :0; -}; -#define btf_dump_type_data_opts__last_field emit_zeroes - -LIBBPF_API int -btf_dump__dump_type_data(struct btf_dump *d, __u32 id, - const void *data, size_t data_sz, - const struct btf_dump_type_data_opts *opts); - -/* - * A set of helpers for easier BTF types handling - */ -static inline __u16 btf_kind(const struct btf_type *t) -{ - return BTF_INFO_KIND(t->info); -} - -static inline __u16 btf_vlen(const struct btf_type *t) -{ - return BTF_INFO_VLEN(t->info); -} - -static inline bool btf_kflag(const struct btf_type *t) -{ - return BTF_INFO_KFLAG(t->info); -} - -static inline bool btf_is_void(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_UNKN; -} - -static inline bool btf_is_int(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_INT; -} - -static inline bool btf_is_ptr(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_PTR; -} - -static inline bool btf_is_array(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_ARRAY; -} - -static inline bool btf_is_struct(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_STRUCT; -} - -static inline bool btf_is_union(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_UNION; -} - -static inline bool btf_is_composite(const struct btf_type *t) -{ - __u16 kind = btf_kind(t); - - return kind == BTF_KIND_STRUCT || kind == BTF_KIND_UNION; -} - -static inline bool btf_is_enum(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_ENUM; -} - -static inline bool btf_is_fwd(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_FWD; -} - -static inline bool btf_is_typedef(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_TYPEDEF; -} - -static inline bool btf_is_volatile(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_VOLATILE; -} - -static inline bool btf_is_const(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_CONST; -} - -static inline bool btf_is_restrict(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_RESTRICT; -} - -static inline bool btf_is_mod(const struct btf_type *t) -{ - __u16 kind = btf_kind(t); - - return kind == BTF_KIND_VOLATILE || - kind == BTF_KIND_CONST || - kind == BTF_KIND_RESTRICT; -} - -static inline bool btf_is_func(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_FUNC; -} - -static inline bool btf_is_func_proto(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_FUNC_PROTO; -} - -static inline bool btf_is_var(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_VAR; -} - -static inline bool btf_is_datasec(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_DATASEC; -} - -static inline bool btf_is_float(const struct btf_type *t) -{ - return btf_kind(t) == BTF_KIND_FLOAT; -} - -static inline __u8 btf_int_encoding(const struct btf_type *t) -{ - return BTF_INT_ENCODING(*(__u32 *)(t + 1)); -} - -static inline __u8 btf_int_offset(const struct btf_type *t) -{ - return BTF_INT_OFFSET(*(__u32 *)(t + 1)); -} - -static inline __u8 btf_int_bits(const struct btf_type *t) -{ - return BTF_INT_BITS(*(__u32 *)(t + 1)); -} - -static inline struct btf_array *btf_array(const struct btf_type *t) -{ - return (struct btf_array *)(t + 1); -} - -static inline struct btf_enum *btf_enum(const struct btf_type *t) -{ - return (struct btf_enum *)(t + 1); -} - -static inline struct btf_member *btf_members(const struct btf_type *t) -{ - return (struct btf_member *)(t + 1); -} - -/* Get bit offset of a member with specified index. */ -static inline __u32 btf_member_bit_offset(const struct btf_type *t, - __u32 member_idx) -{ - const struct btf_member *m = btf_members(t) + member_idx; - bool kflag = btf_kflag(t); - - return kflag ? BTF_MEMBER_BIT_OFFSET(m->offset) : m->offset; -} -/* - * Get bitfield size of a member, assuming t is BTF_KIND_STRUCT or - * BTF_KIND_UNION. If member is not a bitfield, zero is returned. - */ -static inline __u32 btf_member_bitfield_size(const struct btf_type *t, - __u32 member_idx) -{ - const struct btf_member *m = btf_members(t) + member_idx; - bool kflag = btf_kflag(t); - - return kflag ? BTF_MEMBER_BITFIELD_SIZE(m->offset) : 0; -} - -static inline struct btf_param *btf_params(const struct btf_type *t) -{ - return (struct btf_param *)(t + 1); -} - -static inline struct btf_var *btf_var(const struct btf_type *t) -{ - return (struct btf_var *)(t + 1); -} - -static inline struct btf_var_secinfo * -btf_var_secinfos(const struct btf_type *t) -{ - return (struct btf_var_secinfo *)(t + 1); -} - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* __LIBBPF_BTF_H */ diff --git a/internal/include/libbpf/libbpf.h b/internal/include/libbpf/libbpf.h deleted file mode 100644 index 2f6f0e15d..000000000 --- a/internal/include/libbpf/libbpf.h +++ /dev/null @@ -1,922 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -/* - * Common eBPF ELF object loading operations. - * - * Copyright (C) 2013-2015 Alexei Starovoitov - * Copyright (C) 2015 Wang Nan - * Copyright (C) 2015 Huawei Inc. - */ -#ifndef __LIBBPF_LIBBPF_H -#define __LIBBPF_LIBBPF_H - -#include -#include -#include -#include -#include // for size_t -#include - -#include "libbpf_common.h" -#include "libbpf_legacy.h" - -#ifdef __cplusplus -extern "C" { -#endif - -enum libbpf_errno { - __LIBBPF_ERRNO__START = 4000, - - /* Something wrong in libelf */ - LIBBPF_ERRNO__LIBELF = __LIBBPF_ERRNO__START, - LIBBPF_ERRNO__FORMAT, /* BPF object format invalid */ - LIBBPF_ERRNO__KVERSION, /* Incorrect or no 'version' section */ - LIBBPF_ERRNO__ENDIAN, /* Endian mismatch */ - LIBBPF_ERRNO__INTERNAL, /* Internal error in libbpf */ - LIBBPF_ERRNO__RELOC, /* Relocation failed */ - LIBBPF_ERRNO__LOAD, /* Load program failure for unknown reason */ - LIBBPF_ERRNO__VERIFY, /* Kernel verifier blocks program loading */ - LIBBPF_ERRNO__PROG2BIG, /* Program too big */ - LIBBPF_ERRNO__KVER, /* Incorrect kernel version */ - LIBBPF_ERRNO__PROGTYPE, /* Kernel doesn't support this program type */ - LIBBPF_ERRNO__WRNGPID, /* Wrong pid in netlink message */ - LIBBPF_ERRNO__INVSEQ, /* Invalid netlink sequence */ - LIBBPF_ERRNO__NLPARSE, /* netlink parsing error */ - __LIBBPF_ERRNO__END, -}; - -LIBBPF_API int libbpf_strerror(int err, char *buf, size_t size); - -enum libbpf_print_level { - LIBBPF_WARN, - LIBBPF_INFO, - LIBBPF_DEBUG, -}; - -typedef int (*libbpf_print_fn_t)(enum libbpf_print_level level, - const char *, va_list ap); - -LIBBPF_API libbpf_print_fn_t libbpf_set_print(libbpf_print_fn_t fn); - -/* Hide internal to user */ -struct bpf_object; - -struct bpf_object_open_attr { - const char *file; - enum bpf_prog_type prog_type; -}; - -struct bpf_object_open_opts { - /* size of this struct, for forward/backward compatiblity */ - size_t sz; - /* object name override, if provided: - * - for object open from file, this will override setting object - * name from file path's base name; - * - for object open from memory buffer, this will specify an object - * name and will override default "-" name; - */ - const char *object_name; - /* parse map definitions non-strictly, allowing extra attributes/data */ - bool relaxed_maps; - /* DEPRECATED: handle CO-RE relocations non-strictly, allowing failures. - * Value is ignored. Relocations always are processed non-strictly. - * Non-relocatable instructions are replaced with invalid ones to - * prevent accidental errors. - * */ - bool relaxed_core_relocs; - /* maps that set the 'pinning' attribute in their definition will have - * their pin_path attribute set to a file in this directory, and be - * auto-pinned to that path on load; defaults to "/sys/fs/bpf". - */ - const char *pin_root_path; - __u32 attach_prog_fd; - /* Additional kernel config content that augments and overrides - * system Kconfig for CONFIG_xxx externs. - */ - const char *kconfig; - /* Path to the custom BTF to be used for BPF CO-RE relocations. - * This custom BTF completely replaces the use of vmlinux BTF - * for the purpose of CO-RE relocations. - * NOTE: any other BPF feature (e.g., fentry/fexit programs, - * struct_ops, etc) will need actual kernel BTF at /sys/kernel/btf/vmlinux. - */ - const char *btf_custom_path; -}; -#define bpf_object_open_opts__last_field btf_custom_path - -LIBBPF_API struct bpf_object *bpf_object__open(const char *path); -LIBBPF_API struct bpf_object * -bpf_object__open_file(const char *path, const struct bpf_object_open_opts *opts); -LIBBPF_API struct bpf_object * -bpf_object__open_mem(const void *obj_buf, size_t obj_buf_sz, - const struct bpf_object_open_opts *opts); - -/* deprecated bpf_object__open variants */ -LIBBPF_API struct bpf_object * -bpf_object__open_buffer(const void *obj_buf, size_t obj_buf_sz, - const char *name); -LIBBPF_API struct bpf_object * -bpf_object__open_xattr(struct bpf_object_open_attr *attr); - -enum libbpf_pin_type { - LIBBPF_PIN_NONE, - /* PIN_BY_NAME: pin maps by name (in /sys/fs/bpf by default) */ - LIBBPF_PIN_BY_NAME, -}; - -/* pin_maps and unpin_maps can both be called with a NULL path, in which case - * they will use the pin_path attribute of each map (and ignore all maps that - * don't have a pin_path set). - */ -LIBBPF_API int bpf_object__pin_maps(struct bpf_object *obj, const char *path); -LIBBPF_API int bpf_object__unpin_maps(struct bpf_object *obj, - const char *path); -LIBBPF_API int bpf_object__pin_programs(struct bpf_object *obj, - const char *path); -LIBBPF_API int bpf_object__unpin_programs(struct bpf_object *obj, - const char *path); -LIBBPF_API int bpf_object__pin(struct bpf_object *object, const char *path); -LIBBPF_API void bpf_object__close(struct bpf_object *object); - -struct bpf_object_load_attr { - struct bpf_object *obj; - int log_level; - const char *target_btf_path; -}; - -/* Load/unload object into/from kernel */ -LIBBPF_API int bpf_object__load(struct bpf_object *obj); -LIBBPF_API int bpf_object__load_xattr(struct bpf_object_load_attr *attr); -LIBBPF_API int bpf_object__unload(struct bpf_object *obj); - -LIBBPF_API const char *bpf_object__name(const struct bpf_object *obj); -LIBBPF_API unsigned int bpf_object__kversion(const struct bpf_object *obj); -LIBBPF_API int bpf_object__set_kversion(struct bpf_object *obj, __u32 kern_version); - -struct btf; -LIBBPF_API struct btf *bpf_object__btf(const struct bpf_object *obj); -LIBBPF_API int bpf_object__btf_fd(const struct bpf_object *obj); - -LIBBPF_API struct bpf_program * -bpf_object__find_program_by_title(const struct bpf_object *obj, - const char *title); -LIBBPF_API struct bpf_program * -bpf_object__find_program_by_name(const struct bpf_object *obj, - const char *name); - -LIBBPF_API struct bpf_object *bpf_object__next(struct bpf_object *prev); -#define bpf_object__for_each_safe(pos, tmp) \ - for ((pos) = bpf_object__next(NULL), \ - (tmp) = bpf_object__next(pos); \ - (pos) != NULL; \ - (pos) = (tmp), (tmp) = bpf_object__next(tmp)) - -typedef void (*bpf_object_clear_priv_t)(struct bpf_object *, void *); -LIBBPF_API int bpf_object__set_priv(struct bpf_object *obj, void *priv, - bpf_object_clear_priv_t clear_priv); -LIBBPF_API void *bpf_object__priv(const struct bpf_object *prog); - -LIBBPF_API int -libbpf_prog_type_by_name(const char *name, enum bpf_prog_type *prog_type, - enum bpf_attach_type *expected_attach_type); -LIBBPF_API int libbpf_attach_type_by_name(const char *name, - enum bpf_attach_type *attach_type); -LIBBPF_API int libbpf_find_vmlinux_btf_id(const char *name, - enum bpf_attach_type attach_type); - -/* Accessors of bpf_program */ -struct bpf_program; -LIBBPF_API struct bpf_program *bpf_program__next(struct bpf_program *prog, - const struct bpf_object *obj); - -#define bpf_object__for_each_program(pos, obj) \ - for ((pos) = bpf_program__next(NULL, (obj)); \ - (pos) != NULL; \ - (pos) = bpf_program__next((pos), (obj))) - -LIBBPF_API struct bpf_program *bpf_program__prev(struct bpf_program *prog, - const struct bpf_object *obj); - -typedef void (*bpf_program_clear_priv_t)(struct bpf_program *, void *); - -LIBBPF_API int bpf_program__set_priv(struct bpf_program *prog, void *priv, - bpf_program_clear_priv_t clear_priv); - -LIBBPF_API void *bpf_program__priv(const struct bpf_program *prog); -LIBBPF_API void bpf_program__set_ifindex(struct bpf_program *prog, - __u32 ifindex); - -LIBBPF_API const char *bpf_program__name(const struct bpf_program *prog); -LIBBPF_API const char *bpf_program__section_name(const struct bpf_program *prog); -LIBBPF_API LIBBPF_DEPRECATED("BPF program title is confusing term; please use bpf_program__section_name() instead") -const char *bpf_program__title(const struct bpf_program *prog, bool needs_copy); -LIBBPF_API bool bpf_program__autoload(const struct bpf_program *prog); -LIBBPF_API int bpf_program__set_autoload(struct bpf_program *prog, bool autoload); - -/* returns program size in bytes */ -LIBBPF_API size_t bpf_program__size(const struct bpf_program *prog); - -LIBBPF_API int bpf_program__load(struct bpf_program *prog, char *license, - __u32 kern_version); -LIBBPF_API int bpf_program__fd(const struct bpf_program *prog); -LIBBPF_API int bpf_program__pin_instance(struct bpf_program *prog, - const char *path, - int instance); -LIBBPF_API int bpf_program__unpin_instance(struct bpf_program *prog, - const char *path, - int instance); -LIBBPF_API int bpf_program__pin(struct bpf_program *prog, const char *path); -LIBBPF_API int bpf_program__unpin(struct bpf_program *prog, const char *path); -LIBBPF_API void bpf_program__unload(struct bpf_program *prog); - -struct bpf_link; - -LIBBPF_API struct bpf_link *bpf_link__open(const char *path); -LIBBPF_API int bpf_link__fd(const struct bpf_link *link); -LIBBPF_API const char *bpf_link__pin_path(const struct bpf_link *link); -LIBBPF_API int bpf_link__pin(struct bpf_link *link, const char *path); -LIBBPF_API int bpf_link__unpin(struct bpf_link *link); -LIBBPF_API int bpf_link__update_program(struct bpf_link *link, - struct bpf_program *prog); -LIBBPF_API void bpf_link__disconnect(struct bpf_link *link); -LIBBPF_API int bpf_link__detach(struct bpf_link *link); -LIBBPF_API int bpf_link__destroy(struct bpf_link *link); - -LIBBPF_API struct bpf_link * -bpf_program__attach(struct bpf_program *prog); - -struct bpf_perf_event_opts { - /* size of this struct, for forward/backward compatiblity */ - size_t sz; - /* custom user-provided value fetchable through bpf_get_attach_cookie() */ - __u64 bpf_cookie; -}; -#define bpf_perf_event_opts__last_field bpf_cookie - -LIBBPF_API struct bpf_link * -bpf_program__attach_perf_event(struct bpf_program *prog, int pfd); - -LIBBPF_API struct bpf_link * -bpf_program__attach_perf_event_opts(struct bpf_program *prog, int pfd, - const struct bpf_perf_event_opts *opts); - -struct bpf_kprobe_opts { - /* size of this struct, for forward/backward compatiblity */ - size_t sz; - /* custom user-provided value fetchable through bpf_get_attach_cookie() */ - __u64 bpf_cookie; - /* function's offset to install kprobe to */ - unsigned long offset; - /* kprobe is return probe */ - bool retprobe; - size_t :0; -}; -#define bpf_kprobe_opts__last_field retprobe - -LIBBPF_API struct bpf_link * -bpf_program__attach_kprobe(struct bpf_program *prog, bool retprobe, - const char *func_name); -LIBBPF_API struct bpf_link * -bpf_program__attach_kprobe_opts(struct bpf_program *prog, - const char *func_name, - const struct bpf_kprobe_opts *opts); - -struct bpf_uprobe_opts { - /* size of this struct, for forward/backward compatiblity */ - size_t sz; - /* offset of kernel reference counted USDT semaphore, added in - * a6ca88b241d5 ("trace_uprobe: support reference counter in fd-based uprobe") - */ - size_t ref_ctr_offset; - /* custom user-provided value fetchable through bpf_get_attach_cookie() */ - __u64 bpf_cookie; - /* uprobe is return probe, invoked at function return time */ - bool retprobe; - size_t :0; -}; -#define bpf_uprobe_opts__last_field retprobe - -LIBBPF_API struct bpf_link * -bpf_program__attach_uprobe(struct bpf_program *prog, bool retprobe, - pid_t pid, const char *binary_path, - size_t func_offset); -LIBBPF_API struct bpf_link * -bpf_program__attach_uprobe_opts(struct bpf_program *prog, pid_t pid, - const char *binary_path, size_t func_offset, - const struct bpf_uprobe_opts *opts); - -struct bpf_tracepoint_opts { - /* size of this struct, for forward/backward compatiblity */ - size_t sz; - /* custom user-provided value fetchable through bpf_get_attach_cookie() */ - __u64 bpf_cookie; -}; -#define bpf_tracepoint_opts__last_field bpf_cookie - -LIBBPF_API struct bpf_link * -bpf_program__attach_tracepoint(struct bpf_program *prog, - const char *tp_category, - const char *tp_name); -LIBBPF_API struct bpf_link * -bpf_program__attach_tracepoint_opts(struct bpf_program *prog, - const char *tp_category, - const char *tp_name, - const struct bpf_tracepoint_opts *opts); - -LIBBPF_API struct bpf_link * -bpf_program__attach_raw_tracepoint(struct bpf_program *prog, - const char *tp_name); -LIBBPF_API struct bpf_link * -bpf_program__attach_trace(struct bpf_program *prog); -LIBBPF_API struct bpf_link * -bpf_program__attach_lsm(struct bpf_program *prog); -LIBBPF_API struct bpf_link * -bpf_program__attach_cgroup(struct bpf_program *prog, int cgroup_fd); -LIBBPF_API struct bpf_link * -bpf_program__attach_netns(struct bpf_program *prog, int netns_fd); -LIBBPF_API struct bpf_link * -bpf_program__attach_xdp(struct bpf_program *prog, int ifindex); -LIBBPF_API struct bpf_link * -bpf_program__attach_freplace(struct bpf_program *prog, - int target_fd, const char *attach_func_name); - -struct bpf_map; - -LIBBPF_API struct bpf_link *bpf_map__attach_struct_ops(struct bpf_map *map); - -struct bpf_iter_attach_opts { - size_t sz; /* size of this struct for forward/backward compatibility */ - union bpf_iter_link_info *link_info; - __u32 link_info_len; -}; -#define bpf_iter_attach_opts__last_field link_info_len - -LIBBPF_API struct bpf_link * -bpf_program__attach_iter(struct bpf_program *prog, - const struct bpf_iter_attach_opts *opts); - -struct bpf_insn; - -/* - * Libbpf allows callers to adjust BPF programs before being loaded - * into kernel. One program in an object file can be transformed into - * multiple variants to be attached to different hooks. - * - * bpf_program_prep_t, bpf_program__set_prep and bpf_program__nth_fd - * form an API for this purpose. - * - * - bpf_program_prep_t: - * Defines a 'preprocessor', which is a caller defined function - * passed to libbpf through bpf_program__set_prep(), and will be - * called before program is loaded. The processor should adjust - * the program one time for each instance according to the instance id - * passed to it. - * - * - bpf_program__set_prep: - * Attaches a preprocessor to a BPF program. The number of instances - * that should be created is also passed through this function. - * - * - bpf_program__nth_fd: - * After the program is loaded, get resulting FD of a given instance - * of the BPF program. - * - * If bpf_program__set_prep() is not used, the program would be loaded - * without adjustment during bpf_object__load(). The program has only - * one instance. In this case bpf_program__fd(prog) is equal to - * bpf_program__nth_fd(prog, 0). - */ - -struct bpf_prog_prep_result { - /* - * If not NULL, load new instruction array. - * If set to NULL, don't load this instance. - */ - struct bpf_insn *new_insn_ptr; - int new_insn_cnt; - - /* If not NULL, result FD is written to it. */ - int *pfd; -}; - -/* - * Parameters of bpf_program_prep_t: - * - prog: The bpf_program being loaded. - * - n: Index of instance being generated. - * - insns: BPF instructions array. - * - insns_cnt:Number of instructions in insns. - * - res: Output parameter, result of transformation. - * - * Return value: - * - Zero: pre-processing success. - * - Non-zero: pre-processing error, stop loading. - */ -typedef int (*bpf_program_prep_t)(struct bpf_program *prog, int n, - struct bpf_insn *insns, int insns_cnt, - struct bpf_prog_prep_result *res); - -LIBBPF_API int bpf_program__set_prep(struct bpf_program *prog, int nr_instance, - bpf_program_prep_t prep); - -LIBBPF_API int bpf_program__nth_fd(const struct bpf_program *prog, int n); - -/* - * Adjust type of BPF program. Default is kprobe. - */ -LIBBPF_API int bpf_program__set_socket_filter(struct bpf_program *prog); -LIBBPF_API int bpf_program__set_tracepoint(struct bpf_program *prog); -LIBBPF_API int bpf_program__set_raw_tracepoint(struct bpf_program *prog); -LIBBPF_API int bpf_program__set_kprobe(struct bpf_program *prog); -LIBBPF_API int bpf_program__set_lsm(struct bpf_program *prog); -LIBBPF_API int bpf_program__set_sched_cls(struct bpf_program *prog); -LIBBPF_API int bpf_program__set_sched_act(struct bpf_program *prog); -LIBBPF_API int bpf_program__set_xdp(struct bpf_program *prog); -LIBBPF_API int bpf_program__set_perf_event(struct bpf_program *prog); -LIBBPF_API int bpf_program__set_tracing(struct bpf_program *prog); -LIBBPF_API int bpf_program__set_struct_ops(struct bpf_program *prog); -LIBBPF_API int bpf_program__set_extension(struct bpf_program *prog); -LIBBPF_API int bpf_program__set_sk_lookup(struct bpf_program *prog); - -LIBBPF_API enum bpf_prog_type bpf_program__get_type(const struct bpf_program *prog); -LIBBPF_API void bpf_program__set_type(struct bpf_program *prog, - enum bpf_prog_type type); - -LIBBPF_API enum bpf_attach_type -bpf_program__get_expected_attach_type(const struct bpf_program *prog); -LIBBPF_API void -bpf_program__set_expected_attach_type(struct bpf_program *prog, - enum bpf_attach_type type); - -LIBBPF_API int -bpf_program__set_attach_target(struct bpf_program *prog, int attach_prog_fd, - const char *attach_func_name); - -LIBBPF_API bool bpf_program__is_socket_filter(const struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_tracepoint(const struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_raw_tracepoint(const struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_kprobe(const struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_lsm(const struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_sched_cls(const struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_sched_act(const struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_xdp(const struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_perf_event(const struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_tracing(const struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_struct_ops(const struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_extension(const struct bpf_program *prog); -LIBBPF_API bool bpf_program__is_sk_lookup(const struct bpf_program *prog); - -/* - * No need for __attribute__((packed)), all members of 'bpf_map_def' - * are all aligned. In addition, using __attribute__((packed)) - * would trigger a -Wpacked warning message, and lead to an error - * if -Werror is set. - */ -struct bpf_map_def { - unsigned int type; - unsigned int key_size; - unsigned int value_size; - unsigned int max_entries; - unsigned int map_flags; -}; - -/* - * The 'struct bpf_map' in include/linux/bpf.h is internal to the kernel, - * so no need to worry about a name clash. - */ -LIBBPF_API struct bpf_map * -bpf_object__find_map_by_name(const struct bpf_object *obj, const char *name); - -LIBBPF_API int -bpf_object__find_map_fd_by_name(const struct bpf_object *obj, const char *name); - -/* - * Get bpf_map through the offset of corresponding struct bpf_map_def - * in the BPF object file. - */ -LIBBPF_API struct bpf_map * -bpf_object__find_map_by_offset(struct bpf_object *obj, size_t offset); - -LIBBPF_API struct bpf_map * -bpf_map__next(const struct bpf_map *map, const struct bpf_object *obj); -#define bpf_object__for_each_map(pos, obj) \ - for ((pos) = bpf_map__next(NULL, (obj)); \ - (pos) != NULL; \ - (pos) = bpf_map__next((pos), (obj))) -#define bpf_map__for_each bpf_object__for_each_map - -LIBBPF_API struct bpf_map * -bpf_map__prev(const struct bpf_map *map, const struct bpf_object *obj); - -/* get/set map FD */ -LIBBPF_API int bpf_map__fd(const struct bpf_map *map); -LIBBPF_API int bpf_map__reuse_fd(struct bpf_map *map, int fd); -/* get map definition */ -LIBBPF_API const struct bpf_map_def *bpf_map__def(const struct bpf_map *map); -/* get map name */ -LIBBPF_API const char *bpf_map__name(const struct bpf_map *map); -/* get/set map type */ -LIBBPF_API enum bpf_map_type bpf_map__type(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_type(struct bpf_map *map, enum bpf_map_type type); -/* get/set map size (max_entries) */ -LIBBPF_API __u32 bpf_map__max_entries(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_max_entries(struct bpf_map *map, __u32 max_entries); -LIBBPF_API int bpf_map__resize(struct bpf_map *map, __u32 max_entries); -/* get/set map flags */ -LIBBPF_API __u32 bpf_map__map_flags(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_map_flags(struct bpf_map *map, __u32 flags); -/* get/set map NUMA node */ -LIBBPF_API __u32 bpf_map__numa_node(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_numa_node(struct bpf_map *map, __u32 numa_node); -/* get/set map key size */ -LIBBPF_API __u32 bpf_map__key_size(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_key_size(struct bpf_map *map, __u32 size); -/* get/set map value size */ -LIBBPF_API __u32 bpf_map__value_size(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_value_size(struct bpf_map *map, __u32 size); -/* get map key/value BTF type IDs */ -LIBBPF_API __u32 bpf_map__btf_key_type_id(const struct bpf_map *map); -LIBBPF_API __u32 bpf_map__btf_value_type_id(const struct bpf_map *map); -/* get/set map if_index */ -LIBBPF_API __u32 bpf_map__ifindex(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_ifindex(struct bpf_map *map, __u32 ifindex); - -typedef void (*bpf_map_clear_priv_t)(struct bpf_map *, void *); -LIBBPF_API int bpf_map__set_priv(struct bpf_map *map, void *priv, - bpf_map_clear_priv_t clear_priv); -LIBBPF_API void *bpf_map__priv(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_initial_value(struct bpf_map *map, - const void *data, size_t size); -LIBBPF_API const void *bpf_map__initial_value(struct bpf_map *map, size_t *psize); -LIBBPF_API bool bpf_map__is_offload_neutral(const struct bpf_map *map); -LIBBPF_API bool bpf_map__is_internal(const struct bpf_map *map); -LIBBPF_API int bpf_map__set_pin_path(struct bpf_map *map, const char *path); -LIBBPF_API const char *bpf_map__get_pin_path(const struct bpf_map *map); -LIBBPF_API const char *bpf_map__pin_path(const struct bpf_map *map); -LIBBPF_API bool bpf_map__is_pinned(const struct bpf_map *map); -LIBBPF_API int bpf_map__pin(struct bpf_map *map, const char *path); -LIBBPF_API int bpf_map__unpin(struct bpf_map *map, const char *path); - -LIBBPF_API int bpf_map__set_inner_map_fd(struct bpf_map *map, int fd); -LIBBPF_API struct bpf_map *bpf_map__inner_map(struct bpf_map *map); - -LIBBPF_API long libbpf_get_error(const void *ptr); - -struct bpf_prog_load_attr { - const char *file; - enum bpf_prog_type prog_type; - enum bpf_attach_type expected_attach_type; - int ifindex; - int log_level; - int prog_flags; -}; - -LIBBPF_API int bpf_prog_load_xattr(const struct bpf_prog_load_attr *attr, - struct bpf_object **pobj, int *prog_fd); -LIBBPF_API int bpf_prog_load(const char *file, enum bpf_prog_type type, - struct bpf_object **pobj, int *prog_fd); - -/* XDP related API */ -struct xdp_link_info { - __u32 prog_id; - __u32 drv_prog_id; - __u32 hw_prog_id; - __u32 skb_prog_id; - __u8 attach_mode; -}; - -struct bpf_xdp_set_link_opts { - size_t sz; - int old_fd; - size_t :0; -}; -#define bpf_xdp_set_link_opts__last_field old_fd - -LIBBPF_API int bpf_set_link_xdp_fd(int ifindex, int fd, __u32 flags); -LIBBPF_API int bpf_set_link_xdp_fd_opts(int ifindex, int fd, __u32 flags, - const struct bpf_xdp_set_link_opts *opts); -LIBBPF_API int bpf_get_link_xdp_id(int ifindex, __u32 *prog_id, __u32 flags); -LIBBPF_API int bpf_get_link_xdp_info(int ifindex, struct xdp_link_info *info, - size_t info_size, __u32 flags); - -/* TC related API */ -enum bpf_tc_attach_point { - BPF_TC_INGRESS = 1 << 0, - BPF_TC_EGRESS = 1 << 1, - BPF_TC_CUSTOM = 1 << 2, -}; - -#define BPF_TC_PARENT(a, b) \ - ((((a) << 16) & 0xFFFF0000U) | ((b) & 0x0000FFFFU)) - -enum bpf_tc_flags { - BPF_TC_F_REPLACE = 1 << 0, -}; - -struct bpf_tc_hook { - size_t sz; - int ifindex; - enum bpf_tc_attach_point attach_point; - __u32 parent; - size_t :0; -}; -#define bpf_tc_hook__last_field parent - -struct bpf_tc_opts { - size_t sz; - int prog_fd; - __u32 flags; - __u32 prog_id; - __u32 handle; - __u32 priority; - size_t :0; -}; -#define bpf_tc_opts__last_field priority - -LIBBPF_API int bpf_tc_hook_create(struct bpf_tc_hook *hook); -LIBBPF_API int bpf_tc_hook_destroy(struct bpf_tc_hook *hook); -LIBBPF_API int bpf_tc_attach(const struct bpf_tc_hook *hook, - struct bpf_tc_opts *opts); -LIBBPF_API int bpf_tc_detach(const struct bpf_tc_hook *hook, - const struct bpf_tc_opts *opts); -LIBBPF_API int bpf_tc_query(const struct bpf_tc_hook *hook, - struct bpf_tc_opts *opts); - -/* Ring buffer APIs */ -struct ring_buffer; - -typedef int (*ring_buffer_sample_fn)(void *ctx, void *data, size_t size); - -struct ring_buffer_opts { - size_t sz; /* size of this struct, for forward/backward compatiblity */ -}; - -#define ring_buffer_opts__last_field sz - -LIBBPF_API struct ring_buffer * -ring_buffer__new(int map_fd, ring_buffer_sample_fn sample_cb, void *ctx, - const struct ring_buffer_opts *opts); -LIBBPF_API void ring_buffer__free(struct ring_buffer *rb); -LIBBPF_API int ring_buffer__add(struct ring_buffer *rb, int map_fd, - ring_buffer_sample_fn sample_cb, void *ctx); -LIBBPF_API int ring_buffer__poll(struct ring_buffer *rb, int timeout_ms); -LIBBPF_API int ring_buffer__consume(struct ring_buffer *rb); -LIBBPF_API int ring_buffer__epoll_fd(const struct ring_buffer *rb); - -/* Perf buffer APIs */ -struct perf_buffer; - -typedef void (*perf_buffer_sample_fn)(void *ctx, int cpu, - void *data, __u32 size); -typedef void (*perf_buffer_lost_fn)(void *ctx, int cpu, __u64 cnt); - -/* common use perf buffer options */ -struct perf_buffer_opts { - /* if specified, sample_cb is called for each sample */ - perf_buffer_sample_fn sample_cb; - /* if specified, lost_cb is called for each batch of lost samples */ - perf_buffer_lost_fn lost_cb; - /* ctx is provided to sample_cb and lost_cb */ - void *ctx; -}; - -LIBBPF_API struct perf_buffer * -perf_buffer__new(int map_fd, size_t page_cnt, - const struct perf_buffer_opts *opts); - -enum bpf_perf_event_ret { - LIBBPF_PERF_EVENT_DONE = 0, - LIBBPF_PERF_EVENT_ERROR = -1, - LIBBPF_PERF_EVENT_CONT = -2, -}; - -struct perf_event_header; - -typedef enum bpf_perf_event_ret -(*perf_buffer_event_fn)(void *ctx, int cpu, struct perf_event_header *event); - -/* raw perf buffer options, giving most power and control */ -struct perf_buffer_raw_opts { - /* perf event attrs passed directly into perf_event_open() */ - struct perf_event_attr *attr; - /* raw event callback */ - perf_buffer_event_fn event_cb; - /* ctx is provided to event_cb */ - void *ctx; - /* if cpu_cnt == 0, open all on all possible CPUs (up to the number of - * max_entries of given PERF_EVENT_ARRAY map) - */ - int cpu_cnt; - /* if cpu_cnt > 0, cpus is an array of CPUs to open ring buffers on */ - int *cpus; - /* if cpu_cnt > 0, map_keys specify map keys to set per-CPU FDs for */ - int *map_keys; -}; - -LIBBPF_API struct perf_buffer * -perf_buffer__new_raw(int map_fd, size_t page_cnt, - const struct perf_buffer_raw_opts *opts); - -LIBBPF_API void perf_buffer__free(struct perf_buffer *pb); -LIBBPF_API int perf_buffer__epoll_fd(const struct perf_buffer *pb); -LIBBPF_API int perf_buffer__poll(struct perf_buffer *pb, int timeout_ms); -LIBBPF_API int perf_buffer__consume(struct perf_buffer *pb); -LIBBPF_API int perf_buffer__consume_buffer(struct perf_buffer *pb, size_t buf_idx); -LIBBPF_API size_t perf_buffer__buffer_cnt(const struct perf_buffer *pb); -LIBBPF_API int perf_buffer__buffer_fd(const struct perf_buffer *pb, size_t buf_idx); - -typedef enum bpf_perf_event_ret - (*bpf_perf_event_print_t)(struct perf_event_header *hdr, - void *private_data); -LIBBPF_API enum bpf_perf_event_ret -bpf_perf_event_read_simple(void *mmap_mem, size_t mmap_size, size_t page_size, - void **copy_mem, size_t *copy_size, - bpf_perf_event_print_t fn, void *private_data); - -struct bpf_prog_linfo; -struct bpf_prog_info; - -LIBBPF_API void bpf_prog_linfo__free(struct bpf_prog_linfo *prog_linfo); -LIBBPF_API struct bpf_prog_linfo * -bpf_prog_linfo__new(const struct bpf_prog_info *info); -LIBBPF_API const struct bpf_line_info * -bpf_prog_linfo__lfind_addr_func(const struct bpf_prog_linfo *prog_linfo, - __u64 addr, __u32 func_idx, __u32 nr_skip); -LIBBPF_API const struct bpf_line_info * -bpf_prog_linfo__lfind(const struct bpf_prog_linfo *prog_linfo, - __u32 insn_off, __u32 nr_skip); - -/* - * Probe for supported system features - * - * Note that running many of these probes in a short amount of time can cause - * the kernel to reach the maximal size of lockable memory allowed for the - * user, causing subsequent probes to fail. In this case, the caller may want - * to adjust that limit with setrlimit(). - */ -LIBBPF_API bool bpf_probe_prog_type(enum bpf_prog_type prog_type, - __u32 ifindex); -LIBBPF_API bool bpf_probe_map_type(enum bpf_map_type map_type, __u32 ifindex); -LIBBPF_API bool bpf_probe_helper(enum bpf_func_id id, - enum bpf_prog_type prog_type, __u32 ifindex); -LIBBPF_API bool bpf_probe_large_insn_limit(__u32 ifindex); - -/* - * Get bpf_prog_info in continuous memory - * - * struct bpf_prog_info has multiple arrays. The user has option to choose - * arrays to fetch from kernel. The following APIs provide an uniform way to - * fetch these data. All arrays in bpf_prog_info are stored in a single - * continuous memory region. This makes it easy to store the info in a - * file. - * - * Before writing bpf_prog_info_linear to files, it is necessary to - * translate pointers in bpf_prog_info to offsets. Helper functions - * bpf_program__bpil_addr_to_offs() and bpf_program__bpil_offs_to_addr() - * are introduced to switch between pointers and offsets. - * - * Examples: - * # To fetch map_ids and prog_tags: - * __u64 arrays = (1UL << BPF_PROG_INFO_MAP_IDS) | - * (1UL << BPF_PROG_INFO_PROG_TAGS); - * struct bpf_prog_info_linear *info_linear = - * bpf_program__get_prog_info_linear(fd, arrays); - * - * # To save data in file - * bpf_program__bpil_addr_to_offs(info_linear); - * write(f, info_linear, sizeof(*info_linear) + info_linear->data_len); - * - * # To read data from file - * read(f, info_linear, ); - * bpf_program__bpil_offs_to_addr(info_linear); - */ -enum bpf_prog_info_array { - BPF_PROG_INFO_FIRST_ARRAY = 0, - BPF_PROG_INFO_JITED_INSNS = 0, - BPF_PROG_INFO_XLATED_INSNS, - BPF_PROG_INFO_MAP_IDS, - BPF_PROG_INFO_JITED_KSYMS, - BPF_PROG_INFO_JITED_FUNC_LENS, - BPF_PROG_INFO_FUNC_INFO, - BPF_PROG_INFO_LINE_INFO, - BPF_PROG_INFO_JITED_LINE_INFO, - BPF_PROG_INFO_PROG_TAGS, - BPF_PROG_INFO_LAST_ARRAY, -}; - -struct bpf_prog_info_linear { - /* size of struct bpf_prog_info, when the tool is compiled */ - __u32 info_len; - /* total bytes allocated for data, round up to 8 bytes */ - __u32 data_len; - /* which arrays are included in data */ - __u64 arrays; - struct bpf_prog_info info; - __u8 data[]; -}; - -LIBBPF_API struct bpf_prog_info_linear * -bpf_program__get_prog_info_linear(int fd, __u64 arrays); - -LIBBPF_API void -bpf_program__bpil_addr_to_offs(struct bpf_prog_info_linear *info_linear); - -LIBBPF_API void -bpf_program__bpil_offs_to_addr(struct bpf_prog_info_linear *info_linear); - -/* - * A helper function to get the number of possible CPUs before looking up - * per-CPU maps. Negative errno is returned on failure. - * - * Example usage: - * - * int ncpus = libbpf_num_possible_cpus(); - * if (ncpus < 0) { - * // error handling - * } - * long values[ncpus]; - * bpf_map_lookup_elem(per_cpu_map_fd, key, values); - * - */ -LIBBPF_API int libbpf_num_possible_cpus(void); - -struct bpf_map_skeleton { - const char *name; - struct bpf_map **map; - void **mmaped; -}; - -struct bpf_prog_skeleton { - const char *name; - struct bpf_program **prog; - struct bpf_link **link; -}; - -struct bpf_object_skeleton { - size_t sz; /* size of this struct, for forward/backward compatibility */ - - const char *name; - const void *data; - size_t data_sz; - - struct bpf_object **obj; - - int map_cnt; - int map_skel_sz; /* sizeof(struct bpf_skeleton_map) */ - struct bpf_map_skeleton *maps; - - int prog_cnt; - int prog_skel_sz; /* sizeof(struct bpf_skeleton_prog) */ - struct bpf_prog_skeleton *progs; -}; - -LIBBPF_API int -bpf_object__open_skeleton(struct bpf_object_skeleton *s, - const struct bpf_object_open_opts *opts); -LIBBPF_API int bpf_object__load_skeleton(struct bpf_object_skeleton *s); -LIBBPF_API int bpf_object__attach_skeleton(struct bpf_object_skeleton *s); -LIBBPF_API void bpf_object__detach_skeleton(struct bpf_object_skeleton *s); -LIBBPF_API void bpf_object__destroy_skeleton(struct bpf_object_skeleton *s); - -struct gen_loader_opts { - size_t sz; /* size of this struct, for forward/backward compatiblity */ - const char *data; - const char *insns; - __u32 data_sz; - __u32 insns_sz; -}; - -#define gen_loader_opts__last_field insns_sz -LIBBPF_API int bpf_object__gen_loader(struct bpf_object *obj, - struct gen_loader_opts *opts); - -enum libbpf_tristate { - TRI_NO = 0, - TRI_YES = 1, - TRI_MODULE = 2, -}; - -struct bpf_linker_opts { - /* size of this struct, for forward/backward compatiblity */ - size_t sz; -}; -#define bpf_linker_opts__last_field sz - -struct bpf_linker_file_opts { - /* size of this struct, for forward/backward compatiblity */ - size_t sz; -}; -#define bpf_linker_file_opts__last_field sz - -struct bpf_linker; - -LIBBPF_API struct bpf_linker *bpf_linker__new(const char *filename, struct bpf_linker_opts *opts); -LIBBPF_API int bpf_linker__add_file(struct bpf_linker *linker, - const char *filename, - const struct bpf_linker_file_opts *opts); -LIBBPF_API int bpf_linker__finalize(struct bpf_linker *linker); -LIBBPF_API void bpf_linker__free(struct bpf_linker *linker); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* __LIBBPF_LIBBPF_H */ diff --git a/internal/include/libbpf/libbpf_common.h b/internal/include/libbpf/libbpf_common.h deleted file mode 100644 index 947d8bd8a..000000000 --- a/internal/include/libbpf/libbpf_common.h +++ /dev/null @@ -1,42 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -/* - * Common user-facing libbpf helpers. - * - * Copyright (c) 2019 Facebook - */ - -#ifndef __LIBBPF_LIBBPF_COMMON_H -#define __LIBBPF_LIBBPF_COMMON_H - -#include - -#ifndef LIBBPF_API -#define LIBBPF_API __attribute__((visibility("default"))) -#endif - -#define LIBBPF_DEPRECATED(msg) __attribute__((deprecated(msg))) - -/* Helper macro to declare and initialize libbpf options struct - * - * This dance with uninitialized declaration, followed by memset to zero, - * followed by assignment using compound literal syntax is done to preserve - * ability to use a nice struct field initialization syntax and **hopefully** - * have all the padding bytes initialized to zero. It's not guaranteed though, - * when copying literal, that compiler won't copy garbage in literal's padding - * bytes, but that's the best way I've found and it seems to work in practice. - * - * Macro declares opts struct of given type and name, zero-initializes, - * including any extra padding, it with memset() and then assigns initial - * values provided by users in struct initializer-syntax as varargs. - */ -#define DECLARE_LIBBPF_OPTS(TYPE, NAME, ...) \ - struct TYPE NAME = ({ \ - memset(&NAME, 0, sizeof(struct TYPE)); \ - (struct TYPE) { \ - .sz = sizeof(struct TYPE), \ - __VA_ARGS__ \ - }; \ - }) - -#endif /* __LIBBPF_LIBBPF_COMMON_H */ diff --git a/internal/include/libbpf/libbpf_legacy.h b/internal/include/libbpf/libbpf_legacy.h deleted file mode 100644 index df0d03dcf..000000000 --- a/internal/include/libbpf/libbpf_legacy.h +++ /dev/null @@ -1,59 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -/* - * Libbpf legacy APIs (either discouraged or deprecated, as mentioned in [0]) - * - * [0] https://docs.google.com/document/d/1UyjTZuPFWiPFyKk1tV5an11_iaRuec6U-ZESZ54nNTY - * - * Copyright (C) 2021 Facebook - */ -#ifndef __LIBBPF_LEGACY_BPF_H -#define __LIBBPF_LEGACY_BPF_H - -#include -#include -#include -#include -#include "libbpf_common.h" - -#ifdef __cplusplus -extern "C" { -#endif - -enum libbpf_strict_mode { - /* Turn on all supported strict features of libbpf to simulate libbpf - * v1.0 behavior. - * This will be the default behavior in libbpf v1.0. - */ - LIBBPF_STRICT_ALL = 0xffffffff, - - /* - * Disable any libbpf 1.0 behaviors. This is the default before libbpf - * v1.0. It won't be supported anymore in v1.0, please update your - * code so that it handles LIBBPF_STRICT_ALL mode before libbpf v1.0. - */ - LIBBPF_STRICT_NONE = 0x00, - /* - * Return NULL pointers on error, not ERR_PTR(err). - * Additionally, libbpf also always sets errno to corresponding Exx - * (positive) error code. - */ - LIBBPF_STRICT_CLEAN_PTRS = 0x01, - /* - * Return actual error codes from low-level APIs directly, not just -1. - * Additionally, libbpf also always sets errno to corresponding Exx - * (positive) error code. - */ - LIBBPF_STRICT_DIRECT_ERRS = 0x02, - - __LIBBPF_STRICT_LAST, -}; - -LIBBPF_API int libbpf_set_strict_mode(enum libbpf_strict_mode mode); - - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* __LIBBPF_LEGACY_BPF_H */ diff --git a/internal/include/libbpf/libbpf_version.h b/internal/include/libbpf/libbpf_version.h deleted file mode 100644 index 0fefefc35..000000000 --- a/internal/include/libbpf/libbpf_version.h +++ /dev/null @@ -1,9 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -/* Copyright (C) 2021 Facebook */ -#ifndef __LIBBPF_VERSION_H -#define __LIBBPF_VERSION_H - -#define LIBBPF_MAJOR_VERSION 0 -#define LIBBPF_MINOR_VERSION 7 - -#endif /* __LIBBPF_VERSION_H */ diff --git a/internal/include/libbpf/skel_internal.h b/internal/include/libbpf/skel_internal.h deleted file mode 100644 index b22b50c1b..000000000 --- a/internal/include/libbpf/skel_internal.h +++ /dev/null @@ -1,123 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ -/* Copyright (c) 2021 Facebook */ -#ifndef __SKEL_INTERNAL_H -#define __SKEL_INTERNAL_H - -#include -#include -#include - -/* This file is a base header for auto-generated *.lskel.h files. - * Its contents will change and may become part of auto-generation in the future. - * - * The layout of bpf_[map|prog]_desc and bpf_loader_ctx is feature dependent - * and will change from one version of libbpf to another and features - * requested during loader program generation. - */ -struct bpf_map_desc { - union { - /* input for the loader prog */ - struct { - __aligned_u64 initial_value; - __u32 max_entries; - }; - /* output of the loader prog */ - struct { - int map_fd; - }; - }; -}; -struct bpf_prog_desc { - int prog_fd; -}; - -struct bpf_loader_ctx { - size_t sz; - __u32 log_level; - __u32 log_size; - __u64 log_buf; -}; - -struct bpf_load_and_run_opts { - struct bpf_loader_ctx *ctx; - const void *data; - const void *insns; - __u32 data_sz; - __u32 insns_sz; - const char *errstr; -}; - -static inline int skel_sys_bpf(enum bpf_cmd cmd, union bpf_attr *attr, - unsigned int size) -{ - return syscall(__NR_bpf, cmd, attr, size); -} - -static inline int skel_closenz(int fd) -{ - if (fd > 0) - return close(fd); - return -EINVAL; -} - -static inline int bpf_load_and_run(struct bpf_load_and_run_opts *opts) -{ - int map_fd = -1, prog_fd = -1, key = 0, err; - union bpf_attr attr; - - map_fd = bpf_create_map_name(BPF_MAP_TYPE_ARRAY, "__loader.map", 4, - opts->data_sz, 1, 0); - if (map_fd < 0) { - opts->errstr = "failed to create loader map"; - err = -errno; - goto out; - } - - err = bpf_map_update_elem(map_fd, &key, opts->data, 0); - if (err < 0) { - opts->errstr = "failed to update loader map"; - err = -errno; - goto out; - } - - memset(&attr, 0, sizeof(attr)); - attr.prog_type = BPF_PROG_TYPE_SYSCALL; - attr.insns = (long) opts->insns; - attr.insn_cnt = opts->insns_sz / sizeof(struct bpf_insn); - attr.license = (long) "Dual BSD/GPL"; - memcpy(attr.prog_name, "__loader.prog", sizeof("__loader.prog")); - attr.fd_array = (long) &map_fd; - attr.log_level = opts->ctx->log_level; - attr.log_size = opts->ctx->log_size; - attr.log_buf = opts->ctx->log_buf; - attr.prog_flags = BPF_F_SLEEPABLE; - prog_fd = skel_sys_bpf(BPF_PROG_LOAD, &attr, sizeof(attr)); - if (prog_fd < 0) { - opts->errstr = "failed to load loader prog"; - err = -errno; - goto out; - } - - memset(&attr, 0, sizeof(attr)); - attr.test.prog_fd = prog_fd; - attr.test.ctx_in = (long) opts->ctx; - attr.test.ctx_size_in = opts->ctx->sz; - err = skel_sys_bpf(BPF_PROG_RUN, &attr, sizeof(attr)); - if (err < 0 || (int)attr.test.retval < 0) { - opts->errstr = "failed to execute loader prog"; - if (err < 0) - err = -errno; - else - err = (int)attr.test.retval; - goto out; - } - err = 0; -out: - if (map_fd >= 0) - close(map_fd); - if (prog_fd >= 0) - close(prog_fd); - return err; -} - -#endif diff --git a/internal/include/libbpf/update.sh b/internal/include/libbpf/update.sh new file mode 100755 index 000000000..268daa1e4 --- /dev/null +++ b/internal/include/libbpf/update.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Copyright The OpenTelemetry Authors +# SPDX-License-Identifier: Apache-2.0 + +# Copied from https://github.com/cilium/ebpf/blob/v0.15.0/examples/headers/update.sh + +# MIT License +# +# Copyright (c) 2017 Nathan Sweet +# Copyright (c) 2018, 2019 Cloudflare +# Copyright (c) 2019 Authors of Cilium +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +# Version of libbpf to fetch headers from +LIBBPF_VERSION="1.4.5" + +# The headers we want +prefix="libbpf-$LIBBPF_VERSION" +headers=( + "${prefix}/LICENSE" + "${prefix}/LICENSE.BSD-2-Clause" + "${prefix}/LICENSE.LGPL-2.1" + "${prefix}/src/bpf_helpers.h" + "${prefix}/src/bpf_helper_defs.h" + "${prefix}/src/bpf_tracing.h" +) + +# Fetch libbpf release and extract the desired headers +curl -sL "https://github.com/libbpf/libbpf/archive/refs/tags/v${LIBBPF_VERSION}.tar.gz" | \ + tar -xz --xform='s#.*/##' "${headers[@]}" diff --git a/internal/include/libbpf/xsk.h b/internal/include/libbpf/xsk.h deleted file mode 100644 index 01c12dca9..000000000 --- a/internal/include/libbpf/xsk.h +++ /dev/null @@ -1,322 +0,0 @@ -/* SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) */ - -/* - * AF_XDP user-space access library. - * - * Copyright (c) 2018 - 2019 Intel Corporation. - * Copyright (c) 2019 Facebook - * - * Author(s): Magnus Karlsson - */ - -#ifndef __LIBBPF_XSK_H -#define __LIBBPF_XSK_H - -#include -#include -#include -#include - -#include "libbpf.h" - -#ifdef __cplusplus -extern "C" { -#endif - -/* Load-Acquire Store-Release barriers used by the XDP socket - * library. The following macros should *NOT* be considered part of - * the xsk.h API, and is subject to change anytime. - * - * LIBRARY INTERNAL - */ - -#define __XSK_READ_ONCE(x) (*(volatile typeof(x) *)&x) -#define __XSK_WRITE_ONCE(x, v) (*(volatile typeof(x) *)&x) = (v) - -#if defined(__i386__) || defined(__x86_64__) -# define libbpf_smp_store_release(p, v) \ - do { \ - asm volatile("" : : : "memory"); \ - __XSK_WRITE_ONCE(*p, v); \ - } while (0) -# define libbpf_smp_load_acquire(p) \ - ({ \ - typeof(*p) ___p1 = __XSK_READ_ONCE(*p); \ - asm volatile("" : : : "memory"); \ - ___p1; \ - }) -#elif defined(__aarch64__) -# define libbpf_smp_store_release(p, v) \ - asm volatile ("stlr %w1, %0" : "=Q" (*p) : "r" (v) : "memory") -# define libbpf_smp_load_acquire(p) \ - ({ \ - typeof(*p) ___p1; \ - asm volatile ("ldar %w0, %1" \ - : "=r" (___p1) : "Q" (*p) : "memory"); \ - ___p1; \ - }) -#elif defined(__riscv) -# define libbpf_smp_store_release(p, v) \ - do { \ - asm volatile ("fence rw,w" : : : "memory"); \ - __XSK_WRITE_ONCE(*p, v); \ - } while (0) -# define libbpf_smp_load_acquire(p) \ - ({ \ - typeof(*p) ___p1 = __XSK_READ_ONCE(*p); \ - asm volatile ("fence r,rw" : : : "memory"); \ - ___p1; \ - }) -#endif - -#ifndef libbpf_smp_store_release -#define libbpf_smp_store_release(p, v) \ - do { \ - __sync_synchronize(); \ - __XSK_WRITE_ONCE(*p, v); \ - } while (0) -#endif - -#ifndef libbpf_smp_load_acquire -#define libbpf_smp_load_acquire(p) \ - ({ \ - typeof(*p) ___p1 = __XSK_READ_ONCE(*p); \ - __sync_synchronize(); \ - ___p1; \ - }) -#endif - -/* LIBRARY INTERNAL -- END */ - -/* Do not access these members directly. Use the functions below. */ -#define DEFINE_XSK_RING(name) \ -struct name { \ - __u32 cached_prod; \ - __u32 cached_cons; \ - __u32 mask; \ - __u32 size; \ - __u32 *producer; \ - __u32 *consumer; \ - void *ring; \ - __u32 *flags; \ -} - -DEFINE_XSK_RING(xsk_ring_prod); -DEFINE_XSK_RING(xsk_ring_cons); - -/* For a detailed explanation on the memory barriers associated with the - * ring, please take a look at net/xdp/xsk_queue.h. - */ - -struct xsk_umem; -struct xsk_socket; - -static inline __u64 *xsk_ring_prod__fill_addr(struct xsk_ring_prod *fill, - __u32 idx) -{ - __u64 *addrs = (__u64 *)fill->ring; - - return &addrs[idx & fill->mask]; -} - -static inline const __u64 * -xsk_ring_cons__comp_addr(const struct xsk_ring_cons *comp, __u32 idx) -{ - const __u64 *addrs = (const __u64 *)comp->ring; - - return &addrs[idx & comp->mask]; -} - -static inline struct xdp_desc *xsk_ring_prod__tx_desc(struct xsk_ring_prod *tx, - __u32 idx) -{ - struct xdp_desc *descs = (struct xdp_desc *)tx->ring; - - return &descs[idx & tx->mask]; -} - -static inline const struct xdp_desc * -xsk_ring_cons__rx_desc(const struct xsk_ring_cons *rx, __u32 idx) -{ - const struct xdp_desc *descs = (const struct xdp_desc *)rx->ring; - - return &descs[idx & rx->mask]; -} - -static inline int xsk_ring_prod__needs_wakeup(const struct xsk_ring_prod *r) -{ - return *r->flags & XDP_RING_NEED_WAKEUP; -} - -static inline __u32 xsk_prod_nb_free(struct xsk_ring_prod *r, __u32 nb) -{ - __u32 free_entries = r->cached_cons - r->cached_prod; - - if (free_entries >= nb) - return free_entries; - - /* Refresh the local tail pointer. - * cached_cons is r->size bigger than the real consumer pointer so - * that this addition can be avoided in the more frequently - * executed code that computs free_entries in the beginning of - * this function. Without this optimization it whould have been - * free_entries = r->cached_prod - r->cached_cons + r->size. - */ - r->cached_cons = libbpf_smp_load_acquire(r->consumer); - r->cached_cons += r->size; - - return r->cached_cons - r->cached_prod; -} - -static inline __u32 xsk_cons_nb_avail(struct xsk_ring_cons *r, __u32 nb) -{ - __u32 entries = r->cached_prod - r->cached_cons; - - if (entries == 0) { - r->cached_prod = libbpf_smp_load_acquire(r->producer); - entries = r->cached_prod - r->cached_cons; - } - - return (entries > nb) ? nb : entries; -} - -static inline __u32 xsk_ring_prod__reserve(struct xsk_ring_prod *prod, __u32 nb, __u32 *idx) -{ - if (xsk_prod_nb_free(prod, nb) < nb) - return 0; - - *idx = prod->cached_prod; - prod->cached_prod += nb; - - return nb; -} - -static inline void xsk_ring_prod__submit(struct xsk_ring_prod *prod, __u32 nb) -{ - /* Make sure everything has been written to the ring before indicating - * this to the kernel by writing the producer pointer. - */ - libbpf_smp_store_release(prod->producer, *prod->producer + nb); -} - -static inline __u32 xsk_ring_cons__peek(struct xsk_ring_cons *cons, __u32 nb, __u32 *idx) -{ - __u32 entries = xsk_cons_nb_avail(cons, nb); - - if (entries > 0) { - *idx = cons->cached_cons; - cons->cached_cons += entries; - } - - return entries; -} - -static inline void xsk_ring_cons__cancel(struct xsk_ring_cons *cons, __u32 nb) -{ - cons->cached_cons -= nb; -} - -static inline void xsk_ring_cons__release(struct xsk_ring_cons *cons, __u32 nb) -{ - /* Make sure data has been read before indicating we are done - * with the entries by updating the consumer pointer. - */ - libbpf_smp_store_release(cons->consumer, *cons->consumer + nb); - -} - -static inline void *xsk_umem__get_data(void *umem_area, __u64 addr) -{ - return &((char *)umem_area)[addr]; -} - -static inline __u64 xsk_umem__extract_addr(__u64 addr) -{ - return addr & XSK_UNALIGNED_BUF_ADDR_MASK; -} - -static inline __u64 xsk_umem__extract_offset(__u64 addr) -{ - return addr >> XSK_UNALIGNED_BUF_OFFSET_SHIFT; -} - -static inline __u64 xsk_umem__add_offset_to_addr(__u64 addr) -{ - return xsk_umem__extract_addr(addr) + xsk_umem__extract_offset(addr); -} - -LIBBPF_API int xsk_umem__fd(const struct xsk_umem *umem); -LIBBPF_API int xsk_socket__fd(const struct xsk_socket *xsk); - -#define XSK_RING_CONS__DEFAULT_NUM_DESCS 2048 -#define XSK_RING_PROD__DEFAULT_NUM_DESCS 2048 -#define XSK_UMEM__DEFAULT_FRAME_SHIFT 12 /* 4096 bytes */ -#define XSK_UMEM__DEFAULT_FRAME_SIZE (1 << XSK_UMEM__DEFAULT_FRAME_SHIFT) -#define XSK_UMEM__DEFAULT_FRAME_HEADROOM 0 -#define XSK_UMEM__DEFAULT_FLAGS 0 - -struct xsk_umem_config { - __u32 fill_size; - __u32 comp_size; - __u32 frame_size; - __u32 frame_headroom; - __u32 flags; -}; - -LIBBPF_API int xsk_setup_xdp_prog(int ifindex, - int *xsks_map_fd); -LIBBPF_API int xsk_socket__update_xskmap(struct xsk_socket *xsk, - int xsks_map_fd); - -/* Flags for the libbpf_flags field. */ -#define XSK_LIBBPF_FLAGS__INHIBIT_PROG_LOAD (1 << 0) - -struct xsk_socket_config { - __u32 rx_size; - __u32 tx_size; - __u32 libbpf_flags; - __u32 xdp_flags; - __u16 bind_flags; -}; - -/* Set config to NULL to get the default configuration. */ -LIBBPF_API int xsk_umem__create(struct xsk_umem **umem, - void *umem_area, __u64 size, - struct xsk_ring_prod *fill, - struct xsk_ring_cons *comp, - const struct xsk_umem_config *config); -LIBBPF_API int xsk_umem__create_v0_0_2(struct xsk_umem **umem, - void *umem_area, __u64 size, - struct xsk_ring_prod *fill, - struct xsk_ring_cons *comp, - const struct xsk_umem_config *config); -LIBBPF_API int xsk_umem__create_v0_0_4(struct xsk_umem **umem, - void *umem_area, __u64 size, - struct xsk_ring_prod *fill, - struct xsk_ring_cons *comp, - const struct xsk_umem_config *config); -LIBBPF_API int xsk_socket__create(struct xsk_socket **xsk, - const char *ifname, __u32 queue_id, - struct xsk_umem *umem, - struct xsk_ring_cons *rx, - struct xsk_ring_prod *tx, - const struct xsk_socket_config *config); -LIBBPF_API int -xsk_socket__create_shared(struct xsk_socket **xsk_ptr, - const char *ifname, - __u32 queue_id, struct xsk_umem *umem, - struct xsk_ring_cons *rx, - struct xsk_ring_prod *tx, - struct xsk_ring_prod *fill, - struct xsk_ring_cons *comp, - const struct xsk_socket_config *config); - -/* Returns 0 for success and -EBUSY if the umem is still in use. */ -LIBBPF_API int xsk_umem__delete(struct xsk_umem *umem); -LIBBPF_API void xsk_socket__delete(struct xsk_socket *xsk); - -#ifdef __cplusplus -} /* extern "C" */ -#endif - -#endif /* __LIBBPF_XSK_H */