diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7cf914b944..46bb351004 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -761,7 +761,7 @@ jobs: env: RUSTFLAGS: --cfg loom -Cdebug-assertions - xdp-bpf: + xdp: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -771,9 +771,10 @@ jobs: - uses: actions-rs/toolchain@v1.0.7 id: toolchain with: - toolchain: stable + toolchain: 1.66.0 profile: minimal override: true + components: clippy,rustfmt - uses: camshaft/install@v1 with: @@ -781,8 +782,16 @@ jobs: - uses: camshaft/rust-cache@v1 + - name: Run clippy + working-directory: tools/xdp + run: cargo clippy + - name: Build ebpf working-directory: tools/xdp env: RUST_LOG: trace run: cargo xtask ci + + - name: Run tests + working-directory: tools/xdp + run: cargo test diff --git a/tools/xdp/Cargo.toml b/tools/xdp/Cargo.toml index a1df3e416b..8fc70144be 100644 --- a/tools/xdp/Cargo.toml +++ b/tools/xdp/Cargo.toml @@ -1,2 +1,2 @@ [workspace] -members = ["tester", "xtask"] +members = ["s2n-quic-xdp", "tester", "xtask"] diff --git a/tools/xdp/lib/s2n-quic-xdp-bpfeb-trace.ebpf b/tools/xdp/lib/s2n-quic-xdp-bpfeb-trace.ebpf index e576aa9b54..0d10d08c8e 100644 --- a/tools/xdp/lib/s2n-quic-xdp-bpfeb-trace.ebpf +++ b/tools/xdp/lib/s2n-quic-xdp-bpfeb-trace.ebpf @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:c4dbdc5905fd33fa0d83273cc210806e46182ef0a1ff3fbc426fbe2844d76954 +oid sha256:3b4501af6d1055d1f53b5228872301d547cc7f28549cb8aea2d714ce9b55220b size 12152 diff --git a/tools/xdp/lib/s2n-quic-xdp-bpfeb.ebpf b/tools/xdp/lib/s2n-quic-xdp-bpfeb.ebpf index 9cbefdde0d..f140676647 100644 --- a/tools/xdp/lib/s2n-quic-xdp-bpfeb.ebpf +++ b/tools/xdp/lib/s2n-quic-xdp-bpfeb.ebpf @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:122b37efbfeb825b04138121cc94d2f39a33cab924f43c9af9260560c5835c79 +oid sha256:6c2e8a8aa34e465c7e54fd2eb6cac66d9ea71ce945a48dc863afbfc0e9a383f0 size 2488 diff --git a/tools/xdp/lib/s2n-quic-xdp-bpfel-trace.ebpf b/tools/xdp/lib/s2n-quic-xdp-bpfel-trace.ebpf index a1e3c1ed09..bbb5c80ba0 100644 --- a/tools/xdp/lib/s2n-quic-xdp-bpfel-trace.ebpf +++ b/tools/xdp/lib/s2n-quic-xdp-bpfel-trace.ebpf @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:90d74a04ce1fd9a7d9c01f70daa3063d80786e97ab5cd6325de876cb91c8231a +oid sha256:0620fe2e3e5f8fb1c501bf84ea020de5417cf818edcb2277a642d01ce3c5b57c size 12168 diff --git a/tools/xdp/lib/s2n-quic-xdp-bpfel.ebpf b/tools/xdp/lib/s2n-quic-xdp-bpfel.ebpf index 3c870833ce..0c710cac25 100644 --- a/tools/xdp/lib/s2n-quic-xdp-bpfel.ebpf +++ b/tools/xdp/lib/s2n-quic-xdp-bpfel.ebpf @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:b6b65d13bc2a83b6d79a6a085bfc120c37e935bb11bade56eb47d0b0d23b03d8 +oid sha256:2d7123ada5a07d4aae274f99c09a4a59e3d3f13f8a5f6dba0f9c1d37ee852dbf size 2520 diff --git a/tools/xdp/s2n-quic-xdp/Cargo.toml b/tools/xdp/s2n-quic-xdp/Cargo.toml new file mode 100644 index 0000000000..6698da1350 --- /dev/null +++ b/tools/xdp/s2n-quic-xdp/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "s2n-quic-xdp" +version = "0.1.0" +edition = "2021" + +[dependencies] +bitflags = "2" +libc = "0.2" +s2n-quic-core = { path = "../../../quic/s2n-quic-core", version = "0.18" } + +[dev-dependencies] +bolero = "0.8" diff --git a/tools/xdp/s2n-quic-xdp/README.md b/tools/xdp/s2n-quic-xdp/README.md new file mode 100644 index 0000000000..fe95d375d8 --- /dev/null +++ b/tools/xdp/s2n-quic-xdp/README.md @@ -0,0 +1,7 @@ +# s2n-quic-xdp + +AF-XDP IO provider for s2n-quic endpoints. + +## Getting started + +TODO finish once the crate is complete diff --git a/tools/xdp/s2n-quic-xdp/src/if_xdp.rs b/tools/xdp/s2n-quic-xdp/src/if_xdp.rs new file mode 100644 index 0000000000..12c2589a78 --- /dev/null +++ b/tools/xdp/s2n-quic-xdp/src/if_xdp.rs @@ -0,0 +1,293 @@ +// Copyright Intel Corporation +// SPDX-License-Identifier: GPL-2.0 WITH Linux-syscall-note +// Modifications copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use crate::Result; +use bitflags::bitflags; +use core::{ffi::CStr, mem::size_of}; + +bitflags!( + /// Options for the `flags` field in [`Address`] + /// + /// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L16-L27) + /// + /// Note that the size of the value is specified in the [sxdp_flags field]( + /// https://github.com/torvalds/linux/blob/0d3eb744aed40ffce820cded61d7eac515199165/include/uapi/linux/if_xdp.h#L34). + #[derive(Copy, Clone, Debug, Default)] + #[repr(transparent)] + pub struct XdpFlags: u16 { + const SHARED_UMEM = 1 << 0; + /// Force copy mode + const COPY = 1 << 1; + /// Force zero-copy mode + const ZEROCOPY = 1 << 2; + /// If this option is set, the driver might go to sleep and in that case the + /// XDP_RING_NEED_WAKEUP flag in the fill and/or Tx rings will be set. + /// + /// If it is set, the application needs to explicitly wake up the driver with a `poll()` for + /// Rx or `sendto()` for Tx. If you are running the driver and the application on the same + /// core, you should use this option so that the kernel will yield to the user space + /// application. + const USE_NEED_WAKEUP = 1 << 3; + } +); + +bitflags!( + /// Flags for the umem config + /// + /// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L30>) + /// + /// Note that in `if_xdp.h`, the size is left unspecified. However, when it's used in libxdp, + /// the size is set to `u32`: [xsk.h](https://github.com/xdp-project/xdp-tools/blob/a76e7a2b156b8cfe38992206abe9df1df0a29e38/headers/xdp/xsk.h#L203). + #[derive(Copy, Clone, Debug, Default)] + #[repr(transparent)] + pub struct UmemFlags: u32 { + const UNALIGNED_CHUNK_FLAG = 1 << 0; + } +); + +/// A structure for representing the address of an AF_XDP socket +/// +/// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L32-L38) +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct Address { + pub family: u16, + pub flags: XdpFlags, + pub ifindex: u32, + pub queue_id: u32, + pub shared_umem_fd: u32, +} + +impl Default for Address { + fn default() -> Self { + Self { + family: libc::PF_XDP as _, + flags: Default::default(), + ifindex: 0, + queue_id: 0, + shared_umem_fd: 0, + } + } +} + +impl Address { + /// Resolves the `ifindex` for the provided `if_name` and associates it with the address. + /// + /// If the device does not exist, then an error is returned. + #[inline] + pub fn set_if_name(&mut self, name: &CStr) -> Result<&mut Self> { + unsafe { + let ifindex = libc::if_nametoindex(name.as_ptr()); + + // https://man7.org/linux/man-pages/man3/if_nametoindex.3.html + // > On success, if_nametoindex() returns the index number of the + // > network interface; on error, 0 is returned and errno is set to + // > indicate the error. + if ifindex == 0 { + return Err(std::io::Error::last_os_error()); + } + + self.ifindex = ifindex; + } + Ok(self) + } +} + +bitflags!( + /// Flags set on a particular ring state `flags` field + /// + /// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L41) + /// + /// Note that in `if_xdp.h`, the size is left unspecified. However, when it's used in libxdp, + /// the size is set to `u32`: [xsk.h]( + /// https://github.com/xdp-project/xdp-tools/blob/a76e7a2b156b8cfe38992206abe9df1df0a29e38/headers/xdp/xsk.h#L42) + #[derive(Default)] + #[repr(transparent)] + pub struct RingFlags: u32 { + const NEED_WAKEUP = 1 << 0; + } +); + +/// A structure to communicate the offsets for an individual ring for an AF_XDP socket +/// +/// Note that the `flags` field was added in kernel 5.4. +/// +/// See [if_xdp.h](https://github.com/torvalds/linux/blob/9116e5e2b1fff71dce501d971e86a3695acc3dba/include/uapi/linux/if_xdp.h#L28-L32) +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct RingOffsetV1 { + pub producer: u64, + pub consumer: u64, + pub desc: u64, +} + +/// A structure to communicate the offsets for an individual ring for an AF_XDP socket +/// +/// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L43) +#[derive(Clone, Copy, Debug, Default)] +#[repr(C)] +pub struct RingOffsetV2 { + pub producer: u64, + pub consumer: u64, + pub desc: u64, + pub flags: u64, +} + +impl RingOffsetV2 { + /// Converts an offset description from an older version + #[inline] + fn set_v1(&mut self, v1: RingOffsetV1) { + // Logic from https://github.com/xdp-project/xdp-tools/blob/a76e7a2b156b8cfe38992206abe9df1df0a29e38/lib/libxdp/xsk.c#L197 + self.producer = v1.producer; + self.consumer = v1.consumer; + self.desc = v1.desc; + self.flags = v1.consumer + (size_of::() as u64); + } +} + +/// A structure to communicate the offsets of each of the 4 rings for an AF_XDP socket +/// +/// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L50) +#[derive(Clone, Copy, Debug, Default)] +#[repr(C)] +pub struct MmapOffsets { + pub rx: O, + pub tx: O, + pub fill: O, + pub completion: O, +} + +impl MmapOffsets { + /// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L92) + pub const RX_RING: usize = 0; + /// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L93) + pub const TX_RING: usize = 0x8000_0000; + /// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L94) + pub const FILL_RING: usize = 0x1_0000_0000; + /// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L95) + pub const COMPLETION_RING: usize = 0x1_8000_0000; + + #[inline] + pub(crate) fn as_v1(mut self) -> Self { + // getsockopt on a kernel <= 5.3 has no flags fields. + // Copy over the offsets to the correct places in the >=5.4 format + // and put the flags where they would have been on that kernel. + + // Logic from https://github.com/xdp-project/xdp-tools/blob/a76e7a2b156b8cfe38992206abe9df1df0a29e38/lib/libxdp/xsk.c#L197 + let v1 = unsafe { *(&self as *const Self as *const MmapOffsets) }; + self.rx.set_v1(v1.rx); + self.tx.set_v1(v1.tx); + self.fill.set_v1(v1.fill); + self.completion.set_v1(v1.completion); + + self + } +} + +/// Socket option ids passed to an AF_XDP socket with the sockopt syscalls. +/// +/// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L58-L65) +/// +/// Note that in `if_xdp.h`, the size is left unspecified. However, it is used in the `name` +/// argument for [`libc::setsockopt`] and [`libc::getsockopt`], which is a [`libc::c_int`] (i32). +#[derive(Clone, Copy, Debug)] +#[repr(i32)] +pub enum SocketOptions { + MmapOffsets = 1, + RxRing = 2, + TxRing = 3, + UmemReg = 4, + UmemFillRing = 5, + UmemCompletionRing = 6, + Statistics = 7, + Options = 8, +} + +/// Umem configuration on an AF_XDP socket +/// +/// This struct is used with the [`libc::setsockopt`] call with the `name` parameter set to +/// [`SocketOptions::UmemReg`]. +/// +/// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L67-L73). +#[derive(Clone, Copy, Debug)] +#[repr(C)] +pub struct UmemReg { + /// Start of packet data area + pub addr: u64, + /// Length of packet data area + pub len: u64, + pub chunk_size: u32, + /// The length reserved before packets + pub headroom: u32, + /// The flag value for the Umem + pub flags: UmemFlags, +} + +/// Statistics returned from an AF_XDP socket +/// +/// This struct is used with the [`libc::getsockopt`] call with the `name` parameter set to +/// [`SocketOptions::Statistics`]. +/// +/// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L75-L82) +#[derive(Clone, Copy, Debug, Default)] +#[repr(C)] +pub struct Statistics { + /// Dropped for other reasons + pub rx_dropped: u64, + /// Dropped due to invalid descriptor + pub rx_invalid_descriptors: u64, + /// Dropped due to invalid descriptor + pub tx_invalid_descriptors: u64, + /// Dropped due to rx ring being full + pub rx_ring_full: u64, + /// Failed to retrieve item from fill ring + pub rx_fill_ring_empty_descriptors: u64, + /// Failed to retrieve item from tx ring + pub tx_ring_empty_descriptors: u64, +} + +bitflags!( + /// Options returned from an AF_XDP socket + /// + /// This struct is used with the [`libc::getsockopt`] call with the `name` parameter set to + /// [`SocketOptions::Options`]. + /// + /// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L89) + #[derive(Clone, Copy, Debug, Default)] + #[repr(transparent)] + pub struct XdpOptions: u32 { + const ZEROCOPY = 1 << 0; + } +); + +/// Masks for unaligned chunks mode +/// +/// See [if_xdp](https://github.com/torvalds/linux/blob/0d3eb744aed40ffce820cded61d7eac515199165/include/uapi/linux/if_xdp.h#L97-L100). +pub const XSK_UNALIGNED_BUF_ADDR_MASK: u64 = (1 << XSK_UNALIGNED_BUF_OFFSET_SHIFT) - 1; +pub const XSK_UNALIGNED_BUF_OFFSET_SHIFT: u64 = 48; + +/// Rx/Tx descriptor +/// +/// See [if_xdp.h](https://github.com/torvalds/linux/blob/2bac7dc169af3cd4a0cb5200aa1f7b89affa042a/include/uapi/linux/if_xdp.h#L103-L107) +#[derive(Clone, Copy, Debug, Default)] +#[repr(C)] +pub struct RxTxDescriptor { + /// Offset into the umem where the packet starts + pub address: u64, + /// Length of the packet + pub len: u32, + /// Options set on the descriptor + pub options: u32, +} + +/// Umem Descriptor +/// +/// See [if_xdp.h](https://github.com/torvalds/linux/blob/0d3eb744aed40ffce820cded61d7eac515199165/include/uapi/linux/if_xdp.h#L109). +#[derive(Clone, Copy, Debug, Default)] +#[repr(C)] +pub struct UmemDescriptor { + /// Offset into the umem where the packet starts + pub address: u64, +} diff --git a/tools/xdp/s2n-quic-xdp/src/lib.rs b/tools/xdp/s2n-quic-xdp/src/lib.rs new file mode 100644 index 0000000000..2ec4ba5b87 --- /dev/null +++ b/tools/xdp/s2n-quic-xdp/src/lib.rs @@ -0,0 +1,9 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +#![allow(dead_code)] // TODO remove once the crate is finished + +type Result = core::result::Result; + +/// Primitive types for AF-XDP kernel APIs +mod if_xdp; diff --git a/tools/xdp/xtask/src/build_ebpf.rs b/tools/xdp/xtask/src/build_ebpf.rs index 3554993df2..6e363c4c8f 100644 --- a/tools/xdp/xtask/src/build_ebpf.rs +++ b/tools/xdp/xtask/src/build_ebpf.rs @@ -14,7 +14,7 @@ pub fn run() -> Result<(), anyhow::Error> { let mut command = Command::new("cargo"); let dir = PathBuf::from("ebpf"); - command.current_dir(&dir).args(&[ + command.current_dir(&dir).args([ "build", "--target", target,