Add AF_XDP syscalls, constants and types

Cargo.toml

@@ -36,7 +36,7 @@ once_cell = { version = "1.5.2", optional = true }
 # libc backend can be selected via adding `--cfg=rustix_use_libc` to
 # `RUSTFLAGS` or enabling the `use-libc` cargo feature.
 [target.'cfg(all(not(rustix_use_libc), not(miri), target_os = "linux", target_endian = "little", any(target_arch = "arm", all(target_arch = "aarch64", target_pointer_width = "64"), target_arch = "riscv64", all(rustix_use_experimental_asm, target_arch = "powerpc64"), all(rustix_use_experimental_asm, target_arch = "mips"), all(rustix_use_experimental_asm, target_arch = "mips32r6"), all(rustix_use_experimental_asm, target_arch = "mips64"), all(rustix_use_experimental_asm, target_arch = "mips64r6"), target_arch = "x86", all(target_arch = "x86_64", target_pointer_width = "64"))))'.dependencies]
-linux-raw-sys = { version = "0.4.11", default-features = false, features = ["general", "errno", "ioctl", "no_std", "elf"] }
+linux-raw-sys = { version = "0.4.12", default-features = false, features = ["general", "errno", "ioctl", "no_std", "elf"] }
 libc_errno = { package = "errno", version = "0.3.8", default-features = false, optional = true }
 libc = { version = "0.2.152", default-features = false, features = ["extra_traits"], optional = true }
 
@@ -53,7 +53,7 @@ libc = { version = "0.2.152", default-features = false, features = ["extra_trait
 # Some syscalls do not have libc wrappers, such as in `io_uring`. For these,
 # the libc backend uses the linux-raw-sys ABI and `libc::syscall`.
 [target.'cfg(all(any(target_os = "android", target_os = "linux"), any(rustix_use_libc, miri, not(all(target_os = "linux", target_endian = "little", any(target_arch = "arm", all(target_arch = "aarch64", target_pointer_width = "64"), target_arch = "riscv64", all(rustix_use_experimental_asm, target_arch = "powerpc64"), all(rustix_use_experimental_asm, target_arch = "mips"), all(rustix_use_experimental_asm, target_arch = "mips32r6"), all(rustix_use_experimental_asm, target_arch = "mips64"), all(rustix_use_experimental_asm, target_arch = "mips64r6"), target_arch = "x86", all(target_arch = "x86_64", target_pointer_width = "64")))))))'.dependencies]
-linux-raw-sys = { version = "0.4.11", default-features = false, features = ["general", "ioctl", "no_std"] }
+linux-raw-sys = { version = "0.4.12", default-features = false, features = ["general", "ioctl", "no_std"] }
 
 # For the libc backend on Windows, use the Winsock API in windows-sys.
 [target.'cfg(windows)'.dependencies.windows-sys]
@@ -141,7 +141,7 @@ io_uring = ["event", "fs", "net", "linux-raw-sys/io_uring"]
 mount = []
 
 # Enable `rustix::net::*`.
-net = ["linux-raw-sys/net", "linux-raw-sys/netlink", "linux-raw-sys/if_ether"]
+net = ["linux-raw-sys/net", "linux-raw-sys/netlink", "linux-raw-sys/if_ether", "linux-raw-sys/xdp"]
 
 # Enable `rustix::thread::*`.
 thread = ["linux-raw-sys/prctl"]

src/backend/libc/net/msghdr.rs

@@ -5,9 +5,13 @@
 
 use crate::backend::c;
 use crate::backend::conv::{msg_control_len, msg_iov_len};
+#[cfg(target_os = "linux")]
+use crate::backend::net::write_sockaddr::encode_sockaddr_xdp;
 use crate::backend::net::write_sockaddr::{encode_sockaddr_v4, encode_sockaddr_v6};
 
 use crate::io::{self, IoSlice, IoSliceMut};
+#[cfg(target_os = "linux")]
+use crate::net::xdp::SocketAddrXdp;
 use crate::net::{RecvAncillaryBuffer, SendAncillaryBuffer, SocketAddrV4, SocketAddrV6};
 use crate::utils::as_ptr;
 
@@ -124,6 +128,28 @@ pub(crate) fn with_unix_msghdr<R>(
     })
 }
 
+/// Create a message header intended to send with an IPv6 address.
+#[cfg(target_os = "linux")]
+pub(crate) fn with_xdp_msghdr<R>(
+    addr: &SocketAddrXdp,
+    iov: &[IoSlice<'_>],
+    control: &mut SendAncillaryBuffer<'_, '_, '_>,
+    f: impl FnOnce(c::msghdr) -> R,
+) -> R {
+    let encoded = encode_sockaddr_xdp(addr);
+
+    f({
+        let mut h = zero_msghdr();
+        h.msg_name = as_ptr(&encoded) as _;
+        h.msg_namelen = size_of::<SocketAddrXdp>() as _;
+        h.msg_iov = iov.as_ptr() as _;
+        h.msg_iovlen = msg_iov_len(iov.len());
+        h.msg_control = control.as_control_ptr().cast();
+        h.msg_controllen = msg_control_len(control.control_len());
+        h
+    })
+}
+
 /// Create a zero-initialized message header struct value.
 #[cfg(all(unix, not(target_os = "redox")))]
 pub(crate) fn zero_msghdr() -> c::msghdr {

src/backend/libc/net/read_sockaddr.rs

@@ -8,6 +8,8 @@ use crate::backend::c;
 #[cfg(not(windows))]
 use crate::ffi::CStr;
 use crate::io;
+#[cfg(target_os = "linux")]
+use crate::net::xdp::{SockaddrXdpFlags, SocketAddrXdp};
 use crate::net::{Ipv4Addr, Ipv6Addr, SocketAddrAny, SocketAddrV4, SocketAddrV6};
 use core::mem::size_of;
 
@@ -193,6 +195,19 @@ pub(crate) unsafe fn read_sockaddr(
                     .map(SocketAddrAny::Unix)
             }
         }
+        #[cfg(target_os = "linux")]
+        c::AF_XDP => {
+            if len < size_of::<c::sockaddr_xdp>() {
+                return Err(io::Errno::INVAL);
+            }
+            let decode = &*storage.cast::<c::sockaddr_xdp>();
+            Ok(SocketAddrAny::Xdp(SocketAddrXdp::new(
+                SockaddrXdpFlags::from_bits_retain(decode.sxdp_flags),
+                u32::from_be(decode.sxdp_ifindex),
+                u32::from_be(decode.sxdp_queue_id),
+                u32::from_be(decode.sxdp_shared_umem_fd),
+            )))
+        }
         _ => Err(io::Errno::INVAL),
     }
 }
@@ -301,6 +316,17 @@ unsafe fn inner_read_sockaddr_os(
                 )
             }
         }
+        #[cfg(target_os = "linux")]
+        c::AF_XDP => {
+            assert!(len >= size_of::<c::sockaddr_xdp>());
+            let decode = &*storage.cast::<c::sockaddr_xdp>();
+            SocketAddrAny::Xdp(SocketAddrXdp::new(
+                SockaddrXdpFlags::from_bits_retain(decode.sxdp_flags),
+                u32::from_be(decode.sxdp_ifindex),
+                u32::from_be(decode.sxdp_queue_id),
+                u32::from_be(decode.sxdp_shared_umem_fd),
+            ))
+        }
         other => unimplemented!("{:?}", other),
     }
 }

src/backend/libc/net/sockopt.rs

@@ -14,6 +14,8 @@ use crate::fd::BorrowedFd;
 use crate::ffi::CStr;
 use crate::io;
 use crate::net::sockopt::Timeout;
+#[cfg(target_os = "linux")]
+use crate::net::xdp::{XdpMmapOffsets, XdpOptionsFlags, XdpRingOffset, XdpStatistics, XdpUmemReg};
 #[cfg(not(any(
     apple,
     windows,
@@ -73,6 +75,8 @@ use c::TCP_KEEPALIVE as TCP_KEEPIDLE;
 use c::TCP_KEEPIDLE;
 use core::mem::{size_of, MaybeUninit};
 use core::time::Duration;
+#[cfg(target_os = "linux")]
+use linux_raw_sys::xdp::{xdp_mmap_offsets, xdp_statistics, xdp_statistics_v1};
 #[cfg(windows)]
 use windows_sys::Win32::Foundation::BOOL;
 
@@ -963,6 +967,170 @@ pub(crate) fn get_socket_peercred(fd: BorrowedFd<'_>) -> io::Result<UCred> {
     getsockopt(fd, c::SOL_SOCKET, c::SO_PEERCRED)
 }
 
+#[cfg(target_os = "linux")]
+#[inline]
+pub(crate) fn set_xdp_umem_reg(fd: BorrowedFd<'_>, value: XdpUmemReg) -> io::Result<()> {
+    setsockopt(fd, c::SOL_XDP, c::XDP_UMEM_REG, value)
+}
+
+#[cfg(target_os = "linux")]
+#[inline]
+pub(crate) fn set_xdp_umem_fill_ring_size(fd: BorrowedFd<'_>, value: u32) -> io::Result<()> {
+    setsockopt(fd, c::SOL_XDP, c::XDP_UMEM_FILL_RING, value)
+}
+
+#[cfg(target_os = "linux")]
+#[inline]
+pub(crate) fn set_xdp_umem_completion_ring_size(fd: BorrowedFd<'_>, value: u32) -> io::Result<()> {
+    setsockopt(fd, c::SOL_XDP, c::XDP_UMEM_COMPLETION_RING, value)
+}
+
+#[cfg(target_os = "linux")]
+#[inline]
+pub(crate) fn set_xdp_tx_ring_size(fd: BorrowedFd<'_>, value: u32) -> io::Result<()> {
+    setsockopt(fd, c::SOL_XDP, c::XDP_TX_RING, value)
+}
+
+#[cfg(target_os = "linux")]
+#[inline]
+pub(crate) fn set_xdp_rx_ring_size(fd: BorrowedFd<'_>, value: u32) -> io::Result<()> {
+    setsockopt(fd, c::SOL_XDP, c::XDP_RX_RING, value)
+}
+
+#[cfg(target_os = "linux")]
+#[inline]
+pub(crate) fn get_xdp_mmap_offsets(fd: BorrowedFd<'_>) -> io::Result<XdpMmapOffsets> {
+    // The kernel will write `xdp_mmap_offsets` or `xdp_mmap_offsets_v1` to the supplied pointer,
+    // depending on the kernel version. Both structs only contain u64 values.
+    // By using the larger of both as the parameter, we can shuffle the values to the non-v1 version
+    // returned by `get_xdp_mmap_offsets` while keeping the return type unaffected by the kernel
+    // version. This works because C will layout all struct members one after the other.
+
+    let mut optlen = core::mem::size_of::<xdp_mmap_offsets>().try_into().unwrap();
+    debug_assert!(
+        optlen as usize >= core::mem::size_of::<c::c_int>(),
+        "Socket APIs don't ever use `bool` directly"
+    );
+    let mut value = MaybeUninit::<xdp_mmap_offsets>::zeroed();
+    getsockopt_raw(fd, c::SOL_XDP, c::XDP_MMAP_OFFSETS, &mut value, &mut optlen)?;
+
+    if optlen as usize == core::mem::size_of::<c::xdp_mmap_offsets_v1>() {
+        // Safety: All members of xdp_mmap_offsets are u64 and thus are correctly initialized
+        // by `MaybeUninit::<xdp_statistics>::zeroed()`
+        let xpd_mmap_offsets = unsafe { value.assume_init() };
+        Ok(XdpMmapOffsets {
+            rx: XdpRingOffset {
+                producer: xpd_mmap_offsets.rx.producer,
+                consumer: xpd_mmap_offsets.rx.consumer,
+                desc: xpd_mmap_offsets.rx.desc,
+                flags: None,
+            },
+            tx: XdpRingOffset {
+                producer: xpd_mmap_offsets.rx.flags,
+                consumer: xpd_mmap_offsets.tx.producer,
+                desc: xpd_mmap_offsets.tx.consumer,
+                flags: None,
+            },
+            fr: XdpRingOffset {
+                producer: xpd_mmap_offsets.tx.desc,
+                consumer: xpd_mmap_offsets.tx.flags,
+                desc: xpd_mmap_offsets.fr.producer,
+                flags: None,
+            },
+            cr: XdpRingOffset {
+                producer: xpd_mmap_offsets.fr.consumer,
+                consumer: xpd_mmap_offsets.fr.desc,
+                desc: xpd_mmap_offsets.fr.flags,
+                flags: None,
+            },
+        })
+    } else {
+        assert_eq!(
+            optlen as usize,
+            core::mem::size_of::<xdp_mmap_offsets>(),
+            "unexpected getsockopt size"
+        );
+        // Safety: All members of xdp_mmap_offsets are u64 and thus are correctly initialized
+        // by `MaybeUninit::<xdp_statistics>::zeroed()`
+        let xpd_mmap_offsets = unsafe { value.assume_init() };
+        Ok(XdpMmapOffsets {
+            rx: XdpRingOffset {
+                producer: xpd_mmap_offsets.rx.producer,
+                consumer: xpd_mmap_offsets.rx.consumer,
+                desc: xpd_mmap_offsets.rx.desc,
+                flags: Some(xpd_mmap_offsets.rx.flags),
+            },
+            tx: XdpRingOffset {
+                producer: xpd_mmap_offsets.tx.producer,
+                consumer: xpd_mmap_offsets.tx.consumer,
+                desc: xpd_mmap_offsets.tx.desc,
+                flags: Some(xpd_mmap_offsets.tx.flags),
+            },
+            fr: XdpRingOffset {
+                producer: xpd_mmap_offsets.fr.producer,
+                consumer: xpd_mmap_offsets.fr.consumer,
+                desc: xpd_mmap_offsets.fr.desc,
+                flags: Some(xpd_mmap_offsets.fr.flags),
+            },
+            cr: XdpRingOffset {
+                producer: xpd_mmap_offsets.cr.producer,
+                consumer: xpd_mmap_offsets.cr.consumer,
+                desc: xpd_mmap_offsets.cr.desc,
+                flags: Some(xpd_mmap_offsets.cr.flags),
+            },
+        })
+    }
+}
+
+#[cfg(target_os = "linux")]
+#[inline]
+pub(crate) fn get_xdp_statistics(fd: BorrowedFd<'_>) -> io::Result<XdpStatistics> {
+    let mut optlen = core::mem::size_of::<xdp_statistics>().try_into().unwrap();
+    debug_assert!(
+        optlen as usize >= core::mem::size_of::<c::c_int>(),
+        "Socket APIs don't ever use `bool` directly"
+    );
+    let mut value = MaybeUninit::<xdp_statistics>::zeroed();
+    getsockopt_raw(fd, c::SOL_XDP, c::XDP_STATISTICS, &mut value, &mut optlen)?;
+
+    if optlen as usize == core::mem::size_of::<xdp_statistics_v1>() {
+        // Safety: All members of xdp_statistics are u64 and thus are correctly initialized
+        // by `MaybeUninit::<xdp_statistics>::zeroed()`
+        let xdp_statistics = unsafe { value.assume_init() };
+        Ok(XdpStatistics {
+            rx_dropped: xdp_statistics.rx_dropped,
+            rx_invalid_descs: xdp_statistics.rx_dropped,
+            tx_invalid_descs: xdp_statistics.rx_dropped,
+            rx_ring_full: None,
+            rx_fill_ring_empty_descs: None,
+            tx_ring_empty_descs: None,
+        })
+    } else {
+        assert_eq!(
+            optlen as usize,
+            core::mem::size_of::<xdp_statistics>(),
+            "unexpected getsockopt size"
+        );
+        // Safety: All members of xdp_statistics are u64 and thus are correctly initialized
+        // by `MaybeUninit::<xdp_statistics>::zeroed()`
+        let xdp_statistics = unsafe { value.assume_init() };
+        Ok(XdpStatistics {
+            rx_dropped: xdp_statistics.rx_dropped,
+            rx_invalid_descs: xdp_statistics.rx_invalid_descs,
+            tx_invalid_descs: xdp_statistics.tx_invalid_descs,
+            rx_ring_full: Some(xdp_statistics.rx_ring_full),
+            rx_fill_ring_empty_descs: Some(xdp_statistics.rx_fill_ring_empty_descs),
+            tx_ring_empty_descs: Some(xdp_statistics.tx_ring_empty_descs),
+        })
+    }
+}
+
+#[cfg(target_os = "linux")]
+#[inline]
+pub(crate) fn get_xdp_options(fd: BorrowedFd<'_>) -> io::Result<XdpOptionsFlags> {
+    getsockopt(fd, c::SOL_XDP, c::XDP_OPTIONS)
+}
+
 #[inline]
 fn to_ip_mreq(multiaddr: &Ipv4Addr, interface: &Ipv4Addr) -> c::ip_mreq {
     c::ip_mreq {

src/backend/libc/net/syscalls.rs

@@ -2,10 +2,16 @@
 
 #[cfg(unix)]
 use super::addr::SocketAddrUnix;
+#[cfg(target_os = "linux")]
+use super::msghdr::with_xdp_msghdr;
+#[cfg(target_os = "linux")]
+use super::write_sockaddr::encode_sockaddr_xdp;
 use crate::backend::c;
 use crate::backend::conv::{borrowed_fd, ret, ret_owned_fd, ret_send_recv, send_recv_len};
 use crate::fd::{BorrowedFd, OwnedFd};
 use crate::io;
+#[cfg(target_os = "linux")]
+use crate::net::xdp::SocketAddrXdp;
 use crate::net::{SocketAddrAny, SocketAddrV4, SocketAddrV6};
 use crate::utils::as_ptr;
 use core::mem::{size_of, MaybeUninit};
@@ -145,6 +151,25 @@ pub(crate) fn sendto_unix(
     }
 }
 
+#[cfg(target_os = "linux")]
+pub(crate) fn sendto_xdp(
+    fd: BorrowedFd<'_>,
+    buf: &[u8],
+    flags: SendFlags,
+    addr: &SocketAddrXdp,
+) -> io::Result<usize> {
+    unsafe {
+        ret_send_recv(c::sendto(
+            borrowed_fd(fd),
+            buf.as_ptr().cast(),
+            send_recv_len(buf.len()),
+            bitflags_bits!(flags),
+            as_ptr(&encode_sockaddr_xdp(addr)).cast::<c::sockaddr>(),
+            size_of::<c::sockaddr_xdp>() as _,
+        ))
+    }
+}
+
 #[cfg(not(any(target_os = "redox", target_os = "wasi")))]
 pub(crate) fn socket(
     domain: AddressFamily,
@@ -217,6 +242,17 @@ pub(crate) fn bind_unix(sockfd: BorrowedFd<'_>, addr: &SocketAddrUnix) -> io::Re
     }
 }
 
+#[cfg(target_os = "linux")]
+pub(crate) fn bind_xdp(sockfd: BorrowedFd<'_>, addr: &SocketAddrXdp) -> io::Result<()> {
+    unsafe {
+        ret(c::bind(
+            borrowed_fd(sockfd),
+            as_ptr(&encode_sockaddr_xdp(addr)).cast(),
+            size_of::<c::sockaddr_xdp>() as c::socklen_t,
+        ))
+    }
+}
+
 #[cfg(not(any(target_os = "redox", target_os = "wasi")))]
 pub(crate) fn connect_v4(sockfd: BorrowedFd<'_>, addr: &SocketAddrV4) -> io::Result<()> {
     unsafe {
@@ -402,6 +438,23 @@ pub(crate) fn sendmsg_unix(
     })
 }
 
+#[cfg(target_os = "linux")]
+pub(crate) fn sendmsg_xdp(
+    sockfd: BorrowedFd<'_>,
+    addr: &SocketAddrXdp,
+    iov: &[IoSlice<'_>],
+    control: &mut SendAncillaryBuffer<'_, '_, '_>,
+    msg_flags: SendFlags,
+) -> io::Result<usize> {
+    with_xdp_msghdr(addr, iov, control, |msghdr| unsafe {
+        ret_send_recv(c::sendmsg(
+            borrowed_fd(sockfd),
+            &msghdr,
+            bitflags_bits!(msg_flags),
+        ))
+    })
+}
+
 #[cfg(not(any(
     apple,
     windows,