fix(s2n-quic-core): flush packet queue between connections/MTU probes

[camshaft]

Description of changes:

I noticed that some MTU probe packets were getting both EMSGSIZE and EINVAL errors returned, depending on if the packet included a GSO segment or not. This causes extra flakiness for packets that are bundled with the MTU probe, since the whole packet is discarded on error.

This change, instead, forces MTU probes to always be standalone packets. It does this by checking can_gso before writing the message and uses that value to know if it needs to be standalone or not.

Before

$ SCENARIO=./target/netbench/ping.json strace -e 'trace=sendmmsg' --status=failed ./target/debug/netbench-driver-s2n-quic-server --trace=disabled
sendmmsg(8, [{msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(45162), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="L}\200\246\336nI\362\252A\243K\0230\0375\273\246c\234\377\10\27^\210h\340\334\2453\206\226"..., iov_len=5255}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}, {cmsg_len=18, cmsg_level=SOL_UDP, cmsg_type=0x67}], msg_controllen=80, msg_flags=0}}], 1, 0) = -1 EINVAL (Invalid argument)
sendmmsg(8, [{msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(45162), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="I}\200\246\336nI\362\252A\243K\0230\0375\273 `\3\307\241\rT|w\300\235N\377~5"..., iov_len=5255}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}, {cmsg_len=18, cmsg_level=SOL_UDP, cmsg_type=0x67}], msg_controllen=80, msg_flags=0}}], 1, 0) = -1 EINVAL (Invalid argument)
sendmmsg(8, [{msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(45162), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="A}\200\246\336nI\362\252A\243K\0230\0375\273\223\32`\350\226yT\216\357\32\273\34\2+\305"..., iov_len=5255}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}, {cmsg_len=18, cmsg_level=SOL_UDP, cmsg_type=0x67}], msg_controllen=80, msg_flags=0}}], 1, 0) = -1 EINVAL (Invalid argument)
sendmmsg(8, [{msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(45162), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="U}\200\246\336nI\362\252A\243K\0230\0375\273\367\334FX\n\16\247h\314y\273waU\213"..., iov_len=3380}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}, {cmsg_len=18, cmsg_level=SOL_UDP, cmsg_type=0x67}], msg_controllen=80, msg_flags=0}}], 1, 0) = -1 EINVAL (Invalid argument)
sendmmsg(8, [{msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(45162), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="T}\200\246\336nI\362\252A\243K\0230\0375\273~\263'\363\2204\250\323^\200\217\234\223G\361"..., iov_len=3380}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}, {cmsg_len=18, cmsg_level=SOL_UDP, cmsg_type=0x67}], msg_controllen=80, msg_flags=0}}], 1, 0) = -1 EINVAL (Invalid argument)
sendmmsg(8, [{msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(45162), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="J}\200\246\336nI\362\252A\243K\0230\0375\273\177.\376\2227\2458P\201\7\355\207N\307&"..., iov_len=3380}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}, {cmsg_len=18, cmsg_level=SOL_UDP, cmsg_type=0x67}], msg_controllen=80, msg_flags=0}}], 1, 0) = -1 EINVAL (Invalid argument)
sendmmsg(8, [{msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(45162), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="N}\200\246\336nI\362\252A\243K\0230\0375\273\r\245S\241\26\326\323t\310\24\200\261\\\0\216"..., iov_len=2442}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}, {cmsg_len=18, cmsg_level=SOL_UDP, cmsg_type=0x67}], msg_controllen=80, msg_flags=0}}], 1, 0) = -1 EINVAL (Invalid argument)

After

$ SCENARIO=./target/netbench/ping.json strace -e 'trace=sendmmsg' --status=failed ./target/debug/netbench-driver-s2n-quic-server --trace=disabled
sendmmsg(8, [{msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(46846), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="J\303\224<\250)H\346C\32\320\303.K/+\1778\266\215t\35GT\322\337vQ\256A\373\r"..., iov_len=5202}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}], msg_controllen=56, msg_flags=0}}, {msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(46846), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="T\303\224<\250)H\346C\32\320\303.K/+\177\254\256-\362MZ\0305/\3\364\22}l\367"..., iov_len=53}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}], msg_controllen=56, msg_flags=0}}], 2, 0) = -1 EMSGSIZE (Message too long)
sendmmsg(8, [{msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(46846), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="F\303\224<\250)H\346C\32\320\303.K/+\177\365F/\324\357\273\341O\224|\246N\364\"?"..., iov_len=5202}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}], msg_controllen=56, msg_flags=0}}], 1, 0) = -1 EMSGSIZE (Message too long)
sendmmsg(8, [{msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(46846), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="F\303\224<\250)H\346C\32\320\303.K/+\177\35{\21\305k\263f\233\3457\225\354\213y)"..., iov_len=5202}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}], msg_controllen=56, msg_flags=0}}], 1, 0) = -1 EMSGSIZE (Message too long)
sendmmsg(8, [{msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(46846), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="[\303\224<\250)H\346C\32\320\303.K/+\177\n\262\276\37\356%\277\245L6t\7\226\252\360"..., iov_len=3327}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}], msg_controllen=56, msg_flags=0}}], 1, 0) = -1 EMSGSIZE (Message too long)
sendmmsg(8, [{msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(46846), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="T\303\224<\250)H\346C\32\320\303.K/+\177\265\216\273n\246Z\312%Q\366\225\230L\210\232"..., iov_len=3327}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}], msg_controllen=56, msg_flags=0}}], 1, 0) = -1 EMSGSIZE (Message too long)
sendmmsg(8, [{msg_hdr={msg_name={sa_family=AF_INET6, sin6_port=htons(46846), sin6_flowinfo=htonl(0), inet_pton(AF_INET6, "::ffff:10.0.128.116", &sin6_addr), sin6_scope_id=0}, msg_namelen=28, msg_iov=[{iov_base="C\303\224<\250)H\346C\32\320\303.K/+\177\267\7\360\323\342\276\277\33\227\371\303^YuV"..., iov_len=3327}], msg_iovlen=1, msg_control=[{cmsg_len=28, cmsg_level=SOL_IP, cmsg_type=IP_PKTINFO, cmsg_data={ipi_ifindex=0, ipi_spec_dst=inet_addr("10.0.0.10"), ipi_addr=inet_addr("0.0.0.0")}}, {cmsg_len=20, cmsg_level=SOL_IP, cmsg_type=IP_TOS, cmsg_data=[0x2, 0, 0, 0]}], msg_controllen=56, msg_flags=0}}], 1, 0) = -1 EMSGSIZE (Message too long)

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.