From 23ccf20ea58e87efa965c5ab83c213249bb66f19 Mon Sep 17 00:00:00 2001
From: Apoorv Kothari <apoorv@toidiu.com>
Date: Mon, 19 Jun 2023 15:44:21 -0700
Subject: [PATCH 1/6] ktls: config socket ULP

---
 error/s2n_errno.c                         |   7 +
 error/s2n_errno.h                         |   7 +
 tests/testlib/s2n_connection_test_utils.c |   9 +
 tests/testlib/s2n_testlib.h               |   1 +
 tests/unit/s2n_ktls_test.c                | 216 +++++++++++++++++++++-
 tests/unit/s2n_shutdown_test.c            |   9 -
 tls/s2n_ktls.c                            | 167 ++++++++++++++++-
 tls/s2n_ktls.h                            |  21 ++-
 tls/s2n_ktls_linux.h                      |  31 ++++
 tls/s2n_ktls_unsupported.h                |  26 +++
 10 files changed, 478 insertions(+), 16 deletions(-)
 create mode 100644 tls/s2n_ktls_linux.h
 create mode 100644 tls/s2n_ktls_unsupported.h

diff --git a/error/s2n_errno.c b/error/s2n_errno.c
index c958e9f3065..a80185e859e 100644
--- a/error/s2n_errno.c
+++ b/error/s2n_errno.c
@@ -290,6 +290,13 @@ static const char *no_such_error = "Internal s2n error";
     ERR_ENTRY(S2N_ERR_INTERNAL_LIBCRYPTO_ERROR, "An internal error has occurred in the libcrypto API") \
     ERR_ENTRY(S2N_ERR_NO_RENEGOTIATION, "Only secure, server-initiated renegotiation is supported") \
     ERR_ENTRY(S2N_ERR_APP_DATA_BLOCKED, "Blocked on application data during handshake") \
+    ERR_ENTRY(S2N_ERR_KTLS_ALREADY_ENABLED, "kTLS is already enabled on the socket")  \
+    ERR_ENTRY(S2N_ERR_KTLS_MANAGED_IO, "kTLS cannot be enabled while custom I/O is configured for the connection")  \
+    ERR_ENTRY(S2N_ERR_KTLS_HANDSHAKE_NOT_COMPLETE, "kTLS can only be enabled once the handshake is completed") \
+    ERR_ENTRY(S2N_ERR_KTLS_UNSUPPORTED_PLATFORM, "kTLS is unsupported on this platform") \
+    ERR_ENTRY(S2N_ERR_KTLS_UNSUPPORTED_CONN, "kTLS is unsupported for this connection") \
+    ERR_ENTRY(S2N_ERR_KTLS_ULP, "An error occurred when attempting to configure the socket for kTLS. Ensure the 'tls' kernel module is enabled.")  \
+    ERR_ENTRY(S2N_ERR_KTLS_DISABLED_FOR_TEST, "Some kTLS operations have been disabled for testing")  \
     ERR_ENTRY(S2N_ERR_ATOMIC, "Atomic operations in this environment would require locking") \
     /* clang-format on */
 
diff --git a/error/s2n_errno.h b/error/s2n_errno.h
index dd5b841b1c0..725b42b2e6b 100644
--- a/error/s2n_errno.h
+++ b/error/s2n_errno.h
@@ -306,6 +306,13 @@ typedef enum {
     S2N_ERR_SECRET_SCHEDULE_STATE,
     S2N_ERR_CERT_OWNERSHIP,
     S2N_ERR_INTERNAL_LIBCRYPTO_ERROR,
+    S2N_ERR_KTLS_ALREADY_ENABLED,
+    S2N_ERR_KTLS_MANAGED_IO,
+    S2N_ERR_KTLS_HANDSHAKE_NOT_COMPLETE,
+    S2N_ERR_KTLS_UNSUPPORTED_PLATFORM,
+    S2N_ERR_KTLS_UNSUPPORTED_CONN,
+    S2N_ERR_KTLS_ULP,
+    S2N_ERR_KTLS_DISABLED_FOR_TEST,
     S2N_ERR_ATOMIC,
     S2N_ERR_T_USAGE_END,
 } s2n_error;
diff --git a/tests/testlib/s2n_connection_test_utils.c b/tests/testlib/s2n_connection_test_utils.c
index 977d90843c9..546665d45fa 100644
--- a/tests/testlib/s2n_connection_test_utils.c
+++ b/tests/testlib/s2n_connection_test_utils.c
@@ -332,3 +332,12 @@ S2N_RESULT s2n_set_all_mutually_supported_groups(struct s2n_connection *conn)
 
     return S2N_RESULT_OK;
 }
+
+S2N_RESULT s2n_skip_handshake(struct s2n_connection *conn)
+{
+    conn->handshake.handshake_type = NEGOTIATED | FULL_HANDSHAKE;
+    while (!s2n_handshake_is_complete(conn)) {
+        conn->handshake.message_number++;
+    }
+    return S2N_RESULT_OK;
+}
diff --git a/tests/testlib/s2n_testlib.h b/tests/testlib/s2n_testlib.h
index 9e4dd6d4e99..1cb99b052a0 100644
--- a/tests/testlib/s2n_testlib.h
+++ b/tests/testlib/s2n_testlib.h
@@ -75,6 +75,7 @@ int s2n_connection_allow_response_extension(struct s2n_connection *conn, uint16_
 int s2n_connection_allow_all_response_extensions(struct s2n_connection *conn);
 int s2n_connection_set_all_protocol_versions(struct s2n_connection *conn, uint8_t version);
 S2N_RESULT s2n_set_all_mutually_supported_groups(struct s2n_connection *conn);
+S2N_RESULT s2n_skip_handshake(struct s2n_connection *conn);
 
 S2N_RESULT s2n_connection_set_secrets(struct s2n_connection *conn);
 
diff --git a/tests/unit/s2n_ktls_test.c b/tests/unit/s2n_ktls_test.c
index fc8b427f21d..fcd0e487811 100644
--- a/tests/unit/s2n_ktls_test.c
+++ b/tests/unit/s2n_ktls_test.c
@@ -13,13 +13,53 @@
  * permissions and limitations under the License.
  */
 
-#include "crypto/s2n_cipher.h"
+#include "tls/s2n_ktls.h"
+
 #include "s2n_test.h"
+#include "testlib/s2n_testlib.h"
+
+S2N_RESULT s2n_ktls_retrieve_file_descriptor(struct s2n_connection *conn, s2n_ktls_mode ktls_mode, int *fd);
+S2N_RESULT s2n_disable_ktls_socket_config_for_testing(void);
+
+/* set kTLS supported cipher */
+struct s2n_cipher ktls_temp_supported_cipher = {
+    .ktls_supported = true,
+};
+struct s2n_record_algorithm ktls_temp_supported_record_alg = {
+    .cipher = &ktls_temp_supported_cipher,
+};
+struct s2n_cipher_suite ktls_temp_supported_cipher_suite = {
+    .record_alg = &ktls_temp_supported_record_alg,
+};
+
+S2N_RESULT s2n_test_configure_mock_ktls_connection(struct s2n_connection *conn, int fd, bool complete_handshake)
+{
+    RESULT_ENSURE_REF(conn);
+
+    /* config I/O */
+    RESULT_GUARD_POSIX(s2n_connection_set_write_fd(conn, fd));
+    RESULT_GUARD_POSIX(s2n_connection_set_read_fd(conn, fd));
+    conn->managed_send_io = true;
+    conn->managed_recv_io = true;
+    conn->ktls_send_enabled = false;
+    conn->ktls_recv_enabled = false;
+
+    /* configure connection so that the handshake is complete */
+    conn->secure->cipher_suite = &ktls_temp_supported_cipher_suite;
+    conn->actual_protocol_version = S2N_TLS12;
+    if (complete_handshake) {
+        RESULT_GUARD(s2n_skip_handshake(conn));
+    }
+
+    return S2N_RESULT_OK;
+}
 
 int main(int argc, char **argv)
 {
     BEGIN_TEST();
 
+    EXPECT_OK(s2n_disable_ktls_socket_config_for_testing());
+
     /* ktls_supported ciphers */
     {
         struct s2n_cipher cipher = s2n_aes128_gcm;
@@ -38,5 +78,179 @@ int main(int argc, char **argv)
         EXPECT_FALSE(cipher.ktls_supported);
     };
 
+    if (!s2n_ktls_is_supported_on_platform()) {
+        DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
+                s2n_connection_ptr_free);
+        int fd = 1;
+        EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+
+        EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_UNSUPPORTED_PLATFORM);
+        EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_UNSUPPORTED_PLATFORM);
+    } else {
+        /* ktls handshake must be complete */
+        {
+            DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
+                    s2n_connection_ptr_free);
+            int fd = 1;
+            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, false));
+
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_HANDSHAKE_NOT_COMPLETE);
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_HANDSHAKE_NOT_COMPLETE);
+        };
+
+        /* s2n_connection_ktls_enable */
+        {
+            DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
+                    s2n_connection_ptr_free);
+            int fd = 1;
+            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+        };
+
+        /* ktls already enabled */
+        {
+            DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
+                    s2n_connection_ptr_free);
+            int fd = 1;
+            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+
+            server_conn->ktls_send_enabled = true;
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_ALREADY_ENABLED);
+
+            server_conn->ktls_recv_enabled = true;
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_ALREADY_ENABLED);
+        };
+
+        /* unsupported protocols */
+        {
+            DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
+                    s2n_connection_ptr_free);
+            int fd = 1;
+            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+
+            server_conn->actual_protocol_version = S2N_TLS13;
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_UNSUPPORTED_CONN);
+
+            server_conn->actual_protocol_version = S2N_TLS11;
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_UNSUPPORTED_CONN);
+        };
+
+        /* unsupported ciphers */
+        {
+            /* set kTLS un-supported cipher */
+            struct s2n_cipher ktls_temp_unsupported_cipher = {
+                .ktls_supported = false,
+            };
+            struct s2n_record_algorithm ktls_temp_unsupported_record_alg = {
+                .cipher = &ktls_temp_unsupported_cipher,
+            };
+            struct s2n_cipher_suite ktls_temp_unsupported_cipher_suite = {
+                .record_alg = &ktls_temp_unsupported_record_alg,
+            };
+
+            DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
+                    s2n_connection_ptr_free);
+            int fd = 1;
+            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+
+            /* base case */
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+
+            server_conn->ktls_send_enabled = false; /* reset ktls enable connection */
+            server_conn->secure->cipher_suite = &ktls_temp_unsupported_cipher_suite;
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_UNSUPPORTED_CONN);
+        };
+
+        /* drain buffer prior to enabling kTLS */
+        {
+            DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
+                    s2n_connection_ptr_free);
+            int fd = 1;
+            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+
+            /* base case */
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+
+            uint8_t write_byte = 8;
+            uint8_t read_byte = 0;
+            /* write to conn->out buffer and assert error */
+            EXPECT_SUCCESS(s2n_stuffer_write_bytes(&server_conn->out, &write_byte, 1));
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_RECORD_STUFFER_NEEDS_DRAINING);
+            /* drain conn->out buffer and assert base case */
+            EXPECT_SUCCESS(s2n_stuffer_read_bytes(&server_conn->out, &read_byte, 1));
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+
+            /* write to conn->in buffer and assert error */
+            EXPECT_SUCCESS(s2n_stuffer_write_bytes(&server_conn->in, &write_byte, 1));
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_RECORD_STUFFER_NEEDS_DRAINING);
+            /* drain conn->in buffer and assert base case */
+            EXPECT_SUCCESS(s2n_stuffer_read_bytes(&server_conn->in, &read_byte, 1));
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+        };
+
+        /* managed_send_io */
+        {
+            DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
+                    s2n_connection_ptr_free);
+            int fd = 1;
+            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+
+            /* base case */
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+
+            server_conn->ktls_send_enabled = false; /* reset ktls enable connection */
+            /* expect failure if connection is using custom IO */
+            server_conn->managed_send_io = false;
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_MANAGED_IO);
+
+            server_conn->ktls_send_enabled = false; /* reset ktls enable connection */
+            /* expect success if connection is NOT using custom IO */
+            server_conn->managed_send_io = true;
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+        };
+
+        /* managed_recv_io */
+        {
+            DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
+                    s2n_connection_ptr_free);
+            int fd = 1;
+            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+
+            /* base case */
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+
+            server_conn->ktls_recv_enabled = false; /* reset ktls enable connection */
+            /* recv managed io */
+            server_conn->managed_recv_io = false;
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_MANAGED_IO);
+
+            server_conn->ktls_recv_enabled = false; /* reset ktls enable connection */
+            /* expect success if connection is NOT using custom IO */
+            server_conn->managed_recv_io = true;
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+        };
+
+        /* s2n_ktls_retrieve_file_descriptor */
+        {
+            DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
+                    s2n_connection_ptr_free);
+            int write_fd_orig = 1;
+            int read_fd_orig = 2;
+            int fd_ret = 0;
+
+            EXPECT_SUCCESS(s2n_connection_set_write_fd(server_conn, write_fd_orig));
+            EXPECT_OK(s2n_ktls_retrieve_file_descriptor(server_conn, S2N_KTLS_MODE_SEND, &fd_ret));
+            EXPECT_EQUAL(write_fd_orig, fd_ret);
+
+            EXPECT_SUCCESS(s2n_connection_set_read_fd(server_conn, read_fd_orig));
+            EXPECT_OK(s2n_ktls_retrieve_file_descriptor(server_conn, S2N_KTLS_MODE_RECV, &fd_ret));
+            EXPECT_EQUAL(read_fd_orig, fd_ret);
+        };
+    }
+
     END_TEST();
 }
diff --git a/tests/unit/s2n_shutdown_test.c b/tests/unit/s2n_shutdown_test.c
index e0f1e09c2a3..91efa34020a 100644
--- a/tests/unit/s2n_shutdown_test.c
+++ b/tests/unit/s2n_shutdown_test.c
@@ -21,15 +21,6 @@
 
 #define ALERT_LEN (sizeof(uint16_t))
 
-static S2N_RESULT s2n_skip_handshake(struct s2n_connection *conn)
-{
-    conn->handshake.handshake_type = NEGOTIATED | FULL_HANDSHAKE;
-    while (!s2n_handshake_is_complete(conn)) {
-        conn->handshake.message_number++;
-    }
-    return S2N_RESULT_OK;
-}
-
 int main(int argc, char **argv)
 {
     BEGIN_TEST();
diff --git a/tls/s2n_ktls.c b/tls/s2n_ktls.c
index 2900337a83f..6a77eda2b3a 100644
--- a/tls/s2n_ktls.c
+++ b/tls/s2n_ktls.c
@@ -15,11 +15,176 @@
 
 #include "tls/s2n_ktls.h"
 
+#include <sys/socket.h>
+
+#include "utils/s2n_socket.h"
+
+/* These variables are used to disable ktls mechanisms during testing. */
+static bool disable_ktls_socket_config_for_testing = false;
+
 bool s2n_ktls_is_supported_on_platform()
 {
-#if defined(__linux__)
+#if S2N_KTLS_SUPPORTED
     return true;
 #else
     return false;
 #endif
 }
+
+S2N_RESULT s2n_ktls_validate(struct s2n_connection *conn, s2n_ktls_mode ktls_mode)
+{
+    RESULT_ENSURE_REF(conn);
+    RESULT_ENSURE_REF(conn->secure);
+    RESULT_ENSURE_REF(conn->secure->cipher_suite);
+    RESULT_ENSURE_REF(conn->secure->cipher_suite->record_alg);
+    const struct s2n_cipher *cipher = conn->secure->cipher_suite->record_alg->cipher;
+    RESULT_ENSURE_REF(cipher);
+
+    /* kTLS enable should only be called once the handshake has completed. */
+    if (!is_handshake_complete(conn)) {
+        RESULT_BAIL(S2N_ERR_KTLS_HANDSHAKE_NOT_COMPLETE);
+    }
+
+    /* TODO support TLS 1.3
+     *
+     * TLS 1.3 support requires sending the KeyUpdate message when the cryptographic
+     * KeyLimits are met. However, this is currently only possible by applying a
+     * kernel patch to support this functionality.
+     */
+    RESULT_ENSURE(conn->actual_protocol_version == S2N_TLS12, S2N_ERR_KTLS_UNSUPPORTED_CONN);
+
+    /* Check if the cipher supports kTLS */
+    RESULT_ENSURE(cipher->ktls_supported, S2N_ERR_KTLS_UNSUPPORTED_CONN);
+
+    /* kTLS I/O functionality is managed by s2n-tls. kTLS cannot be enabled if the
+     * application sets custom I/O (managed_send_io == false means application has
+     * set custom I/O).
+     */
+    switch (ktls_mode) {
+        case S2N_KTLS_MODE_SEND:
+            RESULT_ENSURE_REF(conn->send_io_context);
+            RESULT_ENSURE(conn->managed_send_io, S2N_ERR_KTLS_MANAGED_IO);
+
+            /* The output stuffer should be empty before enabling kTLS. */
+            RESULT_ENSURE(s2n_stuffer_data_available(&conn->out) == 0, S2N_ERR_RECORD_STUFFER_NEEDS_DRAINING);
+
+            break;
+        case S2N_KTLS_MODE_RECV:
+            RESULT_ENSURE_REF(conn->recv_io_context);
+            RESULT_ENSURE(conn->managed_recv_io, S2N_ERR_KTLS_MANAGED_IO);
+
+            /* The input stuffer should be empty before enabling kTLS. */
+            RESULT_ENSURE(s2n_stuffer_data_available(&conn->in) == 0, S2N_ERR_RECORD_STUFFER_NEEDS_DRAINING);
+
+            break;
+    }
+
+    return S2N_RESULT_OK;
+}
+
+S2N_RESULT s2n_ktls_retrieve_file_descriptor(struct s2n_connection *conn, s2n_ktls_mode ktls_mode, int *fd)
+{
+    RESULT_ENSURE_REF(conn);
+    RESULT_ENSURE_REF(fd);
+
+    if (ktls_mode == S2N_KTLS_MODE_RECV) {
+        /* retrieve the receive fd */
+        RESULT_ENSURE_REF(conn->recv_io_context);
+        const struct s2n_socket_read_io_context *peer_socket_ctx = conn->recv_io_context;
+        *fd = peer_socket_ctx->fd;
+    } else if (ktls_mode == S2N_KTLS_MODE_SEND) {
+        /* retrieve the send fd */
+        RESULT_ENSURE_REF(conn->send_io_context);
+        const struct s2n_socket_write_io_context *peer_socket_ctx = conn->send_io_context;
+        *fd = peer_socket_ctx->fd;
+    }
+
+    return S2N_RESULT_OK;
+}
+
+static S2N_RESULT s2n_ktls_configure_socket(struct s2n_connection *conn, s2n_ktls_mode ktls_mode)
+{
+    RESULT_ENSURE_REF(conn);
+
+    /* If already enabled then return success */
+    if (ktls_mode == S2N_KTLS_MODE_SEND && conn->ktls_send_enabled) {
+        RESULT_BAIL(S2N_ERR_KTLS_ALREADY_ENABLED);
+    }
+    if (ktls_mode == S2N_KTLS_MODE_RECV && conn->ktls_recv_enabled) {
+        RESULT_BAIL(S2N_ERR_KTLS_ALREADY_ENABLED);
+    }
+
+    int fd = 0;
+    RESULT_GUARD(s2n_ktls_retrieve_file_descriptor(conn, ktls_mode, &fd));
+
+    /* Calls to setsockopt require a real socket, which is not used in unit tests. */
+    RESULT_ENSURE(!disable_ktls_socket_config_for_testing, S2N_ERR_KTLS_DISABLED_FOR_TEST);
+
+    /* Enable 'tls' ULP for the socket. https://lwn.net/Articles/730207 */
+    int ret = setsockopt(fd, S2N_SOL_TCP, S2N_TCP_ULP, S2N_TLS_ULP_NAME, S2N_TLS_ULP_NAME_SIZE);
+
+    if (ret != 0) {
+        /* EEXIST: https://man7.org/linux/man-pages/man3/errno.3.html
+         *
+         * TCP_ULP has already been enabled on the socket so the operation is a noop.
+         * Since its possible to call this twice, once for TX and once for RX, consider
+         * the noop a success. */
+        if (errno != EEXIST) {
+            RESULT_BAIL(S2N_ERR_KTLS_ULP);
+        }
+    }
+
+    return S2N_RESULT_OK;
+}
+
+/*
+ * Since kTLS is an optimization, it is possible to continue operation
+ * by using userspace TLS if kTLS is not supported. Upon successfully
+ * enabling kTLS, we set connection->ktls_send_enabled (and recv) to true.
+ *
+ * kTLS configuration errors are recoverable since calls to setsockopt are
+ * non-destructive and its possible to fallback to userspace.
+ */
+int s2n_connection_ktls_enable_send(struct s2n_connection *conn)
+{
+    POSIX_ENSURE_REF(conn);
+
+    if (!s2n_ktls_is_supported_on_platform()) {
+        POSIX_BAIL(S2N_ERR_KTLS_UNSUPPORTED_PLATFORM);
+    }
+
+    POSIX_GUARD_RESULT(s2n_ktls_validate(conn, S2N_KTLS_MODE_SEND));
+
+    POSIX_GUARD_RESULT(s2n_ktls_configure_socket(conn, S2N_KTLS_MODE_SEND));
+
+    return S2N_SUCCESS;
+}
+
+int s2n_connection_ktls_enable_recv(struct s2n_connection *conn)
+{
+    POSIX_ENSURE_REF(conn);
+
+    if (!s2n_ktls_is_supported_on_platform()) {
+        POSIX_BAIL(S2N_ERR_KTLS_UNSUPPORTED_PLATFORM);
+    }
+
+    POSIX_GUARD_RESULT(s2n_ktls_validate(conn, S2N_KTLS_MODE_RECV));
+
+    POSIX_GUARD_RESULT(s2n_ktls_configure_socket(conn, S2N_KTLS_MODE_RECV));
+
+    return S2N_SUCCESS;
+}
+
+/* Use for testing only.
+ *
+ * This function disables the setsockopt call to enable ULP. Calls to setsockopt
+ * require a real socket, which is not used in unit tests.
+ */
+S2N_RESULT s2n_disable_ktls_socket_config_for_testing(void)
+{
+    RESULT_ENSURE(s2n_in_unit_test(), S2N_ERR_NOT_IN_UNIT_TEST);
+
+    disable_ktls_socket_config_for_testing = true;
+
+    return S2N_RESULT_OK;
+}
diff --git a/tls/s2n_ktls.h b/tls/s2n_ktls.h
index 5e2de31955f..ba2151a7e50 100644
--- a/tls/s2n_ktls.h
+++ b/tls/s2n_ktls.h
@@ -17,19 +17,30 @@
 
 #include "tls/s2n_config.h"
 
+/* Define headers needed to enable and use kTLS.
+ *  *
+ *   * The inline header definitions are required to compile kTLS specific code.
+ *    * kTLS has been tested on linux. For all other platforms, kTLS is marked as
+ *     * unsuppored, and will return an un-supported error.
+ *      */
+#if defined(__linux__)
+    #define S2N_KTLS_SUPPORTED true
+    #include "tls/s2n_ktls_linux.h"
+#else
+    #define S2N_KTLS_SUPPORTED false
+    #include "tls/s2n_ktls_unsupported.h"
+#endif
+
 /* A set of kTLS configurations representing the combination of sending
  * and receiving.
  */
 typedef enum {
-    /* Disable kTLS. */
-    S2N_KTLS_MODE_DISABLED,
     /* Enable kTLS for the send socket. */
     S2N_KTLS_MODE_SEND,
     /* Enable kTLS for the receive socket. */
     S2N_KTLS_MODE_RECV,
-    /* Enable kTLS for both receive and send sockets. */
-    S2N_KTLS_MODE_DUPLEX,
 } s2n_ktls_mode;
 
-int s2n_config_set_ktls_mode(struct s2n_config *config, s2n_ktls_mode ktls_mode);
 bool s2n_ktls_is_supported_on_platform();
+int s2n_connection_ktls_enable_send(struct s2n_connection *conn);
+int s2n_connection_ktls_enable_recv(struct s2n_connection *conn);
diff --git a/tls/s2n_ktls_linux.h b/tls/s2n_ktls_linux.h
new file mode 100644
index 00000000000..cd1268db347
--- /dev/null
+++ b/tls/s2n_ktls_linux.h
@@ -0,0 +1,31 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+#pragma once
+
+/*
+ * Linux doesn't expose kTLS headers in its uapi. Its possible to get these headers
+ * via glibc but support can vary depending on the version of glibc on the host.
+ * Instead we define linux specific values inline.
+ *
+ * - https://elixir.bootlin.com/linux/v6.3.8/A/ident/TCP_ULP
+ * - https://elixir.bootlin.com/linux/v6.3.8/A/ident/SOL_TCP
+ */
+
+/* socket definitions */
+#define S2N_TLS_ULP_NAME      "tls"
+#define S2N_TLS_ULP_NAME_SIZE sizeof(S2N_TLS_ULP_NAME)
+#define S2N_TCP_ULP           31 /* Attach a ULP to a TCP connection.  */
+#define S2N_SOL_TCP           6  /* TCP level */
diff --git a/tls/s2n_ktls_unsupported.h b/tls/s2n_ktls_unsupported.h
new file mode 100644
index 00000000000..5e30931e4e6
--- /dev/null
+++ b/tls/s2n_ktls_unsupported.h
@@ -0,0 +1,26 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ *  http://aws.amazon.com/apache2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+#pragma once
+
+/*
+ * For unsupported platforms 0-init all values.
+ */
+
+/* socket definitions */
+#define S2N_TLS_ULP_NAME      "tls"
+#define S2N_TLS_ULP_NAME_SIZE sizeof(S2N_TLS_ULP_NAME)
+#define S2N_TCP_ULP           0
+#define S2N_SOL_TCP           0

From 6fc5620ca544964ab27b3ed3ca224f12c2adbe6a Mon Sep 17 00:00:00 2001
From: Apoorv Kothari <apoorv@toidiu.com>
Date: Fri, 7 Jul 2023 14:57:57 -0700
Subject: [PATCH 2/6] feedback

---
 error/s2n_errno.c          |  4 ++--
 error/s2n_errno.h          |  2 +-
 tests/unit/s2n_ktls_test.c | 10 ++++-----
 tls/s2n_ktls.c             | 44 ++++++++++++++++++++------------------
 tls/s2n_ktls.h             | 10 ++++-----
 5 files changed, 36 insertions(+), 34 deletions(-)

diff --git a/error/s2n_errno.c b/error/s2n_errno.c
index a80185e859e..b85a00e96e7 100644
--- a/error/s2n_errno.c
+++ b/error/s2n_errno.c
@@ -290,9 +290,9 @@ static const char *no_such_error = "Internal s2n error";
     ERR_ENTRY(S2N_ERR_INTERNAL_LIBCRYPTO_ERROR, "An internal error has occurred in the libcrypto API") \
     ERR_ENTRY(S2N_ERR_NO_RENEGOTIATION, "Only secure, server-initiated renegotiation is supported") \
     ERR_ENTRY(S2N_ERR_APP_DATA_BLOCKED, "Blocked on application data during handshake") \
-    ERR_ENTRY(S2N_ERR_KTLS_ALREADY_ENABLED, "kTLS is already enabled on the socket")  \
     ERR_ENTRY(S2N_ERR_KTLS_MANAGED_IO, "kTLS cannot be enabled while custom I/O is configured for the connection")  \
-    ERR_ENTRY(S2N_ERR_KTLS_HANDSHAKE_NOT_COMPLETE, "kTLS can only be enabled once the handshake is completed") \
+    ERR_ENTRY(S2N_ERR_KTLS_ALREADY_ENABLED, "kTLS is already enabled on the socket")  \
+    ERR_ENTRY(S2N_ERR_HANDSHAKE_NOT_COMPLETE, "Operation is only allowed after the handshake is complete") \
     ERR_ENTRY(S2N_ERR_KTLS_UNSUPPORTED_PLATFORM, "kTLS is unsupported on this platform") \
     ERR_ENTRY(S2N_ERR_KTLS_UNSUPPORTED_CONN, "kTLS is unsupported for this connection") \
     ERR_ENTRY(S2N_ERR_KTLS_ULP, "An error occurred when attempting to configure the socket for kTLS. Ensure the 'tls' kernel module is enabled.")  \
diff --git a/error/s2n_errno.h b/error/s2n_errno.h
index 725b42b2e6b..0e360158442 100644
--- a/error/s2n_errno.h
+++ b/error/s2n_errno.h
@@ -306,9 +306,9 @@ typedef enum {
     S2N_ERR_SECRET_SCHEDULE_STATE,
     S2N_ERR_CERT_OWNERSHIP,
     S2N_ERR_INTERNAL_LIBCRYPTO_ERROR,
+    S2N_ERR_HANDSHAKE_NOT_COMPLETE,
     S2N_ERR_KTLS_ALREADY_ENABLED,
     S2N_ERR_KTLS_MANAGED_IO,
-    S2N_ERR_KTLS_HANDSHAKE_NOT_COMPLETE,
     S2N_ERR_KTLS_UNSUPPORTED_PLATFORM,
     S2N_ERR_KTLS_UNSUPPORTED_CONN,
     S2N_ERR_KTLS_ULP,
diff --git a/tests/unit/s2n_ktls_test.c b/tests/unit/s2n_ktls_test.c
index fcd0e487811..51297f272aa 100644
--- a/tests/unit/s2n_ktls_test.c
+++ b/tests/unit/s2n_ktls_test.c
@@ -94,8 +94,8 @@ int main(int argc, char **argv)
             int fd = 1;
             EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, false));
 
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_HANDSHAKE_NOT_COMPLETE);
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_HANDSHAKE_NOT_COMPLETE);
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_HANDSHAKE_NOT_COMPLETE);
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_HANDSHAKE_NOT_COMPLETE);
         };
 
         /* s2n_connection_ktls_enable */
@@ -110,7 +110,7 @@ int main(int argc, char **argv)
             EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
         };
 
-        /* ktls already enabled */
+        /* ktls already enabled is a noop and returns success */
         {
             DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
                     s2n_connection_ptr_free);
@@ -118,10 +118,10 @@ int main(int argc, char **argv)
             EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
 
             server_conn->ktls_send_enabled = true;
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_ALREADY_ENABLED);
+            EXPECT_SUCCESS(s2n_connection_ktls_enable_send(server_conn));
 
             server_conn->ktls_recv_enabled = true;
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_ALREADY_ENABLED);
+            EXPECT_SUCCESS(s2n_connection_ktls_enable_recv(server_conn));
         };
 
         /* unsupported protocols */
diff --git a/tls/s2n_ktls.c b/tls/s2n_ktls.c
index 6a77eda2b3a..786add2f914 100644
--- a/tls/s2n_ktls.c
+++ b/tls/s2n_ktls.c
@@ -42,7 +42,7 @@ S2N_RESULT s2n_ktls_validate(struct s2n_connection *conn, s2n_ktls_mode ktls_mod
 
     /* kTLS enable should only be called once the handshake has completed. */
     if (!is_handshake_complete(conn)) {
-        RESULT_BAIL(S2N_ERR_KTLS_HANDSHAKE_NOT_COMPLETE);
+        RESULT_BAIL(S2N_ERR_HANDSHAKE_NOT_COMPLETE);
     }
 
     /* TODO support TLS 1.3
@@ -76,6 +76,9 @@ S2N_RESULT s2n_ktls_validate(struct s2n_connection *conn, s2n_ktls_mode ktls_mod
             /* The input stuffer should be empty before enabling kTLS. */
             RESULT_ENSURE(s2n_stuffer_data_available(&conn->in) == 0, S2N_ERR_RECORD_STUFFER_NEEDS_DRAINING);
 
+            break;
+        default:
+            RESULT_BAIL(S2N_ERR_SAFETY);
             break;
     }
 
@@ -88,15 +91,9 @@ S2N_RESULT s2n_ktls_retrieve_file_descriptor(struct s2n_connection *conn, s2n_kt
     RESULT_ENSURE_REF(fd);
 
     if (ktls_mode == S2N_KTLS_MODE_RECV) {
-        /* retrieve the receive fd */
-        RESULT_ENSURE_REF(conn->recv_io_context);
-        const struct s2n_socket_read_io_context *peer_socket_ctx = conn->recv_io_context;
-        *fd = peer_socket_ctx->fd;
+        RESULT_GUARD_POSIX(s2n_connection_get_read_fd(conn, fd));
     } else if (ktls_mode == S2N_KTLS_MODE_SEND) {
-        /* retrieve the send fd */
-        RESULT_ENSURE_REF(conn->send_io_context);
-        const struct s2n_socket_write_io_context *peer_socket_ctx = conn->send_io_context;
-        *fd = peer_socket_ctx->fd;
+        RESULT_GUARD_POSIX(s2n_connection_get_write_fd(conn, fd));
     }
 
     return S2N_RESULT_OK;
@@ -124,14 +121,11 @@ static S2N_RESULT s2n_ktls_configure_socket(struct s2n_connection *conn, s2n_ktl
     int ret = setsockopt(fd, S2N_SOL_TCP, S2N_TCP_ULP, S2N_TLS_ULP_NAME, S2N_TLS_ULP_NAME_SIZE);
 
     if (ret != 0) {
-        /* EEXIST: https://man7.org/linux/man-pages/man3/errno.3.html
-         *
-         * TCP_ULP has already been enabled on the socket so the operation is a noop.
-         * Since its possible to call this twice, once for TX and once for RX, consider
-         * the noop a success. */
-        if (errno != EEXIST) {
-            RESULT_BAIL(S2N_ERR_KTLS_ULP);
-        }
+        /* https://man7.org/linux/man-pages/man3/errno.3.html
+         * EEXIST indicates that TCP_ULP has already been enabled on the socket.
+         * This is a noop and therefore safe to ignore.
+         */
+        RESULT_ENSURE(errno == EEXIST, S2N_ERR_KTLS_ULP);
     }
 
     return S2N_RESULT_OK;
@@ -154,8 +148,12 @@ int s2n_connection_ktls_enable_send(struct s2n_connection *conn)
     }
 
     POSIX_GUARD_RESULT(s2n_ktls_validate(conn, S2N_KTLS_MODE_SEND));
-
-    POSIX_GUARD_RESULT(s2n_ktls_configure_socket(conn, S2N_KTLS_MODE_SEND));
+    s2n_result res = s2n_ktls_configure_socket(conn, S2N_KTLS_MODE_SEND);
+    if (s2n_errno == S2N_ERR_KTLS_ALREADY_ENABLED) {
+        return S2N_SUCCESS;
+    } else {
+        POSIX_GUARD_RESULT(res);
+    }
 
     return S2N_SUCCESS;
 }
@@ -169,8 +167,12 @@ int s2n_connection_ktls_enable_recv(struct s2n_connection *conn)
     }
 
     POSIX_GUARD_RESULT(s2n_ktls_validate(conn, S2N_KTLS_MODE_RECV));
-
-    POSIX_GUARD_RESULT(s2n_ktls_configure_socket(conn, S2N_KTLS_MODE_RECV));
+    s2n_result res = s2n_ktls_configure_socket(conn, S2N_KTLS_MODE_RECV);
+    if (s2n_errno == S2N_ERR_KTLS_ALREADY_ENABLED) {
+        return S2N_SUCCESS;
+    } else {
+        POSIX_GUARD_RESULT(res);
+    }
 
     return S2N_SUCCESS;
 }
diff --git a/tls/s2n_ktls.h b/tls/s2n_ktls.h
index ba2151a7e50..7b521ff1ae9 100644
--- a/tls/s2n_ktls.h
+++ b/tls/s2n_ktls.h
@@ -18,11 +18,11 @@
 #include "tls/s2n_config.h"
 
 /* Define headers needed to enable and use kTLS.
- *  *
- *   * The inline header definitions are required to compile kTLS specific code.
- *    * kTLS has been tested on linux. For all other platforms, kTLS is marked as
- *     * unsuppored, and will return an un-supported error.
- *      */
+ *
+ * The inline header definitions are required to compile kTLS specific code.
+ * kTLS has been tested on linux. For all other platforms, kTLS is marked as
+ * unsupported, and will return an unsupported error.
+ */
 #if defined(__linux__)
     #define S2N_KTLS_SUPPORTED true
     #include "tls/s2n_ktls_linux.h"

From ce9c359731eee865ede1f3960ebba0959ca42687 Mon Sep 17 00:00:00 2001
From: Apoorv Kothari <apoorv@toidiu.com>
Date: Mon, 10 Jul 2023 23:26:29 -0700
Subject: [PATCH 3/6] feedback2

---
 error/s2n_errno.c          |  1 -
 error/s2n_errno.h          |  1 -
 tests/unit/s2n_ktls_test.c | 43 +++++++++++---------------------------
 tls/s2n_ktls.c             | 34 +++++++-----------------------
 tls/s2n_ktls.h             |  2 ++
 5 files changed, 22 insertions(+), 59 deletions(-)

diff --git a/error/s2n_errno.c b/error/s2n_errno.c
index b85a00e96e7..c4ee84fe17b 100644
--- a/error/s2n_errno.c
+++ b/error/s2n_errno.c
@@ -296,7 +296,6 @@ static const char *no_such_error = "Internal s2n error";
     ERR_ENTRY(S2N_ERR_KTLS_UNSUPPORTED_PLATFORM, "kTLS is unsupported on this platform") \
     ERR_ENTRY(S2N_ERR_KTLS_UNSUPPORTED_CONN, "kTLS is unsupported for this connection") \
     ERR_ENTRY(S2N_ERR_KTLS_ULP, "An error occurred when attempting to configure the socket for kTLS. Ensure the 'tls' kernel module is enabled.")  \
-    ERR_ENTRY(S2N_ERR_KTLS_DISABLED_FOR_TEST, "Some kTLS operations have been disabled for testing")  \
     ERR_ENTRY(S2N_ERR_ATOMIC, "Atomic operations in this environment would require locking") \
     /* clang-format on */
 
diff --git a/error/s2n_errno.h b/error/s2n_errno.h
index 0e360158442..4b1500e0549 100644
--- a/error/s2n_errno.h
+++ b/error/s2n_errno.h
@@ -312,7 +312,6 @@ typedef enum {
     S2N_ERR_KTLS_UNSUPPORTED_PLATFORM,
     S2N_ERR_KTLS_UNSUPPORTED_CONN,
     S2N_ERR_KTLS_ULP,
-    S2N_ERR_KTLS_DISABLED_FOR_TEST,
     S2N_ERR_ATOMIC,
     S2N_ERR_T_USAGE_END,
 } s2n_error;
diff --git a/tests/unit/s2n_ktls_test.c b/tests/unit/s2n_ktls_test.c
index 51297f272aa..c869c197150 100644
--- a/tests/unit/s2n_ktls_test.c
+++ b/tests/unit/s2n_ktls_test.c
@@ -19,7 +19,6 @@
 #include "testlib/s2n_testlib.h"
 
 S2N_RESULT s2n_ktls_retrieve_file_descriptor(struct s2n_connection *conn, s2n_ktls_mode ktls_mode, int *fd);
-S2N_RESULT s2n_disable_ktls_socket_config_for_testing(void);
 
 /* set kTLS supported cipher */
 struct s2n_cipher ktls_temp_supported_cipher = {
@@ -58,8 +57,6 @@ int main(int argc, char **argv)
 {
     BEGIN_TEST();
 
-    EXPECT_OK(s2n_disable_ktls_socket_config_for_testing());
-
     /* ktls_supported ciphers */
     {
         struct s2n_cipher cipher = s2n_aes128_gcm;
@@ -105,9 +102,10 @@ int main(int argc, char **argv)
             int fd = 1;
             EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
 
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
-
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+            EXPECT_SUCCESS(s2n_connection_ktls_enable_send(server_conn));
+            EXPECT_TRUE(server_conn->ktls_send_enabled);
+            EXPECT_SUCCESS(s2n_connection_ktls_enable_recv(server_conn));
+            EXPECT_TRUE(server_conn->ktls_recv_enabled);
         };
 
         /* ktls already enabled is a noop and returns success */
@@ -140,7 +138,7 @@ int main(int argc, char **argv)
 
         /* unsupported ciphers */
         {
-            /* set kTLS un-supported cipher */
+            /* set kTLS unsupported cipher */
             struct s2n_cipher ktls_temp_unsupported_cipher = {
                 .ktls_supported = false,
             };
@@ -156,12 +154,9 @@ int main(int argc, char **argv)
             int fd = 1;
             EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
 
-            /* base case */
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
-
-            server_conn->ktls_send_enabled = false; /* reset ktls enable connection */
             server_conn->secure->cipher_suite = &ktls_temp_unsupported_cipher_suite;
             EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_UNSUPPORTED_CONN);
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_UNSUPPORTED_CONN);
         };
 
         /* drain buffer prior to enabling kTLS */
@@ -171,25 +166,21 @@ int main(int argc, char **argv)
             int fd = 1;
             EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
 
-            /* base case */
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
-
             uint8_t write_byte = 8;
             uint8_t read_byte = 0;
             /* write to conn->out buffer and assert error */
             EXPECT_SUCCESS(s2n_stuffer_write_bytes(&server_conn->out, &write_byte, 1));
             EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_RECORD_STUFFER_NEEDS_DRAINING);
-            /* drain conn->out buffer and assert base case */
+            /* drain conn->out buffer and assert success case */
             EXPECT_SUCCESS(s2n_stuffer_read_bytes(&server_conn->out, &read_byte, 1));
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+            EXPECT_SUCCESS(s2n_connection_ktls_enable_send(server_conn));
 
             /* write to conn->in buffer and assert error */
             EXPECT_SUCCESS(s2n_stuffer_write_bytes(&server_conn->in, &write_byte, 1));
             EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_RECORD_STUFFER_NEEDS_DRAINING);
-            /* drain conn->in buffer and assert base case */
+            /* drain conn->in buffer and assert success case */
             EXPECT_SUCCESS(s2n_stuffer_read_bytes(&server_conn->in, &read_byte, 1));
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+            EXPECT_SUCCESS(s2n_connection_ktls_enable_recv(server_conn));
         };
 
         /* managed_send_io */
@@ -199,18 +190,13 @@ int main(int argc, char **argv)
             int fd = 1;
             EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
 
-            /* base case */
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
-
-            server_conn->ktls_send_enabled = false; /* reset ktls enable connection */
             /* expect failure if connection is using custom IO */
             server_conn->managed_send_io = false;
             EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_MANAGED_IO);
 
-            server_conn->ktls_send_enabled = false; /* reset ktls enable connection */
             /* expect success if connection is NOT using custom IO */
             server_conn->managed_send_io = true;
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+            EXPECT_SUCCESS(s2n_connection_ktls_enable_send(server_conn));
         };
 
         /* managed_recv_io */
@@ -220,18 +206,13 @@ int main(int argc, char **argv)
             int fd = 1;
             EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
 
-            /* base case */
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
-
-            server_conn->ktls_recv_enabled = false; /* reset ktls enable connection */
             /* recv managed io */
             server_conn->managed_recv_io = false;
             EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_MANAGED_IO);
 
-            server_conn->ktls_recv_enabled = false; /* reset ktls enable connection */
             /* expect success if connection is NOT using custom IO */
             server_conn->managed_recv_io = true;
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_DISABLED_FOR_TEST);
+            EXPECT_SUCCESS(s2n_connection_ktls_enable_recv(server_conn));
         };
 
         /* s2n_ktls_retrieve_file_descriptor */
diff --git a/tls/s2n_ktls.c b/tls/s2n_ktls.c
index 786add2f914..b503af5f3b6 100644
--- a/tls/s2n_ktls.c
+++ b/tls/s2n_ktls.c
@@ -19,9 +19,6 @@
 
 #include "utils/s2n_socket.h"
 
-/* These variables are used to disable ktls mechanisms during testing. */
-static bool disable_ktls_socket_config_for_testing = false;
-
 bool s2n_ktls_is_supported_on_platform()
 {
 #if S2N_KTLS_SUPPORTED
@@ -62,20 +59,14 @@ S2N_RESULT s2n_ktls_validate(struct s2n_connection *conn, s2n_ktls_mode ktls_mod
      */
     switch (ktls_mode) {
         case S2N_KTLS_MODE_SEND:
-            RESULT_ENSURE_REF(conn->send_io_context);
             RESULT_ENSURE(conn->managed_send_io, S2N_ERR_KTLS_MANAGED_IO);
-
             /* The output stuffer should be empty before enabling kTLS. */
             RESULT_ENSURE(s2n_stuffer_data_available(&conn->out) == 0, S2N_ERR_RECORD_STUFFER_NEEDS_DRAINING);
-
             break;
         case S2N_KTLS_MODE_RECV:
-            RESULT_ENSURE_REF(conn->recv_io_context);
             RESULT_ENSURE(conn->managed_recv_io, S2N_ERR_KTLS_MANAGED_IO);
-
             /* The input stuffer should be empty before enabling kTLS. */
             RESULT_ENSURE(s2n_stuffer_data_available(&conn->in) == 0, S2N_ERR_RECORD_STUFFER_NEEDS_DRAINING);
-
             break;
         default:
             RESULT_BAIL(S2N_ERR_SAFETY);
@@ -115,7 +106,9 @@ static S2N_RESULT s2n_ktls_configure_socket(struct s2n_connection *conn, s2n_ktl
     RESULT_GUARD(s2n_ktls_retrieve_file_descriptor(conn, ktls_mode, &fd));
 
     /* Calls to setsockopt require a real socket, which is not used in unit tests. */
-    RESULT_ENSURE(!disable_ktls_socket_config_for_testing, S2N_ERR_KTLS_DISABLED_FOR_TEST);
+    if (s2n_in_unit_test()) {
+        return S2N_RESULT_OK;
+    }
 
     /* Enable 'tls' ULP for the socket. https://lwn.net/Articles/730207 */
     int ret = setsockopt(fd, S2N_SOL_TCP, S2N_TCP_ULP, S2N_TLS_ULP_NAME, S2N_TLS_ULP_NAME_SIZE);
@@ -133,8 +126,7 @@ static S2N_RESULT s2n_ktls_configure_socket(struct s2n_connection *conn, s2n_ktl
 
 /*
  * Since kTLS is an optimization, it is possible to continue operation
- * by using userspace TLS if kTLS is not supported. Upon successfully
- * enabling kTLS, we set connection->ktls_send_enabled (and recv) to true.
+ * by using userspace TLS if kTLS is not supported.
  *
  * kTLS configuration errors are recoverable since calls to setsockopt are
  * non-destructive and its possible to fallback to userspace.
@@ -155,6 +147,8 @@ int s2n_connection_ktls_enable_send(struct s2n_connection *conn)
         POSIX_GUARD_RESULT(res);
     }
 
+    conn->ktls_send_enabled = true;
+
     return S2N_SUCCESS;
 }
 
@@ -174,19 +168,7 @@ int s2n_connection_ktls_enable_recv(struct s2n_connection *conn)
         POSIX_GUARD_RESULT(res);
     }
 
-    return S2N_SUCCESS;
-}
-
-/* Use for testing only.
- *
- * This function disables the setsockopt call to enable ULP. Calls to setsockopt
- * require a real socket, which is not used in unit tests.
- */
-S2N_RESULT s2n_disable_ktls_socket_config_for_testing(void)
-{
-    RESULT_ENSURE(s2n_in_unit_test(), S2N_ERR_NOT_IN_UNIT_TEST);
-
-    disable_ktls_socket_config_for_testing = true;
+    conn->ktls_recv_enabled = true;
 
-    return S2N_RESULT_OK;
+    return S2N_SUCCESS;
 }
diff --git a/tls/s2n_ktls.h b/tls/s2n_ktls.h
index 7b521ff1ae9..f6d9fbb431c 100644
--- a/tls/s2n_ktls.h
+++ b/tls/s2n_ktls.h
@@ -42,5 +42,7 @@ typedef enum {
 } s2n_ktls_mode;
 
 bool s2n_ktls_is_supported_on_platform();
+
+/* These functions will be moved to api/s2n.h when the kTLS feature is released. */
 int s2n_connection_ktls_enable_send(struct s2n_connection *conn);
 int s2n_connection_ktls_enable_recv(struct s2n_connection *conn);

From 4436fed68385c0c0be8ced8dfd86ef842f79a0bd Mon Sep 17 00:00:00 2001
From: Apoorv Kothari <apoorv@toidiu.com>
Date: Tue, 11 Jul 2023 13:59:58 -0700
Subject: [PATCH 4/6] feedback3

---
 error/s2n_errno.c |  1 -
 error/s2n_errno.h |  1 -
 tls/s2n_ktls.c    | 42 ++++++++++++------------------------------
 tls/s2n_ktls.h    |  2 +-
 4 files changed, 13 insertions(+), 33 deletions(-)

diff --git a/error/s2n_errno.c b/error/s2n_errno.c
index c4ee84fe17b..a380e7b1751 100644
--- a/error/s2n_errno.c
+++ b/error/s2n_errno.c
@@ -291,7 +291,6 @@ static const char *no_such_error = "Internal s2n error";
     ERR_ENTRY(S2N_ERR_NO_RENEGOTIATION, "Only secure, server-initiated renegotiation is supported") \
     ERR_ENTRY(S2N_ERR_APP_DATA_BLOCKED, "Blocked on application data during handshake") \
     ERR_ENTRY(S2N_ERR_KTLS_MANAGED_IO, "kTLS cannot be enabled while custom I/O is configured for the connection")  \
-    ERR_ENTRY(S2N_ERR_KTLS_ALREADY_ENABLED, "kTLS is already enabled on the socket")  \
     ERR_ENTRY(S2N_ERR_HANDSHAKE_NOT_COMPLETE, "Operation is only allowed after the handshake is complete") \
     ERR_ENTRY(S2N_ERR_KTLS_UNSUPPORTED_PLATFORM, "kTLS is unsupported on this platform") \
     ERR_ENTRY(S2N_ERR_KTLS_UNSUPPORTED_CONN, "kTLS is unsupported for this connection") \
diff --git a/error/s2n_errno.h b/error/s2n_errno.h
index 4b1500e0549..4e9bca8b071 100644
--- a/error/s2n_errno.h
+++ b/error/s2n_errno.h
@@ -307,7 +307,6 @@ typedef enum {
     S2N_ERR_CERT_OWNERSHIP,
     S2N_ERR_INTERNAL_LIBCRYPTO_ERROR,
     S2N_ERR_HANDSHAKE_NOT_COMPLETE,
-    S2N_ERR_KTLS_ALREADY_ENABLED,
     S2N_ERR_KTLS_MANAGED_IO,
     S2N_ERR_KTLS_UNSUPPORTED_PLATFORM,
     S2N_ERR_KTLS_UNSUPPORTED_CONN,
diff --git a/tls/s2n_ktls.c b/tls/s2n_ktls.c
index b503af5f3b6..9542ed2e7b1 100644
--- a/tls/s2n_ktls.c
+++ b/tls/s2n_ktls.c
@@ -37,15 +37,15 @@ S2N_RESULT s2n_ktls_validate(struct s2n_connection *conn, s2n_ktls_mode ktls_mod
     const struct s2n_cipher *cipher = conn->secure->cipher_suite->record_alg->cipher;
     RESULT_ENSURE_REF(cipher);
 
+    RESULT_ENSURE(s2n_ktls_is_supported_on_platform(), S2N_ERR_KTLS_UNSUPPORTED_PLATFORM);
+
     /* kTLS enable should only be called once the handshake has completed. */
-    if (!is_handshake_complete(conn)) {
-        RESULT_BAIL(S2N_ERR_HANDSHAKE_NOT_COMPLETE);
-    }
+    RESULT_ENSURE(is_handshake_complete(conn), S2N_ERR_HANDSHAKE_NOT_COMPLETE);
 
     /* TODO support TLS 1.3
      *
      * TLS 1.3 support requires sending the KeyUpdate message when the cryptographic
-     * KeyLimits are met. However, this is currently only possible by applying a
+     * key usage limits are met. However, this is currently only possible by applying a
      * kernel patch to support this functionality.
      */
     RESULT_ENSURE(conn->actual_protocol_version == S2N_TLS12, S2N_ERR_KTLS_UNSUPPORTED_CONN);
@@ -94,14 +94,6 @@ static S2N_RESULT s2n_ktls_configure_socket(struct s2n_connection *conn, s2n_ktl
 {
     RESULT_ENSURE_REF(conn);
 
-    /* If already enabled then return success */
-    if (ktls_mode == S2N_KTLS_MODE_SEND && conn->ktls_send_enabled) {
-        RESULT_BAIL(S2N_ERR_KTLS_ALREADY_ENABLED);
-    }
-    if (ktls_mode == S2N_KTLS_MODE_RECV && conn->ktls_recv_enabled) {
-        RESULT_BAIL(S2N_ERR_KTLS_ALREADY_ENABLED);
-    }
-
     int fd = 0;
     RESULT_GUARD(s2n_ktls_retrieve_file_descriptor(conn, ktls_mode, &fd));
 
@@ -134,19 +126,14 @@ static S2N_RESULT s2n_ktls_configure_socket(struct s2n_connection *conn, s2n_ktl
 int s2n_connection_ktls_enable_send(struct s2n_connection *conn)
 {
     POSIX_ENSURE_REF(conn);
-
-    if (!s2n_ktls_is_supported_on_platform()) {
-        POSIX_BAIL(S2N_ERR_KTLS_UNSUPPORTED_PLATFORM);
-    }
-
     POSIX_GUARD_RESULT(s2n_ktls_validate(conn, S2N_KTLS_MODE_SEND));
-    s2n_result res = s2n_ktls_configure_socket(conn, S2N_KTLS_MODE_SEND);
-    if (s2n_errno == S2N_ERR_KTLS_ALREADY_ENABLED) {
+
+    /* If already enabled then return success */
+    if (conn->ktls_send_enabled) {
         return S2N_SUCCESS;
-    } else {
-        POSIX_GUARD_RESULT(res);
     }
 
+    POSIX_GUARD_RESULT(s2n_ktls_configure_socket(conn, S2N_KTLS_MODE_SEND));
     conn->ktls_send_enabled = true;
 
     return S2N_SUCCESS;
@@ -155,19 +142,14 @@ int s2n_connection_ktls_enable_send(struct s2n_connection *conn)
 int s2n_connection_ktls_enable_recv(struct s2n_connection *conn)
 {
     POSIX_ENSURE_REF(conn);
-
-    if (!s2n_ktls_is_supported_on_platform()) {
-        POSIX_BAIL(S2N_ERR_KTLS_UNSUPPORTED_PLATFORM);
-    }
-
     POSIX_GUARD_RESULT(s2n_ktls_validate(conn, S2N_KTLS_MODE_RECV));
-    s2n_result res = s2n_ktls_configure_socket(conn, S2N_KTLS_MODE_RECV);
-    if (s2n_errno == S2N_ERR_KTLS_ALREADY_ENABLED) {
+
+    /* If already enabled then return success */
+    if (conn->ktls_recv_enabled) {
         return S2N_SUCCESS;
-    } else {
-        POSIX_GUARD_RESULT(res);
     }
 
+    POSIX_GUARD_RESULT(s2n_ktls_configure_socket(conn, S2N_KTLS_MODE_RECV));
     conn->ktls_recv_enabled = true;
 
     return S2N_SUCCESS;
diff --git a/tls/s2n_ktls.h b/tls/s2n_ktls.h
index f6d9fbb431c..0d64cf83171 100644
--- a/tls/s2n_ktls.h
+++ b/tls/s2n_ktls.h
@@ -43,6 +43,6 @@ typedef enum {
 
 bool s2n_ktls_is_supported_on_platform();
 
-/* These functions will be moved to api/s2n.h when the kTLS feature is released. */
+/* These functions will be part of the public API. */
 int s2n_connection_ktls_enable_send(struct s2n_connection *conn);
 int s2n_connection_ktls_enable_recv(struct s2n_connection *conn);

From 55cadd348564b6fee022e3f6068562f2116d4855 Mon Sep 17 00:00:00 2001
From: Apoorv Kothari <apoorv@toidiu.com>
Date: Tue, 11 Jul 2023 14:30:58 -0700
Subject: [PATCH 5/6] single header file and comments

---
 tls/s2n_ktls.c                                |  7 ++++-
 tls/s2n_ktls.h                                |  8 +-----
 ...s2n_ktls_linux.h => s2n_ktls_parameters.h} | 19 +++++++++++---
 tls/s2n_ktls_unsupported.h                    | 26 -------------------
 4 files changed, 23 insertions(+), 37 deletions(-)
 rename tls/{s2n_ktls_linux.h => s2n_ktls_parameters.h} (71%)
 delete mode 100644 tls/s2n_ktls_unsupported.h

diff --git a/tls/s2n_ktls.c b/tls/s2n_ktls.c
index 9542ed2e7b1..faa9ea0c03a 100644
--- a/tls/s2n_ktls.c
+++ b/tls/s2n_ktls.c
@@ -102,7 +102,12 @@ static S2N_RESULT s2n_ktls_configure_socket(struct s2n_connection *conn, s2n_ktl
         return S2N_RESULT_OK;
     }
 
-    /* Enable 'tls' ULP for the socket. https://lwn.net/Articles/730207 */
+    /* Enable 'tls' ULP for the socket. https://lwn.net/Articles/730207
+     *
+     * Its not possible to detect kTLS support at compile time. We need rely on
+     * the call to setsockopt(..TCP_ULP...) to determine if kTLS is supported.
+     * This is a safe and non destructive operation on Linux.
+     */
     int ret = setsockopt(fd, S2N_SOL_TCP, S2N_TCP_ULP, S2N_TLS_ULP_NAME, S2N_TLS_ULP_NAME_SIZE);
 
     if (ret != 0) {
diff --git a/tls/s2n_ktls.h b/tls/s2n_ktls.h
index 0d64cf83171..36850374c58 100644
--- a/tls/s2n_ktls.h
+++ b/tls/s2n_ktls.h
@@ -23,13 +23,7 @@
  * kTLS has been tested on linux. For all other platforms, kTLS is marked as
  * unsupported, and will return an unsupported error.
  */
-#if defined(__linux__)
-    #define S2N_KTLS_SUPPORTED true
-    #include "tls/s2n_ktls_linux.h"
-#else
-    #define S2N_KTLS_SUPPORTED false
-    #include "tls/s2n_ktls_unsupported.h"
-#endif
+#include "tls/s2n_ktls_parameters.h"
 
 /* A set of kTLS configurations representing the combination of sending
  * and receiving.
diff --git a/tls/s2n_ktls_linux.h b/tls/s2n_ktls_parameters.h
similarity index 71%
rename from tls/s2n_ktls_linux.h
rename to tls/s2n_ktls_parameters.h
index cd1268db347..45c31d7bfa3 100644
--- a/tls/s2n_ktls_linux.h
+++ b/tls/s2n_ktls_parameters.h
@@ -24,8 +24,21 @@
  * - https://elixir.bootlin.com/linux/v6.3.8/A/ident/SOL_TCP
  */
 
-/* socket definitions */
 #define S2N_TLS_ULP_NAME      "tls"
 #define S2N_TLS_ULP_NAME_SIZE sizeof(S2N_TLS_ULP_NAME)
-#define S2N_TCP_ULP           31 /* Attach a ULP to a TCP connection.  */
-#define S2N_SOL_TCP           6  /* TCP level */
+
+#if defined(__linux__)
+    #define S2N_KTLS_SUPPORTED true
+
+    /* socket definitions */
+    #define S2N_TCP_ULP 31 /* Attach a ULP to a TCP connection.  */
+    #define S2N_SOL_TCP 6  /* TCP level */
+
+#else
+    /* For unsupported platforms 0-init all values. */
+    #define S2N_KTLS_SUPPORTED false
+
+    /* socket definitions */
+    #define S2N_TCP_ULP        0
+    #define S2N_SOL_TCP        0
+#endif
diff --git a/tls/s2n_ktls_unsupported.h b/tls/s2n_ktls_unsupported.h
deleted file mode 100644
index 5e30931e4e6..00000000000
--- a/tls/s2n_ktls_unsupported.h
+++ /dev/null
@@ -1,26 +0,0 @@
-/*
- * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
- *
- * Licensed under the Apache License, Version 2.0 (the "License").
- * You may not use this file except in compliance with the License.
- * A copy of the License is located at
- *
- *  http://aws.amazon.com/apache2.0
- *
- * or in the "license" file accompanying this file. This file is distributed
- * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
- * express or implied. See the License for the specific language governing
- * permissions and limitations under the License.
- */
-
-#pragma once
-
-/*
- * For unsupported platforms 0-init all values.
- */
-
-/* socket definitions */
-#define S2N_TLS_ULP_NAME      "tls"
-#define S2N_TLS_ULP_NAME_SIZE sizeof(S2N_TLS_ULP_NAME)
-#define S2N_TCP_ULP           0
-#define S2N_SOL_TCP           0

From fe8dea1b3b5fd93e326bba0fb7e7c9c795a7f20d Mon Sep 17 00:00:00 2001
From: Apoorv Kothari <apoorv@toidiu.com>
Date: Tue, 11 Jul 2023 16:06:36 -0700
Subject: [PATCH 6/6] feedback

---
 tests/unit/s2n_ktls_test.c | 121 +++++++++++++++++--------------------
 tls/s2n_ktls.c             |   8 +--
 2 files changed, 59 insertions(+), 70 deletions(-)

diff --git a/tests/unit/s2n_ktls_test.c b/tests/unit/s2n_ktls_test.c
index c869c197150..03ca8f17320 100644
--- a/tests/unit/s2n_ktls_test.c
+++ b/tests/unit/s2n_ktls_test.c
@@ -31,24 +31,20 @@ struct s2n_cipher_suite ktls_temp_supported_cipher_suite = {
     .record_alg = &ktls_temp_supported_record_alg,
 };
 
-S2N_RESULT s2n_test_configure_mock_ktls_connection(struct s2n_connection *conn, int fd, bool complete_handshake)
+S2N_RESULT s2n_test_configure_connection_for_ktls(struct s2n_connection *conn)
 {
     RESULT_ENSURE_REF(conn);
 
     /* config I/O */
-    RESULT_GUARD_POSIX(s2n_connection_set_write_fd(conn, fd));
-    RESULT_GUARD_POSIX(s2n_connection_set_read_fd(conn, fd));
-    conn->managed_send_io = true;
-    conn->managed_recv_io = true;
+    RESULT_GUARD_POSIX(s2n_connection_set_write_fd(conn, 1));
+    RESULT_GUARD_POSIX(s2n_connection_set_read_fd(conn, 1));
     conn->ktls_send_enabled = false;
     conn->ktls_recv_enabled = false;
 
     /* configure connection so that the handshake is complete */
     conn->secure->cipher_suite = &ktls_temp_supported_cipher_suite;
     conn->actual_protocol_version = S2N_TLS12;
-    if (complete_handshake) {
-        RESULT_GUARD(s2n_skip_handshake(conn));
-    }
+    RESULT_GUARD(s2n_skip_handshake(conn));
 
     return S2N_RESULT_OK;
 }
@@ -57,7 +53,18 @@ int main(int argc, char **argv)
 {
     BEGIN_TEST();
 
-    /* ktls_supported ciphers */
+    if (!s2n_ktls_is_supported_on_platform()) {
+        DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
+                s2n_connection_ptr_free);
+        EXPECT_OK(s2n_test_configure_connection_for_ktls(server_conn));
+
+        EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_UNSUPPORTED_PLATFORM);
+        EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_UNSUPPORTED_PLATFORM);
+
+        END_TEST();
+    }
+
+    /* Test ktls_supported ciphers */
     {
         struct s2n_cipher cipher = s2n_aes128_gcm;
         EXPECT_TRUE(cipher.ktls_supported);
@@ -75,32 +82,30 @@ int main(int argc, char **argv)
         EXPECT_FALSE(cipher.ktls_supported);
     };
 
-    if (!s2n_ktls_is_supported_on_platform()) {
+    /* Test s2n_ktls_retrieve_file_descriptor */
+    {
         DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
                 s2n_connection_ptr_free);
-        int fd = 1;
-        EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+        int write_fd_orig = 1;
+        int read_fd_orig = 2;
+        int fd_ret = 0;
 
-        EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_UNSUPPORTED_PLATFORM);
-        EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_UNSUPPORTED_PLATFORM);
-    } else {
-        /* ktls handshake must be complete */
-        {
-            DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
-                    s2n_connection_ptr_free);
-            int fd = 1;
-            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, false));
+        EXPECT_SUCCESS(s2n_connection_set_write_fd(server_conn, write_fd_orig));
+        EXPECT_OK(s2n_ktls_retrieve_file_descriptor(server_conn, S2N_KTLS_MODE_SEND, &fd_ret));
+        EXPECT_EQUAL(write_fd_orig, fd_ret);
 
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_HANDSHAKE_NOT_COMPLETE);
-            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_HANDSHAKE_NOT_COMPLETE);
-        };
+        EXPECT_SUCCESS(s2n_connection_set_read_fd(server_conn, read_fd_orig));
+        EXPECT_OK(s2n_ktls_retrieve_file_descriptor(server_conn, S2N_KTLS_MODE_RECV, &fd_ret));
+        EXPECT_EQUAL(read_fd_orig, fd_ret);
+    };
 
-        /* s2n_connection_ktls_enable */
+    /* Test s2n_connection_ktls_enable_recv/send */
+    {
+        /* Success case */
         {
             DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
                     s2n_connection_ptr_free);
-            int fd = 1;
-            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+            EXPECT_OK(s2n_test_configure_connection_for_ktls(server_conn));
 
             EXPECT_SUCCESS(s2n_connection_ktls_enable_send(server_conn));
             EXPECT_TRUE(server_conn->ktls_send_enabled);
@@ -108,12 +113,11 @@ int main(int argc, char **argv)
             EXPECT_TRUE(server_conn->ktls_recv_enabled);
         };
 
-        /* ktls already enabled is a noop and returns success */
+        /* Noop if kTLS is already enabled */
         {
             DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
                     s2n_connection_ptr_free);
-            int fd = 1;
-            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+            EXPECT_OK(s2n_test_configure_connection_for_ktls(server_conn));
 
             server_conn->ktls_send_enabled = true;
             EXPECT_SUCCESS(s2n_connection_ktls_enable_send(server_conn));
@@ -122,12 +126,22 @@ int main(int argc, char **argv)
             EXPECT_SUCCESS(s2n_connection_ktls_enable_recv(server_conn));
         };
 
-        /* unsupported protocols */
+        /* Fail if handshake is not complete */
+        {
+            DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
+                    s2n_connection_ptr_free);
+            EXPECT_OK(s2n_test_configure_connection_for_ktls(server_conn));
+            server_conn->handshake.message_number = 0;
+
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_HANDSHAKE_NOT_COMPLETE);
+            EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_HANDSHAKE_NOT_COMPLETE);
+        };
+
+        /* Fail if unsupported protocols */
         {
             DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
                     s2n_connection_ptr_free);
-            int fd = 1;
-            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+            EXPECT_OK(s2n_test_configure_connection_for_ktls(server_conn));
 
             server_conn->actual_protocol_version = S2N_TLS13;
             EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_UNSUPPORTED_CONN);
@@ -136,7 +150,7 @@ int main(int argc, char **argv)
             EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_UNSUPPORTED_CONN);
         };
 
-        /* unsupported ciphers */
+        /* Fail if unsupported ciphers */
         {
             /* set kTLS unsupported cipher */
             struct s2n_cipher ktls_temp_unsupported_cipher = {
@@ -151,44 +165,41 @@ int main(int argc, char **argv)
 
             DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
                     s2n_connection_ptr_free);
-            int fd = 1;
-            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+            EXPECT_OK(s2n_test_configure_connection_for_ktls(server_conn));
 
             server_conn->secure->cipher_suite = &ktls_temp_unsupported_cipher_suite;
             EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_KTLS_UNSUPPORTED_CONN);
             EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_KTLS_UNSUPPORTED_CONN);
         };
 
-        /* drain buffer prior to enabling kTLS */
+        /* Fail if buffers are not drained */
         {
             DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
                     s2n_connection_ptr_free);
-            int fd = 1;
-            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+            EXPECT_OK(s2n_test_configure_connection_for_ktls(server_conn));
 
             uint8_t write_byte = 8;
             uint8_t read_byte = 0;
             /* write to conn->out buffer and assert error */
-            EXPECT_SUCCESS(s2n_stuffer_write_bytes(&server_conn->out, &write_byte, 1));
+            EXPECT_SUCCESS(s2n_stuffer_write_uint8(&server_conn->out, write_byte));
             EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_send(server_conn), S2N_ERR_RECORD_STUFFER_NEEDS_DRAINING);
             /* drain conn->out buffer and assert success case */
             EXPECT_SUCCESS(s2n_stuffer_read_bytes(&server_conn->out, &read_byte, 1));
             EXPECT_SUCCESS(s2n_connection_ktls_enable_send(server_conn));
 
             /* write to conn->in buffer and assert error */
-            EXPECT_SUCCESS(s2n_stuffer_write_bytes(&server_conn->in, &write_byte, 1));
+            EXPECT_SUCCESS(s2n_stuffer_write_uint8(&server_conn->in, write_byte));
             EXPECT_FAILURE_WITH_ERRNO(s2n_connection_ktls_enable_recv(server_conn), S2N_ERR_RECORD_STUFFER_NEEDS_DRAINING);
             /* drain conn->in buffer and assert success case */
             EXPECT_SUCCESS(s2n_stuffer_read_bytes(&server_conn->in, &read_byte, 1));
             EXPECT_SUCCESS(s2n_connection_ktls_enable_recv(server_conn));
         };
 
-        /* managed_send_io */
+        /* Fail if not using managed IO for send */
         {
             DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
                     s2n_connection_ptr_free);
-            int fd = 1;
-            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+            EXPECT_OK(s2n_test_configure_connection_for_ktls(server_conn));
 
             /* expect failure if connection is using custom IO */
             server_conn->managed_send_io = false;
@@ -199,12 +210,11 @@ int main(int argc, char **argv)
             EXPECT_SUCCESS(s2n_connection_ktls_enable_send(server_conn));
         };
 
-        /* managed_recv_io */
+        /* Fail if not using managed IO for recv */
         {
             DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
                     s2n_connection_ptr_free);
-            int fd = 1;
-            EXPECT_OK(s2n_test_configure_mock_ktls_connection(server_conn, fd, true));
+            EXPECT_OK(s2n_test_configure_connection_for_ktls(server_conn));
 
             /* recv managed io */
             server_conn->managed_recv_io = false;
@@ -214,23 +224,6 @@ int main(int argc, char **argv)
             server_conn->managed_recv_io = true;
             EXPECT_SUCCESS(s2n_connection_ktls_enable_recv(server_conn));
         };
-
-        /* s2n_ktls_retrieve_file_descriptor */
-        {
-            DEFER_CLEANUP(struct s2n_connection *server_conn = s2n_connection_new(S2N_SERVER),
-                    s2n_connection_ptr_free);
-            int write_fd_orig = 1;
-            int read_fd_orig = 2;
-            int fd_ret = 0;
-
-            EXPECT_SUCCESS(s2n_connection_set_write_fd(server_conn, write_fd_orig));
-            EXPECT_OK(s2n_ktls_retrieve_file_descriptor(server_conn, S2N_KTLS_MODE_SEND, &fd_ret));
-            EXPECT_EQUAL(write_fd_orig, fd_ret);
-
-            EXPECT_SUCCESS(s2n_connection_set_read_fd(server_conn, read_fd_orig));
-            EXPECT_OK(s2n_ktls_retrieve_file_descriptor(server_conn, S2N_KTLS_MODE_RECV, &fd_ret));
-            EXPECT_EQUAL(read_fd_orig, fd_ret);
-        };
     }
 
     END_TEST();
diff --git a/tls/s2n_ktls.c b/tls/s2n_ktls.c
index faa9ea0c03a..db3d6ea8096 100644
--- a/tls/s2n_ktls.c
+++ b/tls/s2n_ktls.c
@@ -21,14 +21,10 @@
 
 bool s2n_ktls_is_supported_on_platform()
 {
-#if S2N_KTLS_SUPPORTED
-    return true;
-#else
-    return false;
-#endif
+    return S2N_KTLS_SUPPORTED;
 }
 
-S2N_RESULT s2n_ktls_validate(struct s2n_connection *conn, s2n_ktls_mode ktls_mode)
+static S2N_RESULT s2n_ktls_validate(struct s2n_connection *conn, s2n_ktls_mode ktls_mode)
 {
     RESULT_ENSURE_REF(conn);
     RESULT_ENSURE_REF(conn->secure);