TLS deliver buffer data during shutdown

include/aws/io/private/tls_channel_handler_shared.h

@@ -19,6 +19,12 @@ struct aws_tls_channel_handler_shared {
     struct aws_crt_statistics_tls stats;
 };
 
+enum aws_tls_handler_read_state {
+    AWS_TLS_HANDLER_OPEN,
+    AWS_TLS_HANDLER_READ_SHUTTING_DOWN,
+    AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE,
+};
+
 AWS_EXTERN_C_BEGIN
 
 AWS_IO_API void aws_tls_channel_handler_shared_init(

source/darwin/secure_transport_tls_channel_handler.c

@@ -111,9 +111,8 @@ struct secure_transport_handler {
     bool negotiation_finished;
     bool verify_peer;
     bool read_task_pending;
-    bool delay_shutdown_scheduled;
+    enum aws_tls_handler_read_state read_state;
     int delay_shutdown_error_code;
-    bool read_shutdown_completed;
 };
 
 static OSStatus s_read_cb(SSLConnectionRef conn, void *data, size_t *len) {
@@ -574,7 +573,7 @@ static void s_initialize_read_delay_shutdown(
             " Your application may hang if the read window never opens",
             (void *)handler);
     }
-    secure_transport_handler->delay_shutdown_scheduled = true;
+    secure_transport_handler->read_state = AWS_TLS_HANDLER_READ_SHUTTING_DOWN;
     secure_transport_handler->delay_shutdown_error_code = error_code;
     if (!secure_transport_handler->read_task_pending) {
         /* Kick off read, in case data arrives with TLS negotiation. Shutdown starts right after negotiation.
@@ -604,7 +603,7 @@ static int s_handle_shutdown(
              * data. */
             return AWS_OP_SUCCESS;
         }
-        secure_transport_handler->read_shutdown_completed = true;
+        secure_transport_handler->read_state = AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE;
     } else {
         /* Shutdown in write direction */
         if (!abort_immediately && error_code != AWS_IO_SOCKET_CLOSED) {
@@ -627,7 +626,10 @@ static int s_process_read_message(
     struct aws_io_message *message) {
 
     struct secure_transport_handler *secure_transport_handler = handler->impl;
-    if (secure_transport_handler->read_shutdown_completed) {
+    if (secure_transport_handler->read_state == AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE) {
+        if (message) {
+            aws_mem_release(message->allocator, message);
+        }
         return AWS_OP_SUCCESS;
     }
 
@@ -695,7 +697,7 @@ static int s_process_read_message(
 
         switch (status) {
             case errSSLWouldBlock:
-                if (secure_transport_handler->delay_shutdown_scheduled) {
+                if (secure_transport_handler->read_state == AWS_TLS_HANDLER_READ_SHUTTING_DOWN) {
                     /* Propagate the shutdown as we blocked now. */
                     goto shutdown_channel;
                 } else {
@@ -726,13 +728,13 @@ static int s_process_read_message(
     return AWS_OP_SUCCESS;
 
 shutdown_channel:
-    if (secure_transport_handler->delay_shutdown_scheduled) {
+    if (secure_transport_handler->read_state == AWS_TLS_HANDLER_READ_SHUTTING_DOWN) {
         if (secure_transport_handler->delay_shutdown_error_code != 0) {
             /* Propagate the original error code if it is set. */
             shutdown_error_code = secure_transport_handler->delay_shutdown_error_code;
         }
         /* Continue the shutdown process delayed before. */
-        secure_transport_handler->read_shutdown_completed = true;
+        secure_transport_handler->read_state = AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE;
         aws_channel_slot_on_handler_shutdown_complete(slot, AWS_CHANNEL_DIR_READ, shutdown_error_code, false);
     } else {
         /* Starts the shutdown process */
@@ -753,7 +755,7 @@ static void s_run_read(struct aws_channel_task *task, void *arg, aws_task_status
 
 static int s_increment_read_window(struct aws_channel_handler *handler, struct aws_channel_slot *slot, size_t size) {
     struct secure_transport_handler *secure_transport_handler = handler->impl;
-    if (secure_transport_handler->read_shutdown_completed) {
+    if (secure_transport_handler->read_state == AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE) {
         return AWS_OP_SUCCESS;
     }
 

source/s2n/s2n_tls_channel_handler.c

@@ -59,11 +59,9 @@ struct s2n_handler {
     } state;
     struct aws_channel_task read_task;
     bool read_task_pending;
-
-    bool is_shutting_down;
+    enum aws_tls_handler_read_state read_state;
     int shutdown_error_code;
     struct aws_channel_task delayed_shutdown_task;
-    bool read_shutdown_completed;
 };
 
 struct s2n_ctx {
@@ -523,7 +521,10 @@ static int s_s2n_handler_process_read_message(
 
     struct s2n_handler *s2n_handler = handler->impl;
 
-    if (s2n_handler->read_shutdown_completed) {
+    if (s2n_handler->read_state == AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE) {
+        if (message) {
+            aws_mem_release(message->allocator, message);
+        }
         return AWS_OP_SUCCESS;
     }
 
@@ -590,7 +591,7 @@ static int s_s2n_handler_process_read_message(
 
             /* the socket blocked so exit from the loop */
             if (s2n_error_get_type(s2n_errno) == S2N_ERR_T_BLOCKED) {
-                if (s2n_handler->is_shutting_down) {
+                if (s2n_handler->read_state == AWS_TLS_HANDLER_READ_SHUTTING_DOWN) {
                     /* Propagate the shutdown as we blocked now. */
                     goto shutdown_channel;
                 }
@@ -632,12 +633,12 @@ static int s_s2n_handler_process_read_message(
     return AWS_OP_SUCCESS;
 
 shutdown_channel:
-    if (s2n_handler->is_shutting_down) {
+    if (s2n_handler->read_state == AWS_TLS_HANDLER_READ_SHUTTING_DOWN) {
         if (s2n_handler->shutdown_error_code != 0) {
             /* Propagate the original error code if it is set. */
             shutdown_error_code = s2n_handler->shutdown_error_code;
         }
-        s2n_handler->read_shutdown_completed = true;
+        s2n_handler->read_state = AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE;
         aws_channel_slot_on_handler_shutdown_complete(slot, AWS_CHANNEL_DIR_READ, shutdown_error_code, false);
     } else {
         /* Starts the shutdown process */
@@ -1046,7 +1047,7 @@ static void s_initialize_read_delay_shutdown(
             " Your application may hang if the read window never opens",
             (void *)handler);
     }
-    s2n_handler->is_shutting_down = true;
+    s2n_handler->read_state = AWS_TLS_HANDLER_READ_SHUTTING_DOWN;
     s2n_handler->shutdown_error_code = error_code;
     if (!s2n_handler->read_task_pending) {
         /* Kick off read, in case data arrives with TLS negotiation. Shutdown starts right after negotiation.
@@ -1080,7 +1081,7 @@ static int s_s2n_handler_shutdown(
             s_initialize_read_delay_shutdown(handler, slot, error_code);
             return AWS_OP_SUCCESS;
         }
-        s2n_handler->read_shutdown_completed = true;
+        s2n_handler->read_state = AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE;
     } else {
         /* Shutdown in write direction */
         if (!abort_immediately && error_code != AWS_IO_SOCKET_CLOSED) {
@@ -1105,7 +1106,7 @@ static int s_s2n_handler_increment_read_window(
     size_t size) {
     (void)size;
     struct s2n_handler *s2n_handler = handler->impl;
-    if (s2n_handler->read_shutdown_completed) {
+    if (s2n_handler->read_state == AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE) {
         return AWS_OP_SUCCESS;
     }
 

source/windows/secure_channel_tls_handler.c

@@ -105,9 +105,8 @@ struct secure_channel_handler {
     bool verify_peer;
     struct aws_channel_task read_task;
     bool read_task_pending;
-    bool delay_shutdown_scheduled;
-    int delay_shutdown_error_code;
-    bool read_shutdown_completed;
+    enum aws_tls_handler_state read_state;
+    int shutdown_error_code;
 };
 
 static size_t s_message_overhead(struct aws_channel_handler *handler) {
@@ -1164,7 +1163,7 @@ static int s_do_application_data_decrypt(struct aws_channel_handler *handler) {
 
 static int s_process_pending_output_messages(struct aws_channel_handler *handler) {
     struct secure_channel_handler *sc_handler = handler->impl;
-    if (sc_handler->read_shutdown_completed) {
+    if (sc_handler->read_state == AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE) {
         return AWS_OP_SUCCESS;
     }
 
@@ -1173,6 +1172,7 @@ static int s_process_pending_output_messages(struct aws_channel_handler *handler
     if (sc_handler->slot->adj_right) {
         downstream_window = aws_channel_slot_downstream_read_window(sc_handler->slot);
     }
+    int ret = AWS_OP_ERR;
 
     AWS_LOGF_TRACE(
         AWS_LS_IO_TLS,
@@ -1207,7 +1207,7 @@ static int s_process_pending_output_messages(struct aws_channel_handler *handler
             }
             if (aws_channel_slot_send_message(sc_handler->slot, read_out_msg, AWS_CHANNEL_DIR_READ)) {
                 aws_mem_release(read_out_msg->allocator, read_out_msg);
-                return AWS_OP_ERR;
+                goto done;
             }
 
             if (sc_handler->slot->adj_right) {
@@ -1222,14 +1222,21 @@ static int s_process_pending_output_messages(struct aws_channel_handler *handler
             sc_handler->buffered_read_out_data_buf.len = 0;
         }
     }
-    if (sc_handler->buffered_read_out_data_buf.len == 0 && sc_handler->delay_shutdown_scheduled) {
-        sc_handler->read_shutdown_completed = true;
+    ret = AWS_OP_SUCCESS;
+    if (sc_handler->buffered_read_out_data_buf.len > 0) {
+        /* Still have more data to be delivered */
+        return ret;
+    }
+
+done:
+    if (sc_handler->read_state == AWS_TLS_HANDLER_READ_SHUTTING_DOWN) {
+        sc_handler->read_state = AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE;
         /* Continue the shutdown process delayed before. */
         aws_channel_slot_on_handler_shutdown_complete(
-            sc_handler->slot, AWS_CHANNEL_DIR_READ, sc_handler->delay_shutdown_error_code, false);
+            sc_handler->slot, AWS_CHANNEL_DIR_READ, sc_handler->shutdown_error_code, false);
     }
 
-    return AWS_OP_SUCCESS;
+    return ret;
 }
 
 static void s_process_pending_output_task(struct aws_channel_task *task, void *arg, enum aws_task_status status) {
@@ -1252,7 +1259,10 @@ static int s_process_read_message(
     struct aws_io_message *message) {
 
     struct secure_channel_handler *sc_handler = handler->impl;
-    if (sc_handler->read_shutdown_completed) {
+    if (sc_handler->read_state == AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE) {
+        if (message) {
+            aws_mem_release(message->allocator, message);
+        }
         return AWS_OP_SUCCESS;
     }
 
@@ -1480,7 +1490,7 @@ static int s_process_write_message(
 static int s_increment_read_window(struct aws_channel_handler *handler, struct aws_channel_slot *slot, size_t size) {
     (void)size;
     struct secure_channel_handler *sc_handler = handler->impl;
-    if (sc_handler->read_shutdown_completed) {
+    if (sc_handler->read_state == AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE) {
         return AWS_OP_SUCCESS;
     }
     AWS_LOGF_TRACE(AWS_LS_IO_TLS, "id=%p: Increment read window message received %zu", (void *)handler, size);
@@ -1564,8 +1574,8 @@ static void s_initialize_read_delay_shutdown(
             " Your application may hang if the read window never opens",
             (void *)handler);
     }
-    sc_handler->delay_shutdown_scheduled = true;
-    sc_handler->delay_shutdown_error_code = error_code;
+    sc_handler->read_state = AWS_TLS_HANDLER_READ_SHUTTING_DOWN;
+    sc_handler->shutdown_error_code = error_code;
     if (!sc_handler->read_task_pending) {
         /* Kick off read, in case data arrives with TLS negotiation. Shutdown starts right after negotiation.
          * Nothing will kick off read in that case. */
@@ -1608,7 +1618,7 @@ static int s_handler_shutdown(
             s_initialize_read_delay_shutdown(handler, slot, error_code);
             return AWS_OP_SUCCESS;
         }
-        sc_handler->read_shutdown_completed = true;
+        sc_handler->read_state = AWS_TLS_HANDLER_READ_SHUT_DOWN_COMPLETE;
     } else {
         /* Shutdown in write direction */
         if (!abort_immediately && error_code != AWS_IO_SOCKET_CLOSED) {