Apple Network Framework Socket Changes

README.md

@@ -151,7 +151,8 @@ Core to Async-IO is the event-loop. We provide an implementation for most platfo
 Platform | Implementation
 --- | ---
 Linux | Edge-Triggered Epoll
-BSD Variants and Apple Devices | KQueue
+BSD Variants | KQueue
+Apple Devices | KQueue or Apple Dispatch Queue
 Windows | IOCP (IO Completion Ports)
 
 Also, you can always implement your own as well.
@@ -645,12 +646,49 @@ All exported functions, simply shim into the v-table and return.
 
 We include a cross-platform API for sockets. We support TCP and UDP using IPv4 and IPv6, and Unix Domain sockets. On Windows,
 we use Named Pipes to support the functionality of Unix Domain sockets. On Windows, this is implemented with winsock2, and on
-all unix platforms we use the posix API.
+all unix platforms we use the posix API. Then we will use Apple Network Framework on Apple platforms. 
 
 Upon a connection being established, the new socket (either as the result of a `connect()` or `start_accept()` call)
 will not be attached to any event loops. It is your responsibility to register it with an event loop to begin receiving
 notifications.
 
+
+#### V-Table
+
+    struct aws_socket_vtable {
+        void (*socket_cleanup_fn)(struct aws_socket *socket);
+        int (*socket_connect_fn)(
+            struct aws_socket *socket,
+            const struct aws_socket_endpoint *remote_endpoint,
+            struct aws_event_loop *event_loop,
+            aws_socket_on_connection_result_fn *on_connection_result,
+            void *user_data);
+        int (*socket_bind_fn)(struct aws_socket *socket, const struct aws_socket_endpoint *local_endpoint);
+        int (*socket_listen_fn)(struct aws_socket *socket, int backlog_size);
+        int (*socket_start_accept_fn)(
+            struct aws_socket *socket,
+            struct aws_event_loop *accept_loop,
+            aws_socket_on_accept_result_fn *on_accept_result,
+            void *user_data);
+        int (*socket_stop_accept_fn)(struct aws_socket *socket);
+        int (*socket_close_fn)(struct aws_socket *socket);
+        int (*socket_shutdown_dir_fn)(struct aws_socket *socket, enum aws_channel_direction dir);
+        int (*socket_set_options_fn)(struct aws_socket *socket, const struct aws_socket_options *options);
+        int (*socket_assign_to_event_loop_fn)(struct aws_socket *socket, struct aws_event_loop *event_loop);
+        int (*socket_subscribe_to_readable_events_fn)(
+            struct aws_socket *socket,
+            aws_socket_on_readable_fn *on_readable,
+            void *user_data);
+        int (*socket_read_fn)(struct aws_socket *socket, struct aws_byte_buf *buffer, size_t *amount_read);
+        int (*socket_write_fn)(
+            struct aws_socket *socket,
+            const struct aws_byte_cursor *cursor,
+            aws_socket_on_write_completed_fn *written_fn,
+            void *user_data);
+        int (*socket_get_error_fn)(struct aws_socket *socket);
+        bool (*socket_is_open_fn)(struct aws_socket *socket);
+    };
+
 #### API
     typedef enum aws_socket_domain {
         AWS_SOCKET_IPV4,
@@ -717,32 +755,33 @@ Shuts down any pending operations on the socket, and cleans up state. The socket
 
     int aws_socket_connect(struct aws_socket *socket, struct aws_socket_endpoint *remote_endpoint);
 
-Connects to a remote endpoint. In UDP, this simply binds the socket to a remote address for use with `aws_socket_write()`,
-and if the operation is successful, the socket can immediately be used for write operations.
+Connects to a remote endpoint. In TCP and all Apple Network Framework connections (regardless it is UDP, TCP or LOCAL), when the connection succeed, you still must wait on  the `on_connection_established()` callback to 
+be invoked before using the socket.
 
-In TCP, this will function will not block. If the return value is successful, then you must wait on the `on_connection_established()`
-callback to be invoked before using the socket.
+In UDP, this simply binds the socket to a remote address for use with `aws_socket_write()`, and if the operation is successful, 
+the socket can immediately be used for write operations.
 
 For LOCAL (Unix Domain Sockets or Named Pipes), the socket will be immediately ready for use upon a successful return.
 
     int aws_socket_bind(struct aws_socket *socket, struct aws_socket_endpoint *local_endpoint);
 
-Binds the socket to a local address. In UDP mode, the socket is ready for `aws_socket_read()` operations. In connection oriented
-modes, you still must call `aws_socket_listen()` and `aws_socket_start_accept()` before using the socket.
+Binds the socket to a local address. In UDP mode, the socket is ready for `aws_socket_read()` operations. In connection oriented 
+modes or if you are using Apple Network Framework (regardless it is UDP or TCP), you still must call `aws_socket_listen()` and 
+`aws_socket_start_accept()` before using the socket.
 
     int aws_socket_listen(struct aws_socket *socket, int backlog_size);
 
-TCP and LOCAL only. Sets up the socket to listen on the address bound to in `aws_socket_bind()`.
+TCP, LOCAL, and Apple Network Framework only. Sets up the socket to listen on the address bound to in `aws_socket_bind()`.
 
     int aws_socket_start_accept(struct aws_socket *socket);
 
-TCP and LOCAL only. The socket will begin accepting new connections. This is an asynchronous operation. New connections will
-arrive via the `on_incoming_connection()` callback.
+TCP, LOCAL, and Apple Network Framework only. The socket will begin accepting new connections. This is an asynchronous operation. New 
+connections will arrive via the `on_incoming_connection()` callback.
 
     int aws_socket_stop_accept(struct aws_socket *socket);
 
-TCP and LOCAL only. The socket will shutdown the listener. It is safe to call `aws_socket_start_accept()` again after this
-operation.
+TCP, LOCAL, and Apple Network Framework only. The socket will shutdown the listener. It is safe to call `aws_socket_start_accept()` 
+again after this operation.
 
     int aws_socket_close(struct aws_socket *socket);
 

source/channel.c

@@ -646,7 +646,12 @@ void aws_channel_schedule_task_future(
 }
 
 bool aws_channel_thread_is_callers_thread(struct aws_channel *channel) {
-    return aws_event_loop_thread_is_callers_thread(channel->loop);
+    // As aws_event_loop is not ref-counted, it is possible that the socket read/write callback get triggered after
+    // the event loop is released, ignore the function call here.
+    if (channel && channel->loop) {
+        return aws_event_loop_thread_is_callers_thread(channel->loop);
+    }
+    return true;
 }
 
 static void s_update_channel_slot_message_overheads(struct aws_channel *channel) {

source/channel_bootstrap.c

@@ -523,6 +523,10 @@ static bool s_aws_socket_domain_uses_dns(enum aws_socket_domain domain) {
     return domain == AWS_SOCKET_IPV4 || domain == AWS_SOCKET_IPV6;
 }
 
+/* Called when a socket connection attempt task completes. First socket to successfully open
+ * assigns itself to connection_args->channel_data.socket and flips connection_args->connection_chosen
+ * to true. Subsequent successful sockets will be released and cleaned up
+ */
 static void s_on_client_connection_established(struct aws_socket *socket, int error_code, void *user_data) {
     struct client_connection_args *connection_args = user_data;
 
@@ -565,7 +569,6 @@ static void s_on_client_connection_established(struct aws_socket *socket, int er
             (void *)connection_args->bootstrap,
             (void *)socket);
         aws_socket_close(socket);
-
         aws_socket_clean_up(socket);
         aws_mem_release(connection_args->bootstrap->allocator, socket);
 
@@ -639,7 +642,7 @@ static void s_attempt_connection(struct aws_task *task, void *arg, enum aws_task
         goto task_cancelled;
     }
 
-    struct aws_socket *outgoing_socket = aws_mem_acquire(allocator, sizeof(struct aws_socket));
+    struct aws_socket *outgoing_socket = aws_mem_calloc(allocator, 1, sizeof(struct aws_socket));
     if (aws_socket_init(outgoing_socket, allocator, &task_data->options)) {
         goto socket_init_failed;
     }
@@ -832,10 +835,6 @@ int aws_client_bootstrap_new_socket_channel(struct aws_socket_channel_bootstrap_
     struct client_connection_args *client_connection_args =
         aws_mem_calloc(bootstrap->allocator, 1, sizeof(struct client_connection_args));
 
-    if (!client_connection_args) {
-        return AWS_OP_ERR;
-    }
-
     const char *host_name = options->host_name;
     uint32_t port = options->port;
 
@@ -1361,9 +1360,7 @@ void s_on_server_connection_result(
             (void *)socket);
         struct server_channel_data *channel_data =
             aws_mem_calloc(connection_args->bootstrap->allocator, 1, sizeof(struct server_channel_data));
-        if (!channel_data) {
-            goto error_cleanup;
-        }
+
         channel_data->incoming_called = false;
         channel_data->socket = new_socket;
         channel_data->server_connection_args = connection_args;
@@ -1376,11 +1373,10 @@ void s_on_server_connection_result(
             .setup_user_data = channel_data,
             .shutdown_user_data = channel_data,
             .on_shutdown_completed = s_on_server_channel_on_shutdown,
+            .event_loop = event_loop,
+            .enable_read_back_pressure = channel_data->server_connection_args->enable_read_back_pressure,
         };
 
-        channel_args.event_loop = event_loop;
-        channel_args.enable_read_back_pressure = channel_data->server_connection_args->enable_read_back_pressure;
-
         if (aws_socket_assign_to_event_loop(new_socket, event_loop)) {
             aws_mem_release(connection_args->bootstrap->allocator, (void *)channel_data);
             goto error_cleanup;