Event Loop & Socket Type Multi-Support

.github/workflows/ci.yml

@@ -212,6 +212,10 @@ jobs:
 
   macos:
     runs-on: macos-14 # latest
+    strategy:
+      fail-fast: false
+      matrix:
+        eventloop: ["-DAWS_USE_APPLE_NETWORK_FRAMEWORK=OFF"] # TODO: Add "-DAWS_USE_APPLE_NETWORK_FRAMEWORK=ON" when apple network framework is implemented.
     steps:
     - uses: aws-actions/configure-aws-credentials@v4
       with:
@@ -221,7 +225,7 @@ jobs:
       run: |
         python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
         chmod a+x builder
-        ./builder build -p ${{ env.PACKAGE_NAME }}
+        ./builder build -p ${{ env.PACKAGE_NAME }} --cmake-extra=${{ matrix.eventloop }}
 
   macos-x64:
     runs-on: macos-14-large # latest
@@ -238,6 +242,10 @@ jobs:
 
   macos-debug:
     runs-on: macos-14 # latest
+    strategy:
+      fail-fast: false
+      matrix:
+        eventloop: ["-DAWS_USE_APPLE_NETWORK_FRAMEWORK=OFF"] # TODO: Add "-DAWS_USE_APPLE_NETWORK_FRAMEWORK=ON" when apple network framework is implemented.
     steps:
     - uses: aws-actions/configure-aws-credentials@v4
       with:
@@ -247,7 +255,7 @@ jobs:
       run: |
         python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')"
         chmod a+x builder
-        ./builder build -p ${{ env.PACKAGE_NAME }} --config Debug
+        ./builder build -p ${{ env.PACKAGE_NAME }} --cmake-extra=${{ matrix.eventloop }} --config Debug
 
   freebsd:
     runs-on: ubuntu-24.04  # latest

CMakeLists.txt

@@ -75,7 +75,7 @@ if (WIN32)
                 )
          list(APPEND AWS_IO_OS_SRC ${AWS_IO_IOCP_SRC})
 
-         set(EVENT_LOOP_DEFINE "IO_COMPLETION_PORTS")
+         list(APPEND EVENT_LOOP_DEFINES "IO_COMPLETION_PORTS")
     endif ()
 
     if (MSVC)
@@ -102,7 +102,7 @@ elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" OR CMAKE_SYSTEM_NAME STREQUAL "Androi
             )
     set(PLATFORM_LIBS "")
 
-    set(EVENT_LOOP_DEFINE "EPOLL")
+    list(APPEND EVENT_LOOP_DEFINES "EPOLL")
     set(USE_S2N ON)
 
 elseif (APPLE)
@@ -117,13 +117,18 @@ elseif (APPLE)
             )
 
     find_library(SECURITY_LIB Security)
-    if (NOT SECURITY_LIB)
-        message(FATAL_ERROR "Security framework not found")
+    find_library(NETWORK_LIB Network)
+
+    # Enable dispatch queue if the libraries are avaliable
+    if (NETWORK_LIB AND SECURITY_LIB)
+        list(APPEND PLATFORM_LIBS "-framework Security -framework Network")
+        list(APPEND EVENT_LOOP_DEFINES "DISPATCH_QUEUE")
     endif ()
 
-    #No choice on TLS for apple, darwinssl will always be used.
-    list(APPEND PLATFORM_LIBS "-framework Security")
-    set(EVENT_LOOP_DEFINE "KQUEUE")
+    # Enable KQUEUE on MacOS 
+    if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+        list(APPEND EVENT_LOOP_DEFINES "KQUEUE")
+    endif()
 
 elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD" OR CMAKE_SYSTEM_NAME STREQUAL "NetBSD" OR CMAKE_SYSTEM_NAME STREQUAL "OpenBSD")
     file(GLOB AWS_IO_OS_HEADERS
@@ -134,7 +139,7 @@ elseif (CMAKE_SYSTEM_NAME STREQUAL "FreeBSD" OR CMAKE_SYSTEM_NAME STREQUAL "NetB
             "source/posix/*.c"
             )
 
-    set(EVENT_LOOP_DEFINE "KQUEUE")
+    list(APPEND EVENT_LOOP_DEFINES "KQUEUE")
     set(USE_S2N ON)
 
 endif()
@@ -187,7 +192,12 @@ aws_add_sanitizers(${PROJECT_NAME})
 # We are not ABI stable yet
 set_target_properties(${PROJECT_NAME} PROPERTIES VERSION 1.0.0)
 
-target_compile_definitions(${PROJECT_NAME} PUBLIC "-DAWS_USE_${EVENT_LOOP_DEFINE}")
+if (NOT EVENT_LOOP_DEFINES)
+    message(FATAL_ERROR "Event Loop is not setup on the platform.")
+endif()
+foreach(EVENT_LOOP_DEFINE IN LISTS EVENT_LOOP_DEFINES)
+    target_compile_definitions(${PROJECT_NAME} PUBLIC "-DAWS_ENABLE_${EVENT_LOOP_DEFINE}")
+endforeach()
 
 if (BYO_CRYPTO)
     target_compile_definitions(${PROJECT_NAME} PUBLIC "-DBYO_CRYPTO")
@@ -205,6 +215,10 @@ if (USE_VSOCK)
 	target_compile_definitions(${PROJECT_NAME} PUBLIC "-DUSE_VSOCK")
 endif()
 
+if (AWS_USE_APPLE_NETWORK_FRAMEWORK)
+        target_compile_definitions(${PROJECT_NAME} PUBLIC "-DAWS_USE_APPLE_NETWORK_FRAMEWORK")
+endif()
+
 target_include_directories(${PROJECT_NAME} PUBLIC
         $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
         $<INSTALL_INTERFACE:include>)

include/aws/io/event_loop.h

@@ -12,6 +12,7 @@ AWS_PUSH_SANE_WARNING_LEVEL
 
 struct aws_event_loop;
 struct aws_event_loop_group;
+struct aws_event_loop_options;
 struct aws_shutdown_callback_options;
 struct aws_task;
 
@@ -47,6 +48,25 @@ struct aws_event_loop_vtable {
     bool (*is_on_callers_thread)(struct aws_event_loop *event_loop);
 };
 
+/**
+ * Event Loop Type.  If set to `AWS_EVENT_LOOP_PLATFORM_DEFAULT`, the event loop will automatically use the platform’s
+ * default.
+ *
+ * Default Event Loop Type
+ * Linux       | AWS_EVENT_LOOP_EPOLL
+ * Windows     | AWS_EVENT_LOOP_IOCP
+ * BSD Variants| AWS_EVENT_LOOP_KQUEUE
+ * MacOS       | AWS_EVENT_LOOP_KQUEUE
+ * iOS         | AWS_EVENT_LOOP_DISPATCH_QUEUE
+ */
+enum aws_event_loop_type {
+    AWS_EVENT_LOOP_PLATFORM_DEFAULT = 0,
+    AWS_EVENT_LOOP_EPOLL,
+    AWS_EVENT_LOOP_IOCP,
+    AWS_EVENT_LOOP_KQUEUE,
+    AWS_EVENT_LOOP_DISPATCH_QUEUE,
+};
+
 /**
  * Event loop group configuration options
  */
@@ -58,6 +78,12 @@ struct aws_event_loop_group_options {
      */
     uint16_t loop_count;
 
+    /**
+     * Event loop type. If the event loop type is set to AWS_EVENT_LOOP_PLATFORM_DEFAULT, the
+     * creation function will automatically use the platform’s default event loop type.
+     */
+    enum aws_event_loop_type type;
+
     /**
      * Optional callback to invoke when the event loop group finishes destruction.
      */

include/aws/io/private/event_loop_impl.h

@@ -90,8 +90,30 @@ struct aws_event_loop_local_object {
 struct aws_event_loop_options {
     aws_io_clock_fn *clock;
     struct aws_thread_options *thread_options;
+
+    /**
+     * Event loop type. If the event loop type is set to AWS_EVENT_LOOP_PLATFORM_DEFAULT, the
+     * creation function will automatically use the platform’s default event loop type.
+     */
+    enum aws_event_loop_type type;
 };
 
+struct aws_event_loop *aws_event_loop_new_with_iocp(
+    struct aws_allocator *alloc,
+    const struct aws_event_loop_options *options);
+
+struct aws_event_loop *aws_event_loop_new_with_dispatch_queue(
+    struct aws_allocator *alloc,
+    const struct aws_event_loop_options *options);
+
+struct aws_event_loop *aws_event_loop_new_with_kqueue(
+    struct aws_allocator *alloc,
+    const struct aws_event_loop_options *options);
+
+struct aws_event_loop *aws_event_loop_new_with_epoll(
+    struct aws_allocator *alloc,
+    const struct aws_event_loop_options *options);
+
 typedef struct aws_event_loop *(aws_new_event_loop_fn)(struct aws_allocator *alloc,
                                                        const struct aws_event_loop_options *options,
                                                        void *new_loop_user_data);
@@ -105,7 +127,7 @@ struct aws_event_loop_group {
 
 AWS_EXTERN_C_BEGIN
 
-#ifdef AWS_USE_IO_COMPLETION_PORTS
+#ifdef AWS_ENABLE_IO_COMPLETION_PORTS
 
 /**
  * Prepares aws_overlapped for use, and sets a function to call when the overlapped operation completes.
@@ -128,6 +150,7 @@ void aws_overlapped_reset(struct aws_overlapped *overlapped);
  */
 AWS_IO_API
 struct _OVERLAPPED *aws_overlapped_to_windows_overlapped(struct aws_overlapped *overlapped);
+#endif /* AWS_ENABLE_IO_COMPLETION_PORTS */
 
 /**
  * Associates an aws_io_handle with the event loop's I/O Completion Port.
@@ -144,8 +167,6 @@ int aws_event_loop_connect_handle_to_io_completion_port(
     struct aws_event_loop *event_loop,
     struct aws_io_handle *handle);
 
-#else
-
 /**
  * Subscribes on_event to events on the event-loop for handle. events is a bitwise concatenation of the events that were
  * received. The definition for these values can be found in aws_io_event_type. Currently, only
@@ -161,8 +182,6 @@ int aws_event_loop_subscribe_to_io_events(
     aws_event_loop_on_event_fn *on_event,
     void *user_data);
 
-#endif /* AWS_USE_IO_COMPLETION_PORTS */
-
 /**
  * Creates an instance of the default event loop implementation for the current architecture and operating system.
  */
@@ -174,9 +193,7 @@ struct aws_event_loop *aws_event_loop_new_default(struct aws_allocator *alloc, a
  * extendable options.
  */
 AWS_IO_API
-struct aws_event_loop *aws_event_loop_new_default_with_options(
-    struct aws_allocator *alloc,
-    const struct aws_event_loop_options *options);
+struct aws_event_loop *aws_event_loop_new(struct aws_allocator *alloc, const struct aws_event_loop_options *options);
 
 /**
  * Initializes common event-loop data structures.

include/aws/io/private/socket_impl.h

@@ -0,0 +1,72 @@
+#ifndef AWS_IO_SOCKET_IMPL_H
+#define AWS_IO_SOCKET_IMPL_H
+
+/**
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+#include <aws/io/io.h>
+#include <aws/io/socket.h>
+
+/* These are hacks for working around headers and functions we need for IO work but aren't directly includable or
+   linkable. these are purposely not exported. These functions only get called internally. The awkward aws_ prefixes are
+   just in case someone includes this header somewhere they were able to get these definitions included. */
+#ifdef _WIN32
+typedef void (*aws_ms_fn_ptr)(void);
+
+void aws_check_and_init_winsock(void);
+aws_ms_fn_ptr aws_winsock_get_connectex_fn(void);
+aws_ms_fn_ptr aws_winsock_get_acceptex_fn(void);
+#endif
+
+int aws_socket_init_posix(
+    struct aws_socket *socket,
+    struct aws_allocator *alloc,
+    const struct aws_socket_options *options);
+
+int aws_socket_init_winsock(
+    struct aws_socket *socket,
+    struct aws_allocator *alloc,
+    const struct aws_socket_options *options);
+
+int aws_socket_init_apple_nw_socket(
+    struct aws_socket *socket,
+    struct aws_allocator *alloc,
+    const struct aws_socket_options *options);
+
+struct aws_socket_vtable {
+    void (*socket_cleanup_fn)(struct aws_socket *socket);
+    int (*socket_connect_fn)(
+        struct aws_socket *socket,
+        const struct aws_socket_endpoint *remote_endpoint,
+        struct aws_event_loop *event_loop,
+        aws_socket_on_connection_result_fn *on_connection_result,
+        void *user_data);
+    int (*socket_bind_fn)(struct aws_socket *socket, const struct aws_socket_endpoint *local_endpoint);
+    int (*socket_listen_fn)(struct aws_socket *socket, int backlog_size);
+    int (*socket_start_accept_fn)(
+        struct aws_socket *socket,
+        struct aws_event_loop *accept_loop,
+        aws_socket_on_accept_result_fn *on_accept_result,
+        void *user_data);
+    int (*socket_stop_accept_fn)(struct aws_socket *socket);
+    int (*socket_close_fn)(struct aws_socket *socket);
+    int (*socket_shutdown_dir_fn)(struct aws_socket *socket, enum aws_channel_direction dir);
+    int (*socket_set_options_fn)(struct aws_socket *socket, const struct aws_socket_options *options);
+    int (*socket_assign_to_event_loop_fn)(struct aws_socket *socket, struct aws_event_loop *event_loop);
+    int (*socket_subscribe_to_readable_events_fn)(
+        struct aws_socket *socket,
+        aws_socket_on_readable_fn *on_readable,
+        void *user_data);
+    int (*socket_read_fn)(struct aws_socket *socket, struct aws_byte_buf *buffer, size_t *amount_read);
+    int (*socket_write_fn)(
+        struct aws_socket *socket,
+        const struct aws_byte_cursor *cursor,
+        aws_socket_on_write_completed_fn *written_fn,
+        void *user_data);
+    int (*socket_get_error_fn)(struct aws_socket *socket);
+    bool (*socket_is_open_fn)(struct aws_socket *socket);
+};
+
+#endif // AWS_IO_SOCKET_IMPL_H