From 70de9dfbc7fef9d5308df39b9b3e24c7c2540afa Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 22 Aug 2024 15:17:30 -0700 Subject: [PATCH 01/46] python 313 CI --- .github/workflows/ci.yml | 4 +++- crt/aws-c-compression | 2 +- crt/aws-lc | 2 +- crt/s2n | 2 +- 4 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 6786da5f4..be20926e3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: - 'docs' env: - BUILDER_VERSION: v0.9.62 + BUILDER_VERSION: v0.9.63 BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python @@ -53,6 +53,7 @@ jobs: - cp310-cp310 - cp311-cp311 - cp312-cp312 + - cp313-cp313 steps: # Only aarch64 needs this, but it doesn't hurt anything - name: Install qemu/docker @@ -78,6 +79,7 @@ jobs: - cp310-cp310 - cp311-cp311 - cp312-cp312 + - cp313-cp313 steps: # Only aarch64 needs this, but it doesn't hurt anything - name: Install qemu/docker diff --git a/crt/aws-c-compression b/crt/aws-c-compression index ea1d421a4..f36d01672 160000 --- a/crt/aws-c-compression +++ b/crt/aws-c-compression @@ -1 +1 @@ -Subproject commit ea1d421a421ad83a540309a94c38d50b6a5d836b +Subproject commit f36d01672d61e49d96a777870d456f66fa391cd4 diff --git a/crt/aws-lc b/crt/aws-lc index 057477806..2f1879759 160000 --- a/crt/aws-lc +++ b/crt/aws-lc @@ -1 +1 @@ -Subproject commit 05747780676652f41d0b9c570a495e4bb6608560 +Subproject commit 2f1879759b2e0fc70592665bdf10087b64f44b7d diff --git a/crt/s2n b/crt/s2n index 79c0f1b43..87f4a0585 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit 79c0f1b434742d9f1152c48d3781433649f6f8fe +Subproject commit 87f4a0585dc3056433f193b9305f587cff239be3 From d2b0b271d74eddb885b36d9358d59133bc848cea Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 26 Aug 2024 14:13:10 -0700 Subject: [PATCH 02/46] wip test --- source/http_stream.c | 2 +- source/mqtt_client_connection.c | 27 ++++++++++++++++++++------- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/source/http_stream.c b/source/http_stream.c index bf7021862..09943d89d 100644 --- a/source/http_stream.c +++ b/source/http_stream.c @@ -203,7 +203,7 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro } /* DECREF python self, we don't need to force it to stay alive any longer. */ - Py_DECREF(PyWeakref_GetObject(stream->self_proxy)); + Py_XDECREF(stream->self_proxy); PyGILState_Release(state); /*************** GIL RELEASE ***************/ diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index 9eb73a950..acaae556c 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -140,7 +140,17 @@ static void s_on_connection_success( return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ + PyErr_WriteUnraisable(PyErr_Occurred()); + goto on_done; + } +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ +#endif + if (self != Py_None) { PyObject *success_result = PyObject_CallMethod(self, "_on_connection_success", "(iN)", return_code, PyBool_FromLong(session_present)); @@ -150,7 +160,10 @@ static void s_on_connection_success( PyErr_WriteUnraisable(PyErr_Occurred()); } } - +#if PY_VERSION_HEX >= 0x030D0000 + Py_DECREF(self); +#endif +on_done: PyGILState_Release(state); } @@ -167,7 +180,7 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ if (self != Py_None) { PyObject *success_result = PyObject_CallMethod(self, "_on_connection_failure", "(i)", error_code); if (success_result) { @@ -194,7 +207,7 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } /* Ensure that python class is still alive */ - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_interrupted", "(i)", error_code); if (result) { @@ -227,7 +240,7 @@ static void s_on_connection_resumed( } /* Ensure that python class is still alive */ - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_resumed", "(iN)", return_code, PyBool_FromLong(session_present)); @@ -258,7 +271,7 @@ static void s_on_connection_closed( struct mqtt_connection_binding *py_connection = userdata; /* Ensure that python class is still alive */ - PyObject *self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_closed", "()"); if (result) { @@ -535,7 +548,7 @@ static void s_ws_handshake_transform( } /* Ensure python mqtt connection object is still alive */ - PyObject *connection_py = PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ + PyObject *connection_py = Py_None; // PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ if (connection_py == Py_None) { aws_raise_error(AWS_ERROR_INVALID_STATE); goto done; From f4f6ea242929b799d90ebf63cb4192139d47d090 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 26 Aug 2024 15:45:56 -0700 Subject: [PATCH 03/46] fix warnings --- setup.py | 5 +- source/mqtt_client_connection.c | 86 ++++++++++++++++++++++++++++++--- 2 files changed, 82 insertions(+), 9 deletions(-) diff --git a/setup.py b/setup.py index 32873b2b0..9a7b674f5 100644 --- a/setup.py +++ b/setup.py @@ -396,7 +396,10 @@ def awscrt_ext(): else: extra_link_args += ['-Wl,--fatal-warnings'] - if sys.version_info >= (3, 11): + if sys.version_info >= (3,13): + define_macros.append(('Py_LIMITED_API', '0x030D0000')) + py_limited_api = True + elif sys.version_info >= (3,11): define_macros.append(('Py_LIMITED_API', '0x030B0000')) py_limited_api = True diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index acaae556c..df1eaed6d 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -160,10 +160,10 @@ static void s_on_connection_success( PyErr_WriteUnraisable(PyErr_Occurred()); } } +on_done: #if PY_VERSION_HEX >= 0x030D0000 - Py_DECREF(self); + Py_XDECREF(self); #endif -on_done: PyGILState_Release(state); } @@ -180,7 +180,17 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ + PyErr_WriteUnraisable(PyErr_Occurred()); + goto on_done; + } +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ +#endif + if (self != Py_None) { PyObject *success_result = PyObject_CallMethod(self, "_on_connection_failure", "(i)", error_code); if (success_result) { @@ -190,6 +200,10 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio } } +on_done: +#if PY_VERSION_HEX >= 0x030D0000 + Py_XDECREF(self); +#endif PyGILState_Release(state); } @@ -207,7 +221,17 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } /* Ensure that python class is still alive */ - PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ + PyErr_WriteUnraisable(PyErr_Occurred()); + goto on_done; + } +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ +#endif + if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_interrupted", "(i)", error_code); if (result) { @@ -217,6 +241,10 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } } +on_done: +#if PY_VERSION_HEX >= 0x030D0000 + Py_XDECREF(self); +#endif PyGILState_Release(state); } @@ -240,7 +268,17 @@ static void s_on_connection_resumed( } /* Ensure that python class is still alive */ - PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ + PyErr_WriteUnraisable(PyErr_Occurred()); + goto on_done; + } +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ +#endif + if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_resumed", "(iN)", return_code, PyBool_FromLong(session_present)); @@ -250,7 +288,10 @@ static void s_on_connection_resumed( PyErr_WriteUnraisable(PyErr_Occurred()); } } - +on_done: +#if PY_VERSION_HEX >= 0x030D0000 + Py_XDECREF(self); +#endif PyGILState_Release(state); } @@ -271,7 +312,17 @@ static void s_on_connection_closed( struct mqtt_connection_binding *py_connection = userdata; /* Ensure that python class is still alive */ - PyObject *self = Py_None; // PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ + PyErr_WriteUnraisable(PyErr_Occurred()); + goto on_done; + } +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ +#endif + if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_closed", "()"); if (result) { @@ -281,6 +332,11 @@ static void s_on_connection_closed( } } + +on_done: +#if PY_VERSION_HEX >= 0x030D0000 + Py_XDECREF(self); +#endif PyGILState_Release(state); } @@ -548,7 +604,17 @@ static void s_ws_handshake_transform( } /* Ensure python mqtt connection object is still alive */ - PyObject *connection_py = Py_None; // PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ + PyObject *connection_py = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + if (PyWeakref_GetRef(connection_binding->self_proxy, &connection_py) < 0) { /* strong reference */ + aws_raise_error(AWS_ERROR_INVALID_STATE); + goto done; + } +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ +#endif + if (connection_py == Py_None) { aws_raise_error(AWS_ERROR_INVALID_STATE); goto done; @@ -606,6 +672,10 @@ static void s_ws_handshake_transform( done:; /* Save off error code, so it doesn't got stomped before we pass it to callback*/ int error_code = aws_last_error(); +#if PY_VERSION_HEX >= 0x030D0000 + Py_XDECREF(connection_py); +#endif + if (ws_transform_capsule) { Py_DECREF(ws_transform_capsule); From d19acc871f813490faadc61e1021afe8364c78d5 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 09:19:06 -0700 Subject: [PATCH 04/46] new changes --- crt/aws-c-auth | 2 +- crt/aws-c-cal | 2 +- crt/aws-c-common | 2 +- crt/aws-c-http | 2 +- crt/aws-c-mqtt | 2 +- crt/aws-c-s3 | 2 +- crt/aws-checksums | 2 +- crt/aws-lc | 2 +- crt/s2n | 2 +- source/http_stream.c | 13 ++++++++++++- source/mqtt_client_connection.c | 22 ++++++++++------------ 11 files changed, 31 insertions(+), 22 deletions(-) diff --git a/crt/aws-c-auth b/crt/aws-c-auth index 52bf59161..48d647bf4 160000 --- a/crt/aws-c-auth +++ b/crt/aws-c-auth @@ -1 +1 @@ -Subproject commit 52bf591613d1a001c43ec99af7376f871759c5fe +Subproject commit 48d647bf43f8872e4dc5ec6343b0c5974195fbdd diff --git a/crt/aws-c-cal b/crt/aws-c-cal index 77ca3aea8..2cb1d2eac 160000 --- a/crt/aws-c-cal +++ b/crt/aws-c-cal @@ -1 +1 @@ -Subproject commit 77ca3aea879bc768082fe7ec715adcde8e98c332 +Subproject commit 2cb1d2eac925e2dbc45025eb89af82bd790c23a0 diff --git a/crt/aws-c-common b/crt/aws-c-common index 672cc0032..b9959f592 160000 --- a/crt/aws-c-common +++ b/crt/aws-c-common @@ -1 +1 @@ -Subproject commit 672cc0032eb28d69fbdd22c9463253c89d7a6f30 +Subproject commit b9959f5922a4b969beab8f0b99aa0b34bc9ee55c diff --git a/crt/aws-c-http b/crt/aws-c-http index 4e74ab1e3..6068653e1 160000 --- a/crt/aws-c-http +++ b/crt/aws-c-http @@ -1 +1 @@ -Subproject commit 4e74ab1e3702763e0b87bd1752f5a37c2f0400ac +Subproject commit 6068653e1d582bd8e7d1c9f81f86beaf10444e3d diff --git a/crt/aws-c-mqtt b/crt/aws-c-mqtt index ed7bbd68c..c43232c1b 160000 --- a/crt/aws-c-mqtt +++ b/crt/aws-c-mqtt @@ -1 +1 @@ -Subproject commit ed7bbd68c03d7022c915a2924740ab7992ad2311 +Subproject commit c43232c1bc378344bb7245d7fcb167410f3fe931 diff --git a/crt/aws-c-s3 b/crt/aws-c-s3 index 0ab4d58ef..aede1d8c2 160000 --- a/crt/aws-c-s3 +++ b/crt/aws-c-s3 @@ -1 +1 @@ -Subproject commit 0ab4d58ef0bd97970d43828cb6b57a3de5747343 +Subproject commit aede1d8c24f9f580d5a96c089878e9b258b88d04 diff --git a/crt/aws-checksums b/crt/aws-checksums index aac442a2d..ce04ab00b 160000 --- a/crt/aws-checksums +++ b/crt/aws-checksums @@ -1 +1 @@ -Subproject commit aac442a2dbbb5e72d0a3eca8313cf65e7e1cac2f +Subproject commit ce04ab00b3ecc41912f478bfedca39f8e1919d6b diff --git a/crt/aws-lc b/crt/aws-lc index 2f1879759..d3a598c1b 160000 --- a/crt/aws-lc +++ b/crt/aws-lc @@ -1 +1 @@ -Subproject commit 2f1879759b2e0fc70592665bdf10087b64f44b7d +Subproject commit d3a598c1b419d49b5b08f0677add4581572e2edc diff --git a/crt/s2n b/crt/s2n index 87f4a0585..08d413a0b 160000 --- a/crt/s2n +++ b/crt/s2n @@ -1 +1 @@ -Subproject commit 87f4a0585dc3056433f193b9305f587cff239be3 +Subproject commit 08d413a0b9b3226e775a38f04e3cf02230cc97c4 diff --git a/source/http_stream.c b/source/http_stream.c index 09943d89d..f6f09bdef 100644 --- a/source/http_stream.c +++ b/source/http_stream.c @@ -203,7 +203,18 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro } /* DECREF python self, we don't need to force it to stay alive any longer. */ - Py_XDECREF(stream->self_proxy); + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ + /* Ignore error, stream is already complete */ + PyWeakref_GetRef(stream->self_proxy, &self) < 0);/* strong reference */ +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + PyWeakref_GetObject(stream->self_proxy); /* borrowed reference */ +#endif + Py_XDECREF(self); +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ + Py_XDECREF(self); +#endif PyGILState_Release(state); /*************** GIL RELEASE ***************/ diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index df1eaed6d..ab5c52916 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -141,7 +141,7 @@ static void s_on_connection_success( } PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ PyErr_WriteUnraisable(PyErr_Occurred()); goto on_done; @@ -180,8 +180,8 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ PyErr_WriteUnraisable(PyErr_Occurred()); goto on_done; @@ -222,7 +222,7 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne /* Ensure that python class is still alive */ PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ PyErr_WriteUnraisable(PyErr_Occurred()); goto on_done; @@ -268,8 +268,8 @@ static void s_on_connection_resumed( } /* Ensure that python class is still alive */ - PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ PyErr_WriteUnraisable(PyErr_Occurred()); goto on_done; @@ -312,8 +312,8 @@ static void s_on_connection_closed( struct mqtt_connection_binding *py_connection = userdata; /* Ensure that python class is still alive */ - PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ PyErr_WriteUnraisable(PyErr_Occurred()); goto on_done; @@ -332,7 +332,6 @@ static void s_on_connection_closed( } } - on_done: #if PY_VERSION_HEX >= 0x030D0000 Py_XDECREF(self); @@ -604,8 +603,8 @@ static void s_ws_handshake_transform( } /* Ensure python mqtt connection object is still alive */ - PyObject *connection_py = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 // Check if Python version is 3.13 or higher + PyObject *connection_py = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ if (PyWeakref_GetRef(connection_binding->self_proxy, &connection_py) < 0) { /* strong reference */ aws_raise_error(AWS_ERROR_INVALID_STATE); goto done; @@ -676,7 +675,6 @@ done:; Py_XDECREF(connection_py); #endif - if (ws_transform_capsule) { Py_DECREF(ws_transform_capsule); } else if (ws_transform_data) { From d7d4bf980d03073e2edf1397a32639bdf0735440 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 09:26:39 -0700 Subject: [PATCH 05/46] builder version --- .github/workflows/ci.yml | 2 +- continuous-delivery/build-wheels-manylinux2014-aarch64.sh | 3 +++ continuous-delivery/build-wheels-manylinux2014-x86_64.sh | 3 +++ continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh | 4 ++++ continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh | 4 ++++ continuous-delivery/build-wheels-osx.sh | 3 +++ continuous-delivery/build-wheels-win32.bat | 6 ++++++ continuous-delivery/build-wheels-win64.bat | 6 ++++++ 8 files changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index be20926e3..324561dc3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ on: - 'docs' env: - BUILDER_VERSION: v0.9.63 + BUILDER_VERSION: v0.9.66 BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python diff --git a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh index a140b7d67..8ff18c898 100755 --- a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh @@ -21,6 +21,9 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp313*.whl rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh index 41ce9fdef..2714fe9ec 100755 --- a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh @@ -21,6 +21,9 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp313*.whl rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh index 0cbc1a196..c2f13cff4 100755 --- a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh +++ b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh @@ -22,6 +22,10 @@ auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp313*.whl + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh index 6637d95ac..250cc5bad 100755 --- a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh +++ b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh @@ -22,6 +22,10 @@ auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +/opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel +auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp313*.whl + rm dist/*.whl cp -rv wheelhouse/* dist/ diff --git a/continuous-delivery/build-wheels-osx.sh b/continuous-delivery/build-wheels-osx.sh index a3a92bc63..1faf26880 100755 --- a/continuous-delivery/build-wheels-osx.sh +++ b/continuous-delivery/build-wheels-osx.sh @@ -14,4 +14,7 @@ set -ex # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. +# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +/Library/Frameworks/Python.framework/Versions/3.13/bin/python3 setup.py sdist bdist_wheel + #now you just need to run twine (that's in a different script) diff --git a/continuous-delivery/build-wheels-win32.bat b/continuous-delivery/build-wheels-win32.bat index 6a3e53977..52b82e49b 100644 --- a/continuous-delivery/build-wheels-win32.bat +++ b/continuous-delivery/build-wheels-win32.bat @@ -7,6 +7,12 @@ "C:\Program Files (x86)\Python310-32\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files (x86)\Python311-32\python.exe" setup.py sdist bdist_wheel || goto error +:: Don't need to build wheels for Python 3.12 and later. +:: The 3.11 wheel uses the stable ABI, so it works with newer versions too. + +:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +"C:\Program Files (x86)\Python313-32\python.exe" setup.py sdist bdist_wheel || goto error + goto :EOF :error diff --git a/continuous-delivery/build-wheels-win64.bat b/continuous-delivery/build-wheels-win64.bat index 5a862e6e1..492fe204e 100644 --- a/continuous-delivery/build-wheels-win64.bat +++ b/continuous-delivery/build-wheels-win64.bat @@ -6,6 +6,12 @@ "C:\Program Files\Python310\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files\Python311\python.exe" setup.py sdist bdist_wheel || goto error +:: Don't need to build wheels for Python 3.12 and later. +:: The 3.11 wheel uses the stable ABI, so it works with newer versions too. + +:: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +"C:\Program Files\Python313\python.exe" setup.py sdist bdist_wheel || goto error + goto :EOF :error From d269cc96dd094e2bc9c4e1383092df915aef9a9e Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 09:44:08 -0700 Subject: [PATCH 06/46] try new creds --- .github/workflows/ci.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 324561dc3..5906be604 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,9 +13,8 @@ env: PACKAGE_NAME: aws-crt-python LINUX_BASE_IMAGE: ubuntu-18-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }} + CRT_CI_ROLE: arn:aws:iam::123124136734:role/CRT_CI_Role + AWS_DEFAULT_REGION: us-east-1 AWS_REGION: us-east-1 jobs: @@ -142,6 +141,11 @@ jobs: - gcc-8 steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh From d89746cf95e1e5667fa0b9f6cfb0bbea996588e5 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 09:47:53 -0700 Subject: [PATCH 07/46] fix typo --- .github/workflows/ci.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5906be604..455448506 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -141,11 +141,11 @@ jobs: - gcc-8 steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh From dce2f067305081efe95439b68d143c87ce280068 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 09:56:19 -0700 Subject: [PATCH 08/46] id token --- .github/workflows/ci.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 455448506..58d82d27c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,6 @@ env: LINUX_BASE_IMAGE: ubuntu-18-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} CRT_CI_ROLE: arn:aws:iam::123124136734:role/CRT_CI_Role - AWS_DEFAULT_REGION: us-east-1 AWS_REGION: us-east-1 jobs: @@ -116,6 +115,8 @@ jobs: - fedora-34-x64 - opensuse-leap - rhel8-x64 + permissions: + id-token: write # This is required for requesting the JWT steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} @@ -145,7 +146,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_DEFAULT_REGION }} + aws-region: ${{ env.AWS_REGION }} - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh From 2555dbd08bc4c4eead3db1cad33686781cab1ab2 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 09:58:28 -0700 Subject: [PATCH 09/46] fix ci --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 58d82d27c..533f3207d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -115,8 +115,6 @@ jobs: - fedora-34-x64 - opensuse-leap - rhel8-x64 - permissions: - id-token: write # This is required for requesting the JWT steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} @@ -140,6 +138,8 @@ jobs: - gcc-6 - gcc-7 - gcc-8 + permissions: + id-token: write # This is required for requesting the JWT steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) From 45119b1cd7883c837c30f832907d873cdb4754bd Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 10:23:01 -0700 Subject: [PATCH 10/46] copy paste ci fix --- .github/workflows/ci.yml | 201 ++++++++++++++++++++++++++++----------- 1 file changed, 146 insertions(+), 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 533f3207d..709e79c88 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,7 +13,7 @@ env: PACKAGE_NAME: aws-crt-python LINUX_BASE_IMAGE: ubuntu-18-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} - CRT_CI_ROLE: arn:aws:iam::123124136734:role/CRT_CI_Role + CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }} AWS_REGION: us-east-1 jobs: @@ -29,11 +29,19 @@ jobs: - cp37-cp37m - cp38-cp38 - cp39-cp39 + permissions: + id-token: write # This is required for requesting the JWT steps: - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python manylinux2014: runs-on: ubuntu-20.04 # latest @@ -52,15 +60,23 @@ jobs: - cp311-cp311 - cp312-cp312 - cp313-cp313 + permissions: + id-token: write # This is required for requesting the JWT steps: - # Only aarch64 needs this, but it doesn't hurt anything - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} + # Only aarch64 needs this, but it doesn't hurt anything + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python musllinux-1-1: runs-on: ubuntu-22.04 # latest @@ -78,15 +94,24 @@ jobs: - cp311-cp311 - cp312-cp312 - cp313-cp313 + permissions: + id-token: write # This is required for requesting the JWT steps: - # Only aarch64 needs this, but it doesn't hurt anything - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + # Only aarch64 needs this, but it doesn't hurt anything + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python raspberry: runs-on: ubuntu-20.04 # latest @@ -95,16 +120,24 @@ jobs: matrix: image: - raspbian-bullseye + permissions: + id-token: write # This is required for requesting the JWT steps: - # set arm arch - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + # set arm arch + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} linux-compat: runs-on: ubuntu-22.04 # latest @@ -115,13 +148,20 @@ jobs: - fedora-34-x64 - opensuse-leap - rhel8-x64 + permissions: + id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} linux-compiler-compat: runs-on: ubuntu-22.04 # latest @@ -154,7 +194,15 @@ jobs: use-system-libcrypto: runs-on: ubuntu-20.04 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} - name: Build ${{ env.PACKAGE_NAME }} env: AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: '1' @@ -180,30 +228,54 @@ jobs: strategy: matrix: arch: [x86, x64] + permissions: + id-token: write # This is required for requesting the JWT steps: - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" macos: runs-on: macos-14 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} macos-x64: runs-on: macos-14-large # latest + permissions: + id-token: write # This is required for requesting the JWT steps: - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} openbsd: @@ -213,7 +285,15 @@ jobs: matrix: # OpenBSD only supports the two most recent releases version: ['7.4', '7.5'] + permissions: + id-token: write # This is required for requesting the JWT steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install - uses: actions/checkout@v4 with: @@ -225,7 +305,7 @@ jobs: version: ${{ matrix.version }} cpu_count: 4 shell: bash - environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_REGION run: | sudo pkg_add awscli py3-pip py3-urllib3 python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" @@ -234,7 +314,15 @@ jobs: freebsd: runs-on: ubuntu-22.04 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} - uses: actions/checkout@v4 with: submodules: true @@ -246,7 +334,7 @@ jobs: version: '14.0' cpu_count: 4 shell: bash - environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_REGION run: | sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" @@ -254,18 +342,13 @@ jobs: ./builder build -p ${{ env.PACKAGE_NAME }} # check that tests requiring custom env-vars or AWS credentials are simply skipped - tests-ok-without-env-vars: + tests-ok-without-creds: runs-on: ubuntu-22.04 # latest steps: - uses: actions/checkout@v4 with: submodules: true - - name: Run tests without env-vars or AWS creds - env: - # unset env-vars that provide AWS credentials - AWS_ACCESS_KEY_ID: - AWS_SECRET_ACCESS_KEY: - AWS_DEFAULT_REGION: + - name: Run tests run: | python3 -m pip install --upgrade --requirement requirements-dev.txt python3 -m pip install . --verbose @@ -273,7 +356,15 @@ jobs: package-source: runs-on: ubuntu-22.04 # latest + permissions: + id-token: write # This is required for requesting the JWT steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_REGION }} - uses: actions/checkout@v4 with: submodules: true From f6abe3d4d634d596aa5e5bc655539575068ed5dc Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 10:38:15 -0700 Subject: [PATCH 11/46] test that I have permissions --- .github/workflows/ci.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 709e79c88..5375f254d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,6 +182,10 @@ jobs: id-token: write # This is required for requesting the JWT steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: tests + run: | + aws sts get-caller-identity + aws --region us-east-1 secretsmanager get-secret-value --secret-id unit-test/endpoint - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -305,7 +309,7 @@ jobs: version: ${{ matrix.version }} cpu_count: 4 shell: bash - environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_REGION + environment_variables: AWS_REGION run: | sudo pkg_add awscli py3-pip py3-urllib3 python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" From 7ddadf40f569b8135482bfc0369981862f0549ae Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 10:39:05 -0700 Subject: [PATCH 12/46] stupid me --- .github/workflows/ci.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5375f254d..a12bd8c70 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -182,15 +182,15 @@ jobs: id-token: write # This is required for requesting the JWT steps: # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: tests - run: | - aws sts get-caller-identity - aws --region us-east-1 secretsmanager get-secret-value --secret-id unit-test/endpoint - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ env.CRT_CI_ROLE }} aws-region: ${{ env.AWS_REGION }} + - name: tests + run: | + aws sts get-caller-identity + aws --region us-east-1 secretsmanager get-secret-value --secret-id unit-test/endpoint - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh From 2266d9d398e92bb57777202007b6a47bc9d06d08 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 10:47:21 -0700 Subject: [PATCH 13/46] maybe something uses default region? --- .github/workflows/ci.yml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a12bd8c70..1520c14b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,7 @@ env: RUN: ${{ github.run_id }}-${{ github.run_number }} CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }} AWS_REGION: us-east-1 + AWS_DEFAULT_REGION: us-east-1 jobs: manylinux1: @@ -187,10 +188,6 @@ jobs: with: role-to-assume: ${{ env.CRT_CI_ROLE }} aws-region: ${{ env.AWS_REGION }} - - name: tests - run: | - aws sts get-caller-identity - aws --region us-east-1 secretsmanager get-secret-value --secret-id unit-test/endpoint - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh From 2adb9d44be395c7ae598cbd0ae54d5f6975f3a35 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 13:18:17 -0700 Subject: [PATCH 14/46] test-creds --- .github/workflows/ci.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1520c14b6..9e7caec79 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,8 @@ on: - 'docs' env: - BUILDER_VERSION: v0.9.66 - BUILDER_SOURCE: releases + BUILDER_VERSION: test-creds + BUILDER_SOURCE: channels BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python LINUX_BASE_IMAGE: ubuntu-18-x64 @@ -188,6 +188,9 @@ jobs: with: role-to-assume: ${{ env.CRT_CI_ROLE }} aws-region: ${{ env.AWS_REGION }} + - name: Caller Identity + run: | + aws sts get-caller-identity - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh From 87ffaa19c43ed50b2d52afd6ab632b4ad3231600 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 13:19:28 -0700 Subject: [PATCH 15/46] disable other CIs --- .github/workflows/ci.yml | 726 +++++++++++++++++++-------------------- 1 file changed, 363 insertions(+), 363 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9e7caec79..4f8b1b2d0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,167 +18,167 @@ env: AWS_DEFAULT_REGION: us-east-1 jobs: - manylinux1: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - image: - - x64 - - x86 - python: - - cp37-cp37m - - cp38-cp38 - - cp39-cp39 - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python - - manylinux2014: - runs-on: ubuntu-20.04 # latest - strategy: - fail-fast: false - matrix: - image: - - x64 - - x86 - - aarch64 - python: - - cp37-cp37m - - cp38-cp38 - - cp39-cp39 - - cp310-cp310 - - cp311-cp311 - - cp312-cp312 - - cp313-cp313 - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - # Only aarch64 needs this, but it doesn't hurt anything - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python - - musllinux-1-1: - runs-on: ubuntu-22.04 # latest - strategy: - fail-fast: false - matrix: - image: - - x64 - - aarch64 - python: - - cp37-cp37m - - cp38-cp38 - - cp39-cp39 - - cp310-cp310 - - cp311-cp311 - - cp312-cp312 - - cp313-cp313 - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - # Only aarch64 needs this, but it doesn't hurt anything - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python - - raspberry: - runs-on: ubuntu-20.04 # latest - strategy: - fail-fast: false - matrix: - image: - - raspbian-bullseye - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - # set arm arch - - name: Install qemu/docker - run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes - - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - - linux-compat: - runs-on: ubuntu-22.04 # latest - strategy: - matrix: - image: - - al2-x64 - - fedora-34-x64 - - opensuse-leap - - rhel8-x64 - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: Build ${{ env.PACKAGE_NAME }} - run: | - aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh - ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} - +# manylinux1: +# runs-on: ubuntu-latest +# strategy: +# fail-fast: false +# matrix: +# image: +# - x64 +# - x86 +# python: +# - cp37-cp37m +# - cp38-cp38 +# - cp39-cp39 +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - name: Build ${{ env.PACKAGE_NAME }} +# run: | +# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh +# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python +# +# manylinux2014: +# runs-on: ubuntu-20.04 # latest +# strategy: +# fail-fast: false +# matrix: +# image: +# - x64 +# - x86 +# - aarch64 +# python: +# - cp37-cp37m +# - cp38-cp38 +# - cp39-cp39 +# - cp310-cp310 +# - cp311-cp311 +# - cp312-cp312 +# - cp313-cp313 +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# # Only aarch64 needs this, but it doesn't hurt anything +# - name: Install qemu/docker +# run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# +# - name: Build ${{ env.PACKAGE_NAME }} +# run: | +# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh +# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python +# +# musllinux-1-1: +# runs-on: ubuntu-22.04 # latest +# strategy: +# fail-fast: false +# matrix: +# image: +# - x64 +# - aarch64 +# python: +# - cp37-cp37m +# - cp38-cp38 +# - cp39-cp39 +# - cp310-cp310 +# - cp311-cp311 +# - cp312-cp312 +# - cp313-cp313 +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# +# # Only aarch64 needs this, but it doesn't hurt anything +# - name: Install qemu/docker +# run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# +# - name: Build ${{ env.PACKAGE_NAME }} +# run: | +# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh +# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python +# +# raspberry: +# runs-on: ubuntu-20.04 # latest +# strategy: +# fail-fast: false +# matrix: +# image: +# - raspbian-bullseye +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# +# # set arm arch +# - name: Install qemu/docker +# run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes +# +# - name: Build ${{ env.PACKAGE_NAME }} +# run: | +# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh +# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} +# +# linux-compat: +# runs-on: ubuntu-22.04 # latest +# strategy: +# matrix: +# image: +# - al2-x64 +# - fedora-34-x64 +# - opensuse-leap +# - rhel8-x64 +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: Build ${{ env.PACKAGE_NAME }} +# run: | +# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh +# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} +# linux-compiler-compat: runs-on: ubuntu-22.04 # latest strategy: matrix: compiler: - - clang-3 - - clang-6 - - clang-8 - - clang-9 - - clang-10 +# - clang-3 +# - clang-6 +# - clang-8 +# - clang-9 +# - clang-10 - clang-11 - - gcc-5 - - gcc-6 - - gcc-7 - - gcc-8 +# - gcc-5 +# - gcc-6 +# - gcc-7 +# - gcc-8 permissions: id-token: write # This is required for requesting the JWT steps: @@ -196,211 +196,211 @@ jobs: aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} - use-system-libcrypto: - runs-on: ubuntu-20.04 # latest - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} - env: - AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: '1' - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} - - name: Assert libcrypto.so used - run: | - # assert it's linked against the system's libcrypto.so - AWSCRT_PATH=`aws-crt-python/.venv-builder/bin/python3 -c "import _awscrt; print(_awscrt.__file__)"` - printf "AWSCRT_PATH: $AWSCRT_PATH\n" - - LINKED_AGAINST=`ldd $AWSCRT_PATH` - printf "LINKED AGAINST:\n$LINKED_AGAINST\n" - - USES_LIBCRYPTO_SO=`echo "$LINKED_AGAINST" | grep 'libcrypto*.so' | head -1` - test -n "$USES_LIBCRYPTO_SO" - - - windows: - runs-on: windows-2022 # latest - strategy: - matrix: - arch: [x86, x64] - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" - python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" - - - macos: - runs-on: macos-14 # latest - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} - - macos-x64: - runs-on: macos-14-large # latest - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - name: Build ${{ env.PACKAGE_NAME }} + consumers - run: | - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} - - - openbsd: - runs-on: ubuntu-22.04 # latest - strategy: - fail-fast: false - matrix: - # OpenBSD only supports the two most recent releases - version: ['7.4', '7.5'] - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install - - uses: actions/checkout@v4 - with: - submodules: true - - name: Build ${{ env.PACKAGE_NAME }} + consumers - uses: cross-platform-actions/action@v0.24.0 - with: - operating_system: openbsd - version: ${{ matrix.version }} - cpu_count: 4 - shell: bash - environment_variables: AWS_REGION - run: | - sudo pkg_add awscli py3-pip py3-urllib3 - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} - - freebsd: - runs-on: ubuntu-22.04 # latest - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - uses: actions/checkout@v4 - with: - submodules: true - - - name: Build ${{ env.PACKAGE_NAME }} + consumers - uses: cross-platform-actions/action@v0.23.0 - with: - operating_system: freebsd - version: '14.0' - cpu_count: 4 - shell: bash - environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_REGION - run: | - sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake - python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" - chmod a+x builder - ./builder build -p ${{ env.PACKAGE_NAME }} - - # check that tests requiring custom env-vars or AWS credentials are simply skipped - tests-ok-without-creds: - runs-on: ubuntu-22.04 # latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Run tests - run: | - python3 -m pip install --upgrade --requirement requirements-dev.txt - python3 -m pip install . --verbose - python3 -m unittest discover --failfast --verbose - - package-source: - runs-on: ubuntu-22.04 # latest - permissions: - id-token: write # This is required for requesting the JWT - steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - - name: configure AWS credentials (containers) - uses: aws-actions/configure-aws-credentials@v4 - with: - role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} - - uses: actions/checkout@v4 - with: - submodules: true - - name: Package source + install - run: | - python3 setup.py sdist - cd dist - python3 -m pip install -v awscrt-1.0.0.dev0.tar.gz - python3 -c "import awscrt.io" - - # check that docs can still build - check-docs: - runs-on: ubuntu-22.04 # latest - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Check docs - run: | - python3 -m pip install -r requirements-dev.txt - python3 -m pip install --verbose . - ./scripts/make-docs.py - - check-submodules: - runs-on: ubuntu-22.04 # latest - steps: - - name: Checkout Source - uses: actions/checkout@v4 - with: - submodules: true - fetch-depth: 0 - - name: Check Submodules - # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work - # https://github.com/actions/runner/issues/480 - uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main +# use-system-libcrypto: +# runs-on: ubuntu-20.04 # latest +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - name: Build ${{ env.PACKAGE_NAME }} +# env: +# AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: '1' +# run: | +# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" +# chmod a+x builder +# ./builder build -p ${{ env.PACKAGE_NAME }} +# - name: Assert libcrypto.so used +# run: | +# # assert it's linked against the system's libcrypto.so +# AWSCRT_PATH=`aws-crt-python/.venv-builder/bin/python3 -c "import _awscrt; print(_awscrt.__file__)"` +# printf "AWSCRT_PATH: $AWSCRT_PATH\n" +# +# LINKED_AGAINST=`ldd $AWSCRT_PATH` +# printf "LINKED AGAINST:\n$LINKED_AGAINST\n" +# +# USES_LIBCRYPTO_SO=`echo "$LINKED_AGAINST" | grep 'libcrypto*.so' | head -1` +# test -n "$USES_LIBCRYPTO_SO" +# +# +# windows: +# runs-on: windows-2022 # latest +# strategy: +# matrix: +# arch: [x86, x64] +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - name: Build ${{ env.PACKAGE_NAME }} + consumers +# run: | +# python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" +# python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" +# +# +# macos: +# runs-on: macos-14 # latest +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - name: Build ${{ env.PACKAGE_NAME }} + consumers +# run: | +# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" +# chmod a+x builder +# ./builder build -p ${{ env.PACKAGE_NAME }} +# +# macos-x64: +# runs-on: macos-14-large # latest +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - name: Build ${{ env.PACKAGE_NAME }} + consumers +# run: | +# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" +# chmod a+x builder +# ./builder build -p ${{ env.PACKAGE_NAME }} +# +# +# openbsd: +# runs-on: ubuntu-22.04 # latest +# strategy: +# fail-fast: false +# matrix: +# # OpenBSD only supports the two most recent releases +# version: ['7.4', '7.5'] +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install +# - uses: actions/checkout@v4 +# with: +# submodules: true +# - name: Build ${{ env.PACKAGE_NAME }} + consumers +# uses: cross-platform-actions/action@v0.24.0 +# with: +# operating_system: openbsd +# version: ${{ matrix.version }} +# cpu_count: 4 +# shell: bash +# environment_variables: AWS_REGION +# run: | +# sudo pkg_add awscli py3-pip py3-urllib3 +# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" +# chmod a+x builder +# ./builder build -p ${{ env.PACKAGE_NAME }} +# +# freebsd: +# runs-on: ubuntu-22.04 # latest +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - uses: actions/checkout@v4 +# with: +# submodules: true +# +# - name: Build ${{ env.PACKAGE_NAME }} + consumers +# uses: cross-platform-actions/action@v0.23.0 +# with: +# operating_system: freebsd +# version: '14.0' +# cpu_count: 4 +# shell: bash +# environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_REGION +# run: | +# sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake +# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" +# chmod a+x builder +# ./builder build -p ${{ env.PACKAGE_NAME }} +# +# # check that tests requiring custom env-vars or AWS credentials are simply skipped +# tests-ok-without-creds: +# runs-on: ubuntu-22.04 # latest +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: true +# - name: Run tests +# run: | +# python3 -m pip install --upgrade --requirement requirements-dev.txt +# python3 -m pip install . --verbose +# python3 -m unittest discover --failfast --verbose +# +# package-source: +# runs-on: ubuntu-22.04 # latest +# permissions: +# id-token: write # This is required for requesting the JWT +# steps: +# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages +# - name: configure AWS credentials (containers) +# uses: aws-actions/configure-aws-credentials@v4 +# with: +# role-to-assume: ${{ env.CRT_CI_ROLE }} +# aws-region: ${{ env.AWS_REGION }} +# - uses: actions/checkout@v4 +# with: +# submodules: true +# - name: Package source + install +# run: | +# python3 setup.py sdist +# cd dist +# python3 -m pip install -v awscrt-1.0.0.dev0.tar.gz +# python3 -c "import awscrt.io" +# +# # check that docs can still build +# check-docs: +# runs-on: ubuntu-22.04 # latest +# steps: +# - uses: actions/checkout@v4 +# with: +# submodules: true +# - name: Check docs +# run: | +# python3 -m pip install -r requirements-dev.txt +# python3 -m pip install --verbose . +# ./scripts/make-docs.py +# +# check-submodules: +# runs-on: ubuntu-22.04 # latest +# steps: +# - name: Checkout Source +# uses: actions/checkout@v4 +# with: +# submodules: true +# fetch-depth: 0 +# - name: Check Submodules +# # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work +# # https://github.com/actions/runner/issues/480 +# uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main From f820bab41b53acc0f0b535b1ffdfee4350146dd2 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:05:42 -0700 Subject: [PATCH 16/46] fix unused warning --- source/mqtt_client_connection.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index ab5c52916..ee6496acb 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -160,6 +160,7 @@ static void s_on_connection_success( PyErr_WriteUnraisable(PyErr_Occurred()); } } + goto on_done; /* fixes unused label waring */ on_done: #if PY_VERSION_HEX >= 0x030D0000 Py_XDECREF(self); @@ -200,6 +201,7 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio } } + goto on_done; /* fixes unused label waring */ on_done: #if PY_VERSION_HEX >= 0x030D0000 Py_XDECREF(self); @@ -241,6 +243,7 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } } + goto on_done; /* fixes unused label waring */ on_done: #if PY_VERSION_HEX >= 0x030D0000 Py_XDECREF(self); @@ -288,6 +291,7 @@ static void s_on_connection_resumed( PyErr_WriteUnraisable(PyErr_Occurred()); } } + goto on_done; /* fixes unused label waring */ on_done: #if PY_VERSION_HEX >= 0x030D0000 Py_XDECREF(self); @@ -332,6 +336,7 @@ static void s_on_connection_closed( } } + goto on_done; /* fixes unused label waring */ on_done: #if PY_VERSION_HEX >= 0x030D0000 Py_XDECREF(self); From 281d3c69f94c6400948e2f2a989b0c260ae5dd84 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:21:03 -0700 Subject: [PATCH 17/46] fix bugs --- source/http_stream.c | 5 +++-- source/mqtt_client_connection.c | 12 ++++++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/source/http_stream.c b/source/http_stream.c index f6f09bdef..bcf0f9487 100644 --- a/source/http_stream.c +++ b/source/http_stream.c @@ -205,11 +205,12 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro /* DECREF python self, we don't need to force it to stay alive any longer. */ PyObject *self = Py_None; #if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ + int result_getref = PyWeakref_GetRef(stream->self_proxy, &self);/* strong reference */ /* Ignore error, stream is already complete */ - PyWeakref_GetRef(stream->self_proxy, &self) < 0);/* strong reference */ + (void) result_getref; #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(stream->self_proxy); /* borrowed reference */ + self = PyWeakref_GetObject(stream->self_proxy); /* borrowed reference */ #endif Py_XDECREF(self); #if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index ee6496acb..513604d9a 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -148,7 +148,7 @@ static void s_on_connection_success( } #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ #endif if (self != Py_None) { @@ -189,7 +189,7 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio } #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ #endif if (self != Py_None) { @@ -231,7 +231,7 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ #endif if (self != Py_None) { @@ -279,7 +279,7 @@ static void s_on_connection_resumed( } #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ #endif if (self != Py_None) { @@ -324,7 +324,7 @@ static void s_on_connection_closed( } #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ + self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ #endif if (self != Py_None) { @@ -616,7 +616,7 @@ static void s_ws_handshake_transform( } #else /* PyWeakref_GetObject is deprecated since python 3.13 */ - PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ + connection_py = PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ #endif if (connection_py == Py_None) { From e94ed91b44f1876715393acbbe3fcb9170f347a3 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:25:58 -0700 Subject: [PATCH 18/46] reenable CI --- .github/workflows/ci.yml | 729 +++++++++++++++++++-------------------- 1 file changed, 364 insertions(+), 365 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f8b1b2d0..e5369d826 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,171 +14,170 @@ env: LINUX_BASE_IMAGE: ubuntu-18-x64 RUN: ${{ github.run_id }}-${{ github.run_number }} CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }} - AWS_REGION: us-east-1 AWS_DEFAULT_REGION: us-east-1 jobs: -# manylinux1: -# runs-on: ubuntu-latest -# strategy: -# fail-fast: false -# matrix: -# image: -# - x64 -# - x86 -# python: -# - cp37-cp37m -# - cp38-cp38 -# - cp39-cp39 -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - name: Build ${{ env.PACKAGE_NAME }} -# run: | -# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh -# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python -# -# manylinux2014: -# runs-on: ubuntu-20.04 # latest -# strategy: -# fail-fast: false -# matrix: -# image: -# - x64 -# - x86 -# - aarch64 -# python: -# - cp37-cp37m -# - cp38-cp38 -# - cp39-cp39 -# - cp310-cp310 -# - cp311-cp311 -# - cp312-cp312 -# - cp313-cp313 -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# # Only aarch64 needs this, but it doesn't hurt anything -# - name: Install qemu/docker -# run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -# -# - name: Build ${{ env.PACKAGE_NAME }} -# run: | -# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh -# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python -# -# musllinux-1-1: -# runs-on: ubuntu-22.04 # latest -# strategy: -# fail-fast: false -# matrix: -# image: -# - x64 -# - aarch64 -# python: -# - cp37-cp37m -# - cp38-cp38 -# - cp39-cp39 -# - cp310-cp310 -# - cp311-cp311 -# - cp312-cp312 -# - cp313-cp313 -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# -# # Only aarch64 needs this, but it doesn't hurt anything -# - name: Install qemu/docker -# run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -# -# - name: Build ${{ env.PACKAGE_NAME }} -# run: | -# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh -# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python -# -# raspberry: -# runs-on: ubuntu-20.04 # latest -# strategy: -# fail-fast: false -# matrix: -# image: -# - raspbian-bullseye -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# -# # set arm arch -# - name: Install qemu/docker -# run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -# -# - name: Build ${{ env.PACKAGE_NAME }} -# run: | -# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh -# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} -# -# linux-compat: -# runs-on: ubuntu-22.04 # latest -# strategy: -# matrix: -# image: -# - al2-x64 -# - fedora-34-x64 -# - opensuse-leap -# - rhel8-x64 -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: Build ${{ env.PACKAGE_NAME }} -# run: | -# aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh -# ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} -# + manylinux1: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + image: + - x64 + - x86 + python: + - cp37-cp37m + - cp38-cp38 + - cp39-cp39 + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + + manylinux2014: + runs-on: ubuntu-20.04 # latest + strategy: + fail-fast: false + matrix: + image: + - x64 + - x86 + - aarch64 + python: + - cp37-cp37m + - cp38-cp38 + - cp39-cp39 + - cp310-cp310 + - cp311-cp311 + - cp312-cp312 + - cp313-cp313 + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + # Only aarch64 needs this, but it doesn't hurt anything + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-manylinux2014-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + + musllinux-1-1: + runs-on: ubuntu-22.04 # latest + strategy: + fail-fast: false + matrix: + image: + - x64 + - aarch64 + python: + - cp37-cp37m + - cp38-cp38 + - cp39-cp39 + - cp310-cp310 + - cp311-cp311 + - cp312-cp312 + - cp313-cp313 + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + # Only aarch64 needs this, but it doesn't hurt anything + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-musllinux-1-1-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} --python /opt/python/${{ matrix.python }}/bin/python + + raspberry: + runs-on: ubuntu-20.04 # latest + strategy: + fail-fast: false + matrix: + image: + - raspbian-bullseye + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + + # set arm arch + - name: Install qemu/docker + run: docker run --rm --privileged multiarch/qemu-user-static --reset -p yes + + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + + linux-compat: + runs-on: ubuntu-22.04 # latest + strategy: + matrix: + image: + - al2-x64 + - fedora-34-x64 + - opensuse-leap + - rhel8-x64 + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: Build ${{ env.PACKAGE_NAME }} + run: | + aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh + ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ matrix.image }} build -p ${{ env.PACKAGE_NAME }} + linux-compiler-compat: runs-on: ubuntu-22.04 # latest strategy: matrix: compiler: -# - clang-3 -# - clang-6 -# - clang-8 -# - clang-9 -# - clang-10 + - clang-3 + - clang-6 + - clang-8 + - clang-9 + - clang-10 - clang-11 -# - gcc-5 -# - gcc-6 -# - gcc-7 -# - gcc-8 + - gcc-5 + - gcc-6 + - gcc-7 + - gcc-8 permissions: id-token: write # This is required for requesting the JWT steps: @@ -187,7 +186,7 @@ jobs: uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ env.CRT_CI_ROLE }} - aws-region: ${{ env.AWS_REGION }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} - name: Caller Identity run: | aws sts get-caller-identity @@ -196,211 +195,211 @@ jobs: aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh ./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-${{ env.LINUX_BASE_IMAGE }} build -p ${{ env.PACKAGE_NAME }} --compiler=${{ matrix.compiler }} -# use-system-libcrypto: -# runs-on: ubuntu-20.04 # latest -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - name: Build ${{ env.PACKAGE_NAME }} -# env: -# AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: '1' -# run: | -# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" -# chmod a+x builder -# ./builder build -p ${{ env.PACKAGE_NAME }} -# - name: Assert libcrypto.so used -# run: | -# # assert it's linked against the system's libcrypto.so -# AWSCRT_PATH=`aws-crt-python/.venv-builder/bin/python3 -c "import _awscrt; print(_awscrt.__file__)"` -# printf "AWSCRT_PATH: $AWSCRT_PATH\n" -# -# LINKED_AGAINST=`ldd $AWSCRT_PATH` -# printf "LINKED AGAINST:\n$LINKED_AGAINST\n" -# -# USES_LIBCRYPTO_SO=`echo "$LINKED_AGAINST" | grep 'libcrypto*.so' | head -1` -# test -n "$USES_LIBCRYPTO_SO" -# -# -# windows: -# runs-on: windows-2022 # latest -# strategy: -# matrix: -# arch: [x86, x64] -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - name: Build ${{ env.PACKAGE_NAME }} + consumers -# run: | -# python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" -# python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" -# -# -# macos: -# runs-on: macos-14 # latest -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - name: Build ${{ env.PACKAGE_NAME }} + consumers -# run: | -# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" -# chmod a+x builder -# ./builder build -p ${{ env.PACKAGE_NAME }} -# -# macos-x64: -# runs-on: macos-14-large # latest -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - name: Build ${{ env.PACKAGE_NAME }} + consumers -# run: | -# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" -# chmod a+x builder -# ./builder build -p ${{ env.PACKAGE_NAME }} -# -# -# openbsd: -# runs-on: ubuntu-22.04 # latest -# strategy: -# fail-fast: false -# matrix: -# # OpenBSD only supports the two most recent releases -# version: ['7.4', '7.5'] -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install -# - uses: actions/checkout@v4 -# with: -# submodules: true -# - name: Build ${{ env.PACKAGE_NAME }} + consumers -# uses: cross-platform-actions/action@v0.24.0 -# with: -# operating_system: openbsd -# version: ${{ matrix.version }} -# cpu_count: 4 -# shell: bash -# environment_variables: AWS_REGION -# run: | -# sudo pkg_add awscli py3-pip py3-urllib3 -# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" -# chmod a+x builder -# ./builder build -p ${{ env.PACKAGE_NAME }} -# -# freebsd: -# runs-on: ubuntu-22.04 # latest -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - uses: actions/checkout@v4 -# with: -# submodules: true -# -# - name: Build ${{ env.PACKAGE_NAME }} + consumers -# uses: cross-platform-actions/action@v0.23.0 -# with: -# operating_system: freebsd -# version: '14.0' -# cpu_count: 4 -# shell: bash -# environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_REGION -# run: | -# sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake -# python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" -# chmod a+x builder -# ./builder build -p ${{ env.PACKAGE_NAME }} -# -# # check that tests requiring custom env-vars or AWS credentials are simply skipped -# tests-ok-without-creds: -# runs-on: ubuntu-22.04 # latest -# steps: -# - uses: actions/checkout@v4 -# with: -# submodules: true -# - name: Run tests -# run: | -# python3 -m pip install --upgrade --requirement requirements-dev.txt -# python3 -m pip install . --verbose -# python3 -m unittest discover --failfast --verbose -# -# package-source: -# runs-on: ubuntu-22.04 # latest -# permissions: -# id-token: write # This is required for requesting the JWT -# steps: -# # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages -# - name: configure AWS credentials (containers) -# uses: aws-actions/configure-aws-credentials@v4 -# with: -# role-to-assume: ${{ env.CRT_CI_ROLE }} -# aws-region: ${{ env.AWS_REGION }} -# - uses: actions/checkout@v4 -# with: -# submodules: true -# - name: Package source + install -# run: | -# python3 setup.py sdist -# cd dist -# python3 -m pip install -v awscrt-1.0.0.dev0.tar.gz -# python3 -c "import awscrt.io" -# -# # check that docs can still build -# check-docs: -# runs-on: ubuntu-22.04 # latest -# steps: -# - uses: actions/checkout@v4 -# with: -# submodules: true -# - name: Check docs -# run: | -# python3 -m pip install -r requirements-dev.txt -# python3 -m pip install --verbose . -# ./scripts/make-docs.py -# -# check-submodules: -# runs-on: ubuntu-22.04 # latest -# steps: -# - name: Checkout Source -# uses: actions/checkout@v4 -# with: -# submodules: true -# fetch-depth: 0 -# - name: Check Submodules -# # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work -# # https://github.com/actions/runner/issues/480 -# uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main + use-system-libcrypto: + runs-on: ubuntu-20.04 # latest + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + env: + AWS_CRT_BUILD_USE_SYSTEM_LIBCRYPTO: '1' + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + - name: Assert libcrypto.so used + run: | + # assert it's linked against the system's libcrypto.so + AWSCRT_PATH=`aws-crt-python/.venv-builder/bin/python3 -c "import _awscrt; print(_awscrt.__file__)"` + printf "AWSCRT_PATH: $AWSCRT_PATH\n" + + LINKED_AGAINST=`ldd $AWSCRT_PATH` + printf "LINKED AGAINST:\n$LINKED_AGAINST\n" + + USES_LIBCRYPTO_SO=`echo "$LINKED_AGAINST" | grep 'libcrypto*.so' | head -1` + test -n "$USES_LIBCRYPTO_SO" + + + windows: + runs-on: windows-2022 # latest + strategy: + matrix: + arch: [x86, x64] + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')" + python builder.pyz build -p ${{ env.PACKAGE_NAME }} --python "C:\\hostedtoolcache\\windows\\Python\\3.7.9\\${{ matrix.arch }}\\python.exe" + + + macos: + runs-on: macos-14 # latest + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + + macos-x64: + runs-on: macos-14-large # latest + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - name: Build ${{ env.PACKAGE_NAME }} + consumers + run: | + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + + + openbsd: + runs-on: ubuntu-22.04 # latest + strategy: + fail-fast: false + matrix: + # OpenBSD only supports the two most recent releases + version: ['7.4', '7.5'] + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + # Cannot use builder to checkout as OpenBSD doesn't ship git in the base install + - uses: actions/checkout@v4 + with: + submodules: true + - name: Build ${{ env.PACKAGE_NAME }} + consumers + uses: cross-platform-actions/action@v0.24.0 + with: + operating_system: openbsd + version: ${{ matrix.version }} + cpu_count: 4 + shell: bash + environment_variables: AWS_DEFAULT_REGION + run: | + sudo pkg_add awscli py3-pip py3-urllib3 + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + + freebsd: + runs-on: ubuntu-22.04 # latest + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - uses: actions/checkout@v4 + with: + submodules: true + + - name: Build ${{ env.PACKAGE_NAME }} + consumers + uses: cross-platform-actions/action@v0.23.0 + with: + operating_system: freebsd + version: '14.0' + cpu_count: 4 + shell: bash + environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + run: | + sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake + python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" + chmod a+x builder + ./builder build -p ${{ env.PACKAGE_NAME }} + + # check that tests requiring custom env-vars or AWS credentials are simply skipped + tests-ok-without-creds: + runs-on: ubuntu-22.04 # latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Run tests + run: | + python3 -m pip install --upgrade --requirement requirements-dev.txt + python3 -m pip install . --verbose + python3 -m unittest discover --failfast --verbose + + package-source: + runs-on: ubuntu-22.04 # latest + permissions: + id-token: write # This is required for requesting the JWT + steps: + # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages + - name: configure AWS credentials (containers) + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ env.CRT_CI_ROLE }} + aws-region: ${{ env.AWS_DEFAULT_REGION }} + - uses: actions/checkout@v4 + with: + submodules: true + - name: Package source + install + run: | + python3 setup.py sdist + cd dist + python3 -m pip install -v awscrt-1.0.0.dev0.tar.gz + python3 -c "import awscrt.io" + + # check that docs can still build + check-docs: + runs-on: ubuntu-22.04 # latest + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Check docs + run: | + python3 -m pip install -r requirements-dev.txt + python3 -m pip install --verbose . + ./scripts/make-docs.py + + check-submodules: + runs-on: ubuntu-22.04 # latest + steps: + - name: Checkout Source + uses: actions/checkout@v4 + with: + submodules: true + fetch-depth: 0 + - name: Check Submodules + # note: using "@main" because "@${{env.BUILDER_VERSION}}" doesn't work + # https://github.com/actions/runner/issues/480 + uses: awslabs/aws-crt-builder/.github/actions/check-submodules@main From c5ff663600c196f1bb614a60c5da7473dbb412aa Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:26:46 -0700 Subject: [PATCH 19/46] lint --- setup.py | 4 ++-- source/http_stream.c | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index cece200cc..a14239bd1 100644 --- a/setup.py +++ b/setup.py @@ -403,10 +403,10 @@ def awscrt_ext(): else: extra_link_args += ['-Wl,--fatal-warnings'] - if sys.version_info >= (3,13): + if sys.version_info >= (3, 13): define_macros.append(('Py_LIMITED_API', '0x030D0000')) py_limited_api = True - elif sys.version_info >= (3,11): + elif sys.version_info >= (3, 11): define_macros.append(('Py_LIMITED_API', '0x030B0000')) py_limited_api = True diff --git a/source/http_stream.c b/source/http_stream.c index bcf0f9487..5097771eb 100644 --- a/source/http_stream.c +++ b/source/http_stream.c @@ -204,10 +204,10 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro /* DECREF python self, we don't need to force it to stay alive any longer. */ PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ - int result_getref = PyWeakref_GetRef(stream->self_proxy, &self);/* strong reference */ +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ + int result_getref = PyWeakref_GetRef(stream->self_proxy, &self); /* strong reference */ /* Ignore error, stream is already complete */ - (void) result_getref; + (void)result_getref; #else /* PyWeakref_GetObject is deprecated since python 3.13 */ self = PyWeakref_GetObject(stream->self_proxy); /* borrowed reference */ From 3577f753171e081864639660c96b518c314ed2df Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:30:37 -0700 Subject: [PATCH 20/46] fix freebsd --- .github/workflows/ci.yml | 2 +- continuous-delivery/build-wheels-manylinux2014-aarch64.sh | 2 +- continuous-delivery/build-wheels-manylinux2014-x86_64.sh | 2 +- continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh | 2 +- continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh | 2 +- continuous-delivery/build-wheels-osx.sh | 2 +- 6 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e5369d826..abccae403 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -337,7 +337,7 @@ jobs: version: '14.0' cpu_count: 4 shell: bash - environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_DEFAULT_REGION AWS_REGION + environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_DEFAULT_REGION run: | sudo pkg install -y python3 devel/py-pip net/py-urllib3 devel/py-awscli cmake python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" diff --git a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh index 8ff18c898..3deed7ffa 100755 --- a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh @@ -21,7 +21,7 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. -# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp313*.whl diff --git a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh index 2714fe9ec..0b66d34e8 100755 --- a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh @@ -21,7 +21,7 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. -# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp313*.whl diff --git a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh index c2f13cff4..add3a38bc 100755 --- a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh +++ b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh @@ -22,7 +22,7 @@ auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. -# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp313*.whl diff --git a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh index 250cc5bad..9f405705f 100755 --- a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh +++ b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh @@ -22,7 +22,7 @@ auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. -# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp313*.whl diff --git a/continuous-delivery/build-wheels-osx.sh b/continuous-delivery/build-wheels-osx.sh index 1faf26880..802c5d349 100755 --- a/continuous-delivery/build-wheels-osx.sh +++ b/continuous-delivery/build-wheels-osx.sh @@ -14,7 +14,7 @@ set -ex # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. -# We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. +# We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. /Library/Frameworks/Python.framework/Versions/3.13/bin/python3 setup.py sdist bdist_wheel #now you just need to run twine (that's in a different script) From 5b38b528e17c3fe5b730a8a96c8c896c69491494 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:36:39 -0700 Subject: [PATCH 21/46] fix openbsd --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index abccae403..bb7aec11d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -308,7 +308,7 @@ jobs: version: ${{ matrix.version }} cpu_count: 4 shell: bash - environment_variables: AWS_DEFAULT_REGION + environment_variables: AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_SESSION_TOKEN AWS_DEFAULT_REGION run: | sudo pkg_add awscli py3-pip py3-urllib3 python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz', 'builder')" From 0b598b01079aec8785682126ac34e8620b240fa3 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 14:53:51 -0700 Subject: [PATCH 22/46] fix default chain test --- test/test_mqtt5_credentials.py | 44 ++++++++-------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/test/test_mqtt5_credentials.py b/test/test_mqtt5_credentials.py index 03e5c401e..b902e2bbd 100644 --- a/test/test_mqtt5_credentials.py +++ b/test/test_mqtt5_credentials.py @@ -222,39 +222,6 @@ def sign_function(transform_args, **kwargs): client.stop() callbacks.future_stopped.result(TIMEOUT) - def test_mqtt5_ws_cred_default(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") - input_region = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_REGION") - - client_options = mqtt5.ClientOptions( - host_name=input_host_name, - port=443 - ) - credentials = auth.AwsCredentialsProvider.new_default_chain() - - def sign_function(transform_args, **kwargs): - signing_config = auth.AwsSigningConfig( - algorithm=auth.AwsSigningAlgorithm.V4, - signature_type=auth.AwsSignatureType.HTTP_REQUEST_QUERY_PARAMS, - credentials_provider=credentials, - region=input_region, - service="iotdevicegateway", - omit_session_token=True - ) - signing_future = auth.aws_sign_request( - http_request=transform_args.http_request, - signing_config=signing_config) - signing_future.add_done_callback(lambda x: transform_args.set_done(x.exception())) - client_options.websocket_handshake_transform = sign_function - client_options.tls_ctx = io.ClientTlsContext(io.TlsContextOptions()) - - callbacks = Mqtt5TestCallbacks() - client = self._create_client(client_options=client_options, callbacks=callbacks) - client.start() - callbacks.future_connection_success.result(TIMEOUT) - client.stop() - callbacks.future_stopped.result(TIMEOUT) - def test_mqtt5_ws_cred_cognito(self): input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") input_cognito_endpoint = _get_env_variable("AWS_TEST_MQTT5_COGNITO_ENDPOINT") @@ -380,6 +347,12 @@ def sign_function(transform_args, **kwargs): callbacks.future_stopped.result(TIMEOUT) def test_mqtt5_ws_cred_environment(self): + self._test_mqtt5_ws_cred_environment(use_default_chain=False) + + def test_mqtt5_ws_cred_default_chain(self): + self._test_mqtt5_ws_cred_environment(use_default_chain=True) + + def _test_mqtt5_ws_cred_environment(self, use_default_chain): input_host_name = _get_env_variable("AWS_TEST_MQTT5_IOT_CORE_HOST") input_access_key = _get_env_variable("AWS_TEST_MQTT5_ROLE_CREDENTIAL_ACCESS_KEY") input_secret_access_key = _get_env_variable("AWS_TEST_MQTT5_ROLE_CREDENTIAL_SECRET_ACCESS_KEY") @@ -399,7 +372,10 @@ def test_mqtt5_ws_cred_environment(self): os.environ["AWS_SECRET_ACCESS_KEY"] = input_secret_access_key os.environ["AWS_SESSION_TOKEN"] = input_session_token # This should load the environment variables we just set - credentials = auth.AwsCredentialsProvider.new_environment() + if use_default_chain: + credentials = auth.AwsCredentialsProvider.new_default_chain() + else: + credentials = auth.AwsCredentialsProvider.new_environment() def sign_function(transform_args, **kwargs): signing_config = auth.AwsSigningConfig( From 3fc70f21d6572abf3399d50e6e47454cd78071d1 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 15:00:21 -0700 Subject: [PATCH 23/46] fix static --- test/test_mqtt5_credentials.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/test_mqtt5_credentials.py b/test/test_mqtt5_credentials.py index b902e2bbd..8a5d24b09 100644 --- a/test/test_mqtt5_credentials.py +++ b/test/test_mqtt5_credentials.py @@ -197,7 +197,6 @@ def test_mqtt5_ws_cred_static(self): input_role_secret_access_key, input_role_session_token ) - credentials = auth.AwsCredentialsProvider.new_default_chain() def sign_function(transform_args, **kwargs): signing_config = auth.AwsSigningConfig( From dfcafc44c2dc09c804d0bb4039cb68c86b3cc10f Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 15:08:05 -0700 Subject: [PATCH 24/46] fix mqtt311 --- test/test_mqtt_credentials.py | 47 ++++++++--------------------------- 1 file changed, 11 insertions(+), 36 deletions(-) diff --git a/test/test_mqtt_credentials.py b/test/test_mqtt_credentials.py index 15335e3e2..26487b371 100644 --- a/test/test_mqtt_credentials.py +++ b/test/test_mqtt_credentials.py @@ -136,40 +136,6 @@ def sign_function(transform_args, **kwargs): connection.connect().result(TIMEOUT) connection.disconnect().result(TIMEOUT) - def test_mqtt311_ws_cred_default(self): - input_host_name = _get_env_variable("AWS_TEST_MQTT311_IOT_CORE_HOST") - input_region = _get_env_variable("AWS_TEST_MQTT311_IOT_CORE_REGION") - - credentials = auth.AwsCredentialsProvider.new_default_chain() - - def sign_function(transform_args, **kwargs): - signing_config = auth.AwsSigningConfig( - algorithm=auth.AwsSigningAlgorithm.V4, - signature_type=auth.AwsSignatureType.HTTP_REQUEST_QUERY_PARAMS, - credentials_provider=credentials, - region=input_region, - service="iotdevicegateway", - omit_session_token=True - ) - signing_future = auth.aws_sign_request( - http_request=transform_args.http_request, - signing_config=signing_config) - signing_future.add_done_callback(lambda x: transform_args.set_done(x.exception())) - - elg = EventLoopGroup() - resolver = DefaultHostResolver(elg) - bootstrap = ClientBootstrap(elg, resolver) - client = Client(bootstrap, ClientTlsContext(TlsContextOptions())) - connection = Connection( - client=client, - client_id=create_client_id(), - host_name=input_host_name, - port=int(443), - use_websockets=True, - websocket_handshake_transform=sign_function) - connection.connect().result(TIMEOUT) - connection.disconnect().result(TIMEOUT) - def test_mqtt311_ws_cred_cognito(self): input_cognito_endpoint = _get_env_variable("AWS_TEST_MQTT311_COGNITO_ENDPOINT") input_cognito_identity = _get_env_variable("AWS_TEST_MQTT311_COGNITO_IDENTITY") @@ -300,6 +266,12 @@ def sign_function(transform_args, **kwargs): connection.disconnect().result(TIMEOUT) def test_mqtt311_ws_cred_environment(self): + self._test_mqtt311_ws_cred_environment(use_default_chain = False); + + def test_mqtt311_ws_cred_default(self): + self._test_mqtt311_ws_cred_environment(use_default_chain = True); + + def _test_mqtt311_ws_cred_environment(self, use_default_chain): input_access_key = _get_env_variable("AWS_TEST_MQTT311_ROLE_CREDENTIAL_ACCESS_KEY") input_secret_access_key = _get_env_variable("AWS_TEST_MQTT311_ROLE_CREDENTIAL_SECRET_ACCESS_KEY") input_session_token = _get_env_variable("AWS_TEST_MQTT311_ROLE_CREDENTIAL_SESSION_TOKEN") @@ -314,8 +286,11 @@ def test_mqtt311_ws_cred_environment(self): os.environ["AWS_ACCESS_KEY_ID"] = input_access_key os.environ["AWS_SECRET_ACCESS_KEY"] = input_secret_access_key os.environ["AWS_SESSION_TOKEN"] = input_session_token - # This should load the environment variables we just set - credentials = auth.AwsCredentialsProvider.new_environment() + if use_default_chain: + credentials = auth.AwsCredentialsProvider.new_default_chain() + else: + # This should load the environment variables we just set + credentials = auth.AwsCredentialsProvider.new_environment() signing_config = auth.AwsSigningConfig( algorithm=auth.AwsSigningAlgorithm.V4, signature_type=auth.AwsSignatureType.HTTP_REQUEST_QUERY_PARAMS, From c517439a128878d5c51b7883eff4eb855573f3a6 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 15:40:03 -0700 Subject: [PATCH 25/46] fix something --- test/test_mqtt_credentials.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_mqtt_credentials.py b/test/test_mqtt_credentials.py index 26487b371..e1bd2c68d 100644 --- a/test/test_mqtt_credentials.py +++ b/test/test_mqtt_credentials.py @@ -266,10 +266,10 @@ def sign_function(transform_args, **kwargs): connection.disconnect().result(TIMEOUT) def test_mqtt311_ws_cred_environment(self): - self._test_mqtt311_ws_cred_environment(use_default_chain = False); + self._test_mqtt311_ws_cred_environment(use_default_chain=False) def test_mqtt311_ws_cred_default(self): - self._test_mqtt311_ws_cred_environment(use_default_chain = True); + self._test_mqtt311_ws_cred_environment(use_default_chain=True) def _test_mqtt311_ws_cred_environment(self, use_default_chain): input_access_key = _get_env_variable("AWS_TEST_MQTT311_ROLE_CREDENTIAL_ACCESS_KEY") From 267a4b97cd6e09c308c1cf0b411311703ebc0323 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 15:44:16 -0700 Subject: [PATCH 26/46] remove unneeded comment --- .github/workflows/ci.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb7aec11d..ee01f1aa3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,7 +32,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -63,7 +62,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -97,7 +95,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -123,7 +120,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -151,13 +147,11 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: role-to-assume: ${{ env.CRT_CI_ROLE }} aws-region: ${{ env.AWS_DEFAULT_REGION }} - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: Build ${{ env.PACKAGE_NAME }} run: | aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh @@ -181,7 +175,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -200,7 +193,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -234,7 +226,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -251,7 +242,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -268,7 +258,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -291,7 +280,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -320,7 +308,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: @@ -362,7 +349,6 @@ jobs: permissions: id-token: write # This is required for requesting the JWT steps: - # We can't use the `uses: docker://image` version yet, GitHub lacks authentication for actions -> packages - name: configure AWS credentials (containers) uses: aws-actions/configure-aws-credentials@v4 with: From 7801abebeefea2aa5351d02a5b65a5f0f603a401 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Thu, 19 Sep 2024 16:02:31 -0700 Subject: [PATCH 27/46] builder fix --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ee01f1aa3..ec7bb0f60 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,8 +7,8 @@ on: - 'docs' env: - BUILDER_VERSION: test-creds - BUILDER_SOURCE: channels + BUILDER_VERSION: v0.9.67 + BUILDER_SOURCE: releases BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net PACKAGE_NAME: aws-crt-python LINUX_BASE_IMAGE: ubuntu-18-x64 From 2544616055b8d641eda79aa2c93d1ded93c2d216 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 09:12:47 -0700 Subject: [PATCH 28/46] move code duplication to a function --- source/http_stream.c | 14 +---- source/module.c | 20 ++++++ source/module.h | 5 ++ source/mqtt_client_connection.c | 106 ++++---------------------------- 4 files changed, 40 insertions(+), 105 deletions(-) diff --git a/source/http_stream.c b/source/http_stream.c index 5097771eb..cfd6223eb 100644 --- a/source/http_stream.c +++ b/source/http_stream.c @@ -203,19 +203,9 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro } /* DECREF python self, we don't need to force it to stay alive any longer. */ - PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ - int result_getref = PyWeakref_GetRef(stream->self_proxy, &self); /* strong reference */ - /* Ignore error, stream is already complete */ - (void)result_getref; -#else - /* PyWeakref_GetObject is deprecated since python 3.13 */ - self = PyWeakref_GetObject(stream->self_proxy); /* borrowed reference */ -#endif + PyObject *self = aws_py_weakref_get_ref(stream->self_proxy); Py_XDECREF(self); -#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ - Py_XDECREF(self); -#endif + aws_py_weakref_release_ref(self); PyGILState_Release(state); /*************** GIL RELEASE ***************/ diff --git a/source/module.c b/source/module.c index 4230ab6bf..8c7d76288 100644 --- a/source/module.c +++ b/source/module.c @@ -516,6 +516,26 @@ PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf) { return PyMemoryView_FromMemory(mem_start, mem_size, PyBUF_WRITE); } +PyObject *aws_py_weakref_get_ref(PyObject *object) { + PyObject *self = Py_None; +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ + if (PyWeakref_GetRef(object, &self) < 0) { /* strong reference */ + return Py_None; + } +#else + /* PyWeakref_GetObject is deprecated since python 3.13 */ + self = PyWeakref_GetObject(object); /* borrowed reference */ +#endif + return self; +} + +PyObject *aws_py_weakref_release_ref(PyObject *object) { + /* Python versions before 3.13 returns a borrowed reference */ +#if PY_VERSION_HEX >= 0x030D0000 + Py_XDECREF(object); +#endif +} + int aws_py_gilstate_ensure(PyGILState_STATE *out_state) { if (AWS_LIKELY(Py_IsInitialized())) { *out_state = PyGILState_Ensure(); diff --git a/source/module.h b/source/module.h index d7254d8c4..e6930f9f2 100644 --- a/source/module.h +++ b/source/module.h @@ -106,6 +106,11 @@ PyObject *aws_py_get_error_message(PyObject *self, PyObject *args); /* Create a write-only memoryview from the remaining free space in an aws_byte_buf */ PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf); +/* Python 3.13+ changed the function to get a reference from WeakRef. This function is an abstraction over two different methods since we support Python versions before 3.13. */ +PyObject *aws_py_weakref_get_ref(PyObject *object); +/* Release the weakwef provided by the `aws_py_weakref_get_ref`. */ +PyObject *aws_py_weakref_release_ref(PyObject *object); + /* Allocator that calls into PyObject_[Malloc|Free|Realloc] */ struct aws_allocator *aws_py_get_allocator(void); diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index 513604d9a..668d6eed0 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -140,17 +140,7 @@ static void s_on_connection_success( return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ - if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ - PyErr_WriteUnraisable(PyErr_Occurred()); - goto on_done; - } -#else - /* PyWeakref_GetObject is deprecated since python 3.13 */ - self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ -#endif - + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); if (self != Py_None) { PyObject *success_result = PyObject_CallMethod(self, "_on_connection_success", "(iN)", return_code, PyBool_FromLong(session_present)); @@ -160,11 +150,8 @@ static void s_on_connection_success( PyErr_WriteUnraisable(PyErr_Occurred()); } } - goto on_done; /* fixes unused label waring */ -on_done: -#if PY_VERSION_HEX >= 0x030D0000 - Py_XDECREF(self); -#endif + + aws_py_weakref_release_ref(self); PyGILState_Release(state); } @@ -181,17 +168,7 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ - if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ - PyErr_WriteUnraisable(PyErr_Occurred()); - goto on_done; - } -#else - /* PyWeakref_GetObject is deprecated since python 3.13 */ - self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ -#endif - + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); if (self != Py_None) { PyObject *success_result = PyObject_CallMethod(self, "_on_connection_failure", "(i)", error_code); if (success_result) { @@ -201,11 +178,7 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio } } - goto on_done; /* fixes unused label waring */ -on_done: -#if PY_VERSION_HEX >= 0x030D0000 - Py_XDECREF(self); -#endif + aws_py_weakref_release_ref(self); PyGILState_Release(state); } @@ -223,17 +196,7 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } /* Ensure that python class is still alive */ - PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ - if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ - PyErr_WriteUnraisable(PyErr_Occurred()); - goto on_done; - } -#else - /* PyWeakref_GetObject is deprecated since python 3.13 */ - self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ -#endif - + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_interrupted", "(i)", error_code); if (result) { @@ -243,11 +206,7 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } } - goto on_done; /* fixes unused label waring */ -on_done: -#if PY_VERSION_HEX >= 0x030D0000 - Py_XDECREF(self); -#endif + aws_py_weakref_release_ref(self); PyGILState_Release(state); } @@ -271,17 +230,7 @@ static void s_on_connection_resumed( } /* Ensure that python class is still alive */ - PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ - if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ - PyErr_WriteUnraisable(PyErr_Occurred()); - goto on_done; - } -#else - /* PyWeakref_GetObject is deprecated since python 3.13 */ - self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ -#endif - + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_resumed", "(iN)", return_code, PyBool_FromLong(session_present)); @@ -291,11 +240,7 @@ static void s_on_connection_resumed( PyErr_WriteUnraisable(PyErr_Occurred()); } } - goto on_done; /* fixes unused label waring */ -on_done: -#if PY_VERSION_HEX >= 0x030D0000 - Py_XDECREF(self); -#endif + aws_py_weakref_release_ref(self); PyGILState_Release(state); } @@ -316,17 +261,7 @@ static void s_on_connection_closed( struct mqtt_connection_binding *py_connection = userdata; /* Ensure that python class is still alive */ - PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ - if (PyWeakref_GetRef(py_connection->self_proxy, &self) < 0) { /* strong reference */ - PyErr_WriteUnraisable(PyErr_Occurred()); - goto on_done; - } -#else - /* PyWeakref_GetObject is deprecated since python 3.13 */ - self = PyWeakref_GetObject(py_connection->self_proxy); /* borrowed reference */ -#endif - + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); if (self != Py_None) { PyObject *result = PyObject_CallMethod(self, "_on_connection_closed", "()"); if (result) { @@ -336,11 +271,7 @@ static void s_on_connection_closed( } } - goto on_done; /* fixes unused label waring */ -on_done: -#if PY_VERSION_HEX >= 0x030D0000 - Py_XDECREF(self); -#endif + aws_py_weakref_release_ref(self); PyGILState_Release(state); } @@ -608,17 +539,8 @@ static void s_ws_handshake_transform( } /* Ensure python mqtt connection object is still alive */ - PyObject *connection_py = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ - if (PyWeakref_GetRef(connection_binding->self_proxy, &connection_py) < 0) { /* strong reference */ - aws_raise_error(AWS_ERROR_INVALID_STATE); - goto done; - } -#else - /* PyWeakref_GetObject is deprecated since python 3.13 */ - connection_py = PyWeakref_GetObject(connection_binding->self_proxy); /* borrowed reference */ -#endif + PyObject *connection_py = aws_py_weakref_get_ref(connection_binding->self_proxy); if (connection_py == Py_None) { aws_raise_error(AWS_ERROR_INVALID_STATE); goto done; @@ -676,9 +598,7 @@ static void s_ws_handshake_transform( done:; /* Save off error code, so it doesn't got stomped before we pass it to callback*/ int error_code = aws_last_error(); -#if PY_VERSION_HEX >= 0x030D0000 - Py_XDECREF(connection_py); -#endif + aws_py_weakref_release_ref(connection_py); if (ws_transform_capsule) { Py_DECREF(ws_transform_capsule); From 9fa3ac2eac814650e715d28895fe7cdd9ae537f6 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 09:17:54 -0700 Subject: [PATCH 29/46] warning fix --- source/module.c | 3 ++- source/module.h | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/source/module.c b/source/module.c index 8c7d76288..b2a3d21b5 100644 --- a/source/module.c +++ b/source/module.c @@ -518,7 +518,7 @@ PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf) { PyObject *aws_py_weakref_get_ref(PyObject *object) { PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ +#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ if (PyWeakref_GetRef(object, &self) < 0) { /* strong reference */ return Py_None; } @@ -530,6 +530,7 @@ PyObject *aws_py_weakref_get_ref(PyObject *object) { } PyObject *aws_py_weakref_release_ref(PyObject *object) { + (void)object; /* Python versions before 3.13 returns a borrowed reference */ #if PY_VERSION_HEX >= 0x030D0000 Py_XDECREF(object); diff --git a/source/module.h b/source/module.h index e6930f9f2..71181c58d 100644 --- a/source/module.h +++ b/source/module.h @@ -106,7 +106,8 @@ PyObject *aws_py_get_error_message(PyObject *self, PyObject *args); /* Create a write-only memoryview from the remaining free space in an aws_byte_buf */ PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf); -/* Python 3.13+ changed the function to get a reference from WeakRef. This function is an abstraction over two different methods since we support Python versions before 3.13. */ +/* Python 3.13+ changed the function to get a reference from WeakRef. This function is an abstraction over two different + * methods since we support Python versions before 3.13. */ PyObject *aws_py_weakref_get_ref(PyObject *object); /* Release the weakwef provided by the `aws_py_weakref_get_ref`. */ PyObject *aws_py_weakref_release_ref(PyObject *object); From 0b7bd7fc633959a8170c31a78d69932ead80ddd9 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 09:20:57 -0700 Subject: [PATCH 30/46] blank lines --- continuous-delivery/build-wheels-manylinux2014-aarch64.sh | 1 + continuous-delivery/build-wheels-manylinux2014-x86_64.sh | 1 + 2 files changed, 2 insertions(+) diff --git a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh index 3deed7ffa..a8b59efe6 100755 --- a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh @@ -21,6 +21,7 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. + # We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp313*.whl diff --git a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh index 0b66d34e8..0580f04e1 100755 --- a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh @@ -21,6 +21,7 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl # Don't need to build wheels for Python 3.12 and later. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. + # We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. /opt/python/cp313-cp313/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp313*.whl From b3d571e7fd2b7e9de8b94ff7e3803eedf0e06869 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 09:22:15 -0700 Subject: [PATCH 31/46] fix doc --- source/module.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/module.h b/source/module.h index 71181c58d..56d099e81 100644 --- a/source/module.h +++ b/source/module.h @@ -107,7 +107,7 @@ PyObject *aws_py_get_error_message(PyObject *self, PyObject *args); PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf); /* Python 3.13+ changed the function to get a reference from WeakRef. This function is an abstraction over two different - * methods since we support Python versions before 3.13. */ + * APIs since we support Python versions before 3.13. */ PyObject *aws_py_weakref_get_ref(PyObject *object); /* Release the weakwef provided by the `aws_py_weakref_get_ref`. */ PyObject *aws_py_weakref_release_ref(PyObject *object); From 0d591fa9c5d34be76160c2a83ce44d1fed4b30ee Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 09:40:42 -0700 Subject: [PATCH 32/46] fix return type --- source/module.c | 2 +- source/module.h | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/source/module.c b/source/module.c index b2a3d21b5..dc38c30cf 100644 --- a/source/module.c +++ b/source/module.c @@ -529,7 +529,7 @@ PyObject *aws_py_weakref_get_ref(PyObject *object) { return self; } -PyObject *aws_py_weakref_release_ref(PyObject *object) { +void aws_py_weakref_release_ref(PyObject *object) { (void)object; /* Python versions before 3.13 returns a borrowed reference */ #if PY_VERSION_HEX >= 0x030D0000 diff --git a/source/module.h b/source/module.h index 56d099e81..b2cfc8a85 100644 --- a/source/module.h +++ b/source/module.h @@ -107,10 +107,10 @@ PyObject *aws_py_get_error_message(PyObject *self, PyObject *args); PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf); /* Python 3.13+ changed the function to get a reference from WeakRef. This function is an abstraction over two different - * APIs since we support Python versions before 3.13. */ + * APIs since we support Python versions before 3.13.*/ PyObject *aws_py_weakref_get_ref(PyObject *object); /* Release the weakwef provided by the `aws_py_weakref_get_ref`. */ -PyObject *aws_py_weakref_release_ref(PyObject *object); +void aws_py_weakref_release_ref(PyObject *object); /* Allocator that calls into PyObject_[Malloc|Free|Realloc] */ struct aws_allocator *aws_py_get_allocator(void); From 5b960e84adbd98f5944b3a3f6f711249c0f4bba8 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 16:28:41 -0700 Subject: [PATCH 33/46] Update continuous-delivery/build-wheels-manylinux2014-x86_64.sh Co-authored-by: Michael Graeb --- continuous-delivery/build-wheels-manylinux2014-x86_64.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh index 0580f04e1..01ca703d9 100755 --- a/continuous-delivery/build-wheels-manylinux2014-x86_64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-x86_64.sh @@ -19,7 +19,7 @@ auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_x86_64 dist/awscrt-*cp311*.whl -# Don't need to build wheels for Python 3.12 and later. +# Don't need to build wheels for Python 3.12. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. # We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. From ef9ddd15097102d0b43f776810e42ae892eeeaf9 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 16:28:58 -0700 Subject: [PATCH 34/46] Update continuous-delivery/build-wheels-manylinux2014-aarch64.sh Co-authored-by: Michael Graeb --- continuous-delivery/build-wheels-manylinux2014-aarch64.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh index a8b59efe6..62bd8531c 100755 --- a/continuous-delivery/build-wheels-manylinux2014-aarch64.sh +++ b/continuous-delivery/build-wheels-manylinux2014-aarch64.sh @@ -19,7 +19,7 @@ auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat manylinux2014_aarch64 dist/awscrt-*cp311*.whl -# Don't need to build wheels for Python 3.12 and later. +# Don't need to build wheels for Python 3.12. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. # We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. From 9d3c6741f80306275973b3d84d2135a0bbce4d53 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 16:29:06 -0700 Subject: [PATCH 35/46] Update continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh Co-authored-by: Michael Graeb --- continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh index add3a38bc..6f7f26645 100755 --- a/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh +++ b/continuous-delivery/build-wheels-musllinux-1-1-aarch64.sh @@ -19,7 +19,7 @@ auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat musllinux_1_1_aarch64 dist/awscrt-*cp311*.whl -# Don't need to build wheels for Python 3.12 and later. +# Don't need to build wheels for Python 3.12. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. # We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. From d5c0226465b40d8d4db8d5f9e5c2a23809a843e2 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 16:29:14 -0700 Subject: [PATCH 36/46] Update continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh Co-authored-by: Michael Graeb --- continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh index 9f405705f..dccf0eef0 100755 --- a/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh +++ b/continuous-delivery/build-wheels-musllinux-1-1-x86_64.sh @@ -19,7 +19,7 @@ auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp310*.whl /opt/python/cp311-cp311/bin/python setup.py sdist bdist_wheel auditwheel repair --plat musllinux_1_1_x86_64 dist/awscrt-*cp311*.whl -# Don't need to build wheels for Python 3.12 and later. +# Don't need to build wheels for Python 3.12. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. # We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. From bfb523b8dac4e204f26ed3b0f1e28e7ce2cbad19 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 16:29:21 -0700 Subject: [PATCH 37/46] Update continuous-delivery/build-wheels-osx.sh Co-authored-by: Michael Graeb --- continuous-delivery/build-wheels-osx.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous-delivery/build-wheels-osx.sh b/continuous-delivery/build-wheels-osx.sh index 802c5d349..a9feb731d 100755 --- a/continuous-delivery/build-wheels-osx.sh +++ b/continuous-delivery/build-wheels-osx.sh @@ -11,7 +11,7 @@ set -ex /Library/Frameworks/Python.framework/Versions/3.10/bin/python3 setup.py sdist bdist_wheel /Library/Frameworks/Python.framework/Versions/3.11/bin/python3 setup.py sdist bdist_wheel -# Don't need to build wheels for Python 3.12 and later. +# Don't need to build wheels for Python 3.12. # The 3.11 wheel uses the stable ABI, so it works with newer versions too. # We are using the Python 3.13 stable ABI from Python 3.13 onwards because of deprecated functions. From 634cccf6c0b3e476db02e432d25028881526ac72 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 16:29:36 -0700 Subject: [PATCH 38/46] Update setup.py Co-authored-by: Michael Graeb --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index a14239bd1..c3f46adf1 100644 --- a/setup.py +++ b/setup.py @@ -404,6 +404,7 @@ def awscrt_ext(): extra_link_args += ['-Wl,--fatal-warnings'] if sys.version_info >= (3, 13): + # 3.13 deprecates PyWeakref_GetObject(), adds alternative define_macros.append(('Py_LIMITED_API', '0x030D0000')) py_limited_api = True elif sys.version_info >= (3, 11): From fdf644ce38862528f844e4ad837c35e3b1362eeb Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 16:29:48 -0700 Subject: [PATCH 39/46] Update setup.py Co-authored-by: Michael Graeb --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index c3f46adf1..7ebd2dc12 100644 --- a/setup.py +++ b/setup.py @@ -403,6 +403,7 @@ def awscrt_ext(): else: extra_link_args += ['-Wl,--fatal-warnings'] + # prefer building with stable ABI, so a wheel can work with multiple major versions if sys.version_info >= (3, 13): # 3.13 deprecates PyWeakref_GetObject(), adds alternative define_macros.append(('Py_LIMITED_API', '0x030D0000')) From 80de4fc351c2ad82e7d42206c038de0bd4ee85e9 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 16:29:56 -0700 Subject: [PATCH 40/46] Update setup.py Co-authored-by: Michael Graeb --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 7ebd2dc12..50393a8b6 100644 --- a/setup.py +++ b/setup.py @@ -409,6 +409,7 @@ def awscrt_ext(): define_macros.append(('Py_LIMITED_API', '0x030D0000')) py_limited_api = True elif sys.version_info >= (3, 11): + # 3.11 is the first stable ABI that has everything we need define_macros.append(('Py_LIMITED_API', '0x030B0000')) py_limited_api = True From 171843e85e4e19ea27f20733e997ab04e7b3ced7 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 16:30:08 -0700 Subject: [PATCH 41/46] Update continuous-delivery/build-wheels-win32.bat Co-authored-by: Michael Graeb --- continuous-delivery/build-wheels-win32.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous-delivery/build-wheels-win32.bat b/continuous-delivery/build-wheels-win32.bat index 52b82e49b..d0f2576e3 100644 --- a/continuous-delivery/build-wheels-win32.bat +++ b/continuous-delivery/build-wheels-win32.bat @@ -7,7 +7,7 @@ "C:\Program Files (x86)\Python310-32\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files (x86)\Python311-32\python.exe" setup.py sdist bdist_wheel || goto error -:: Don't need to build wheels for Python 3.12 and later. +:: Don't need to build wheels for Python 3.12. :: The 3.11 wheel uses the stable ABI, so it works with newer versions too. :: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. From 7848649549553272958fcab0122475083bedebc5 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 16:30:18 -0700 Subject: [PATCH 42/46] Update continuous-delivery/build-wheels-win64.bat Co-authored-by: Michael Graeb --- continuous-delivery/build-wheels-win64.bat | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous-delivery/build-wheels-win64.bat b/continuous-delivery/build-wheels-win64.bat index 492fe204e..fd94419e5 100644 --- a/continuous-delivery/build-wheels-win64.bat +++ b/continuous-delivery/build-wheels-win64.bat @@ -6,7 +6,7 @@ "C:\Program Files\Python310\python.exe" setup.py sdist bdist_wheel || goto error "C:\Program Files\Python311\python.exe" setup.py sdist bdist_wheel || goto error -:: Don't need to build wheels for Python 3.12 and later. +:: Don't need to build wheels for Python 3.12. :: The 3.11 wheel uses the stable ABI, so it works with newer versions too. :: We are using the 3.13 stable ABI from 3.13 onwards because of deprecated functions. From 76df344faf8021d2d7b14eb38b15aab5b975856c Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Mon, 23 Sep 2024 16:38:06 -0700 Subject: [PATCH 43/46] do incref for pre 3.13 --- source/http_stream.c | 3 ++- source/module.c | 9 +-------- source/module.h | 4 +--- source/mqtt_client_connection.c | 12 ++++++------ 4 files changed, 10 insertions(+), 18 deletions(-) diff --git a/source/http_stream.c b/source/http_stream.c index cfd6223eb..6843e0ea8 100644 --- a/source/http_stream.c +++ b/source/http_stream.c @@ -204,8 +204,9 @@ static void s_on_stream_complete(struct aws_http_stream *native_stream, int erro /* DECREF python self, we don't need to force it to stay alive any longer. */ PyObject *self = aws_py_weakref_get_ref(stream->self_proxy); + /* DECREF twice because `aws_py_weakref_get_ref` returns a strong reference */ + Py_XDECREF(self); Py_XDECREF(self); - aws_py_weakref_release_ref(self); PyGILState_Release(state); /*************** GIL RELEASE ***************/ diff --git a/source/module.c b/source/module.c index dc38c30cf..23d77cda9 100644 --- a/source/module.c +++ b/source/module.c @@ -525,18 +525,11 @@ PyObject *aws_py_weakref_get_ref(PyObject *object) { #else /* PyWeakref_GetObject is deprecated since python 3.13 */ self = PyWeakref_GetObject(object); /* borrowed reference */ + Py_XINCREF(self); #endif return self; } -void aws_py_weakref_release_ref(PyObject *object) { - (void)object; - /* Python versions before 3.13 returns a borrowed reference */ -#if PY_VERSION_HEX >= 0x030D0000 - Py_XDECREF(object); -#endif -} - int aws_py_gilstate_ensure(PyGILState_STATE *out_state) { if (AWS_LIKELY(Py_IsInitialized())) { *out_state = PyGILState_Ensure(); diff --git a/source/module.h b/source/module.h index b2cfc8a85..7777ab8bc 100644 --- a/source/module.h +++ b/source/module.h @@ -107,10 +107,8 @@ PyObject *aws_py_get_error_message(PyObject *self, PyObject *args); PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf); /* Python 3.13+ changed the function to get a reference from WeakRef. This function is an abstraction over two different - * APIs since we support Python versions before 3.13.*/ + * APIs since we support Python versions before 3.13. Returns a strong reference if non-null, which you must release. */ PyObject *aws_py_weakref_get_ref(PyObject *object); -/* Release the weakwef provided by the `aws_py_weakref_get_ref`. */ -void aws_py_weakref_release_ref(PyObject *object); /* Allocator that calls into PyObject_[Malloc|Free|Realloc] */ struct aws_allocator *aws_py_get_allocator(void); diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index 668d6eed0..fe629a959 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -151,7 +151,7 @@ static void s_on_connection_success( } } - aws_py_weakref_release_ref(self); + Py_XDECREF(self); PyGILState_Release(state); } @@ -178,7 +178,7 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio } } - aws_py_weakref_release_ref(self); + Py_XDECREF(self); PyGILState_Release(state); } @@ -206,7 +206,7 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } } - aws_py_weakref_release_ref(self); + Py_XDECREF(self); PyGILState_Release(state); } @@ -240,7 +240,7 @@ static void s_on_connection_resumed( PyErr_WriteUnraisable(PyErr_Occurred()); } } - aws_py_weakref_release_ref(self); + Py_XDECREF(self); PyGILState_Release(state); } @@ -271,7 +271,7 @@ static void s_on_connection_closed( } } - aws_py_weakref_release_ref(self); + Py_XDECREF(self); PyGILState_Release(state); } @@ -598,7 +598,7 @@ static void s_ws_handshake_transform( done:; /* Save off error code, so it doesn't got stomped before we pass it to callback*/ int error_code = aws_last_error(); - aws_py_weakref_release_ref(connection_py); + Py_XDECREF(connection_py); if (ws_transform_capsule) { Py_DECREF(ws_transform_capsule); From 4b68a143ce1395e85138b208fded8e773572300d Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Tue, 24 Sep 2024 16:00:04 -0700 Subject: [PATCH 44/46] Return NULL instead of Py_NONE (so it works better with Py_XDECREF) AWS_FATAL_ASSERT if these functions raise a python exception (which we never checked for before) --- source/module.c | 39 ++++++++++++++++++++++++--------- source/module.h | 18 ++++++++++++++- source/mqtt_client_connection.c | 35 +++++++++++++++-------------- 3 files changed, 64 insertions(+), 28 deletions(-) diff --git a/source/module.c b/source/module.c index 23d77cda9..7aa0502b1 100644 --- a/source/module.c +++ b/source/module.c @@ -516,18 +516,37 @@ PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf) { return PyMemoryView_FromMemory(mem_start, mem_size, PyBUF_WRITE); } -PyObject *aws_py_weakref_get_ref(PyObject *object) { - PyObject *self = Py_None; -#if PY_VERSION_HEX >= 0x030D0000 /* Check if Python version is 3.13 or higher */ - if (PyWeakref_GetRef(object, &self) < 0) { /* strong reference */ - return Py_None; - } +PyObject *aws_py_weakref_get_ref(PyObject *ref) { + /* If Python >= 3.13 */ +#if PY_VERSION_HEX >= 0x030D0000 + /* Use PyWeakref_GetRef() (new in Python 3.13), which gets you: + /* a new strong reference, + * or NULL because ref is dead, + * or -1 because you called it wrong */ + PyObject *obj = NULL; + if (PyWeakref_GetRef(ref, &obj) == -1) { + PyErr_WriteUnraisable(PyErr_Occurred()); + AWS_FATAL_ASSERT(0 && "expected a weakref"); + } + return obj; + #else - /* PyWeakref_GetObject is deprecated since python 3.13 */ - self = PyWeakref_GetObject(object); /* borrowed reference */ - Py_XINCREF(self); + /* Use PyWeakref_GetObject() (deprecated as of Python 3.13), which gets you: + * a borrowed reference, + * or Py_None because ref is dead, + * or NULL because you called it wrong */ + PyObject *obj = PyWeakref_GetObject(ref); /* borrowed reference */ + if (obj == NULL) { + PyErr_WriteUnraisable(PyErr_Occurred()); + AWS_FATAL_ASSERT(0 && "expected a weakref"); + } else if (obj == Py_None) { + return NULL; + } else { + /* Be like PyWeakref_GetRef() and make it new strong reference */ + Py_INCREF(obj); + return obj; + } #endif - return self; } int aws_py_gilstate_ensure(PyGILState_STATE *out_state) { diff --git a/source/module.h b/source/module.h index 7777ab8bc..ca73f2a5b 100644 --- a/source/module.h +++ b/source/module.h @@ -108,7 +108,23 @@ PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf); /* Python 3.13+ changed the function to get a reference from WeakRef. This function is an abstraction over two different * APIs since we support Python versions before 3.13. Returns a strong reference if non-null, which you must release. */ -PyObject *aws_py_weakref_get_ref(PyObject *object); + +/** + * Given a weak reference, returns a NEW strong reference to the referenced object, + * or NULL if the reference is dead (this function NEVER raises a python exception or AWS Error). + * + * This is a simplified version of PyWeakref_GetRef() / PyWeakref_GetObject(). + * Simpler because: + * - Python 3.13 adds PyWeakref_GetRef() and deprecates PyWeakref_GetObject(). + * This function calls the appropriate one. + * + * - This will AWS_FATAL_ASSERT if ref is not a weak reference, + * So you only need to handle 2 outcomes instead of 3 + * (the 3rd being a Python exception for calling it incorrectly). + * But this means it's only safe to call if we created the ref ourselves. + * Do not call if ref could have come from the user. + */ +PyObject *aws_py_weakref_get_ref(PyObject *ref); /* Allocator that calls into PyObject_[Malloc|Free|Realloc] */ struct aws_allocator *aws_py_get_allocator(void); diff --git a/source/mqtt_client_connection.c b/source/mqtt_client_connection.c index fe629a959..78a26057e 100644 --- a/source/mqtt_client_connection.c +++ b/source/mqtt_client_connection.c @@ -140,8 +140,8 @@ static void s_on_connection_success( return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); - if (self != Py_None) { + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */ + if (self != NULL) { PyObject *success_result = PyObject_CallMethod(self, "_on_connection_success", "(iN)", return_code, PyBool_FromLong(session_present)); if (success_result) { @@ -149,9 +149,9 @@ static void s_on_connection_success( } else { PyErr_WriteUnraisable(PyErr_Occurred()); } + Py_DECREF(self); } - Py_XDECREF(self); PyGILState_Release(state); } @@ -168,17 +168,17 @@ static void s_on_connection_failure(struct aws_mqtt_client_connection *connectio return; /* Python has shut down. Nothing matters anymore, but don't crash */ } - PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); - if (self != Py_None) { + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */ + if (self != NULL) { PyObject *success_result = PyObject_CallMethod(self, "_on_connection_failure", "(i)", error_code); if (success_result) { Py_DECREF(success_result); } else { PyErr_WriteUnraisable(PyErr_Occurred()); } + Py_DECREF(self); } - Py_XDECREF(self); PyGILState_Release(state); } @@ -196,17 +196,17 @@ static void s_on_connection_interrupted(struct aws_mqtt_client_connection *conne } /* Ensure that python class is still alive */ - PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); - if (self != Py_None) { + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */ + if (self != NULL) { PyObject *result = PyObject_CallMethod(self, "_on_connection_interrupted", "(i)", error_code); if (result) { Py_DECREF(result); } else { PyErr_WriteUnraisable(PyErr_Occurred()); } + Py_DECREF(self); } - Py_XDECREF(self); PyGILState_Release(state); } @@ -230,8 +230,8 @@ static void s_on_connection_resumed( } /* Ensure that python class is still alive */ - PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); - if (self != Py_None) { + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */ + if (self != NULL) { PyObject *result = PyObject_CallMethod(self, "_on_connection_resumed", "(iN)", return_code, PyBool_FromLong(session_present)); if (result) { @@ -239,8 +239,9 @@ static void s_on_connection_resumed( } else { PyErr_WriteUnraisable(PyErr_Occurred()); } + Py_DECREF(self); } - Py_XDECREF(self); + PyGILState_Release(state); } @@ -261,17 +262,17 @@ static void s_on_connection_closed( struct mqtt_connection_binding *py_connection = userdata; /* Ensure that python class is still alive */ - PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); - if (self != Py_None) { + PyObject *self = aws_py_weakref_get_ref(py_connection->self_proxy); /* new reference */ + if (self != NULL) { PyObject *result = PyObject_CallMethod(self, "_on_connection_closed", "()"); if (result) { Py_DECREF(result); } else { PyErr_WriteUnraisable(PyErr_Occurred()); } + Py_DECREF(self); } - Py_XDECREF(self); PyGILState_Release(state); } @@ -540,8 +541,8 @@ static void s_ws_handshake_transform( /* Ensure python mqtt connection object is still alive */ - PyObject *connection_py = aws_py_weakref_get_ref(connection_binding->self_proxy); - if (connection_py == Py_None) { + PyObject *connection_py = aws_py_weakref_get_ref(connection_binding->self_proxy); /* new reference */ + if (connection_py == NULL) { aws_raise_error(AWS_ERROR_INVALID_STATE); goto done; } From af82781b287ac3ed9f169f6e6302daf0aa6a7550 Mon Sep 17 00:00:00 2001 From: Michael Graeb Date: Tue, 24 Sep 2024 16:17:36 -0700 Subject: [PATCH 45/46] remove AWS_FATAL_ASSERT I'm worried about introducing crashes --- source/module.c | 8 ++++---- source/module.h | 11 ++++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/source/module.c b/source/module.c index 7aa0502b1..ff5bb4f28 100644 --- a/source/module.c +++ b/source/module.c @@ -526,7 +526,7 @@ PyObject *aws_py_weakref_get_ref(PyObject *ref) { PyObject *obj = NULL; if (PyWeakref_GetRef(ref, &obj) == -1) { PyErr_WriteUnraisable(PyErr_Occurred()); - AWS_FATAL_ASSERT(0 && "expected a weakref"); + AWS_ASSERT(0 && "expected a weakref"); } return obj; @@ -538,14 +538,14 @@ PyObject *aws_py_weakref_get_ref(PyObject *ref) { PyObject *obj = PyWeakref_GetObject(ref); /* borrowed reference */ if (obj == NULL) { PyErr_WriteUnraisable(PyErr_Occurred()); - AWS_FATAL_ASSERT(0 && "expected a weakref"); + AWS_ASSERT(0 && "expected a weakref"); } else if (obj == Py_None) { - return NULL; + obj = NULL; } else { /* Be like PyWeakref_GetRef() and make it new strong reference */ Py_INCREF(obj); - return obj; } + return obj; #endif } diff --git a/source/module.h b/source/module.h index ca73f2a5b..f626cb1e5 100644 --- a/source/module.h +++ b/source/module.h @@ -113,16 +113,17 @@ PyObject *aws_py_memory_view_from_byte_buffer(struct aws_byte_buf *buf); * Given a weak reference, returns a NEW strong reference to the referenced object, * or NULL if the reference is dead (this function NEVER raises a python exception or AWS Error). * + * You MUST NOT call this if ref came from a user, or ref is NULL. + * * This is a simplified version of PyWeakref_GetRef() / PyWeakref_GetObject(). * Simpler because: * - Python 3.13 adds PyWeakref_GetRef() and deprecates PyWeakref_GetObject(). * This function calls the appropriate one. * - * - This will AWS_FATAL_ASSERT if ref is not a weak reference, - * So you only need to handle 2 outcomes instead of 3 - * (the 3rd being a Python exception for calling it incorrectly). - * But this means it's only safe to call if we created the ref ourselves. - * Do not call if ref could have come from the user. + * - This functions has 2 outcomes instead of 3: + * The 3rd being a Python exception for calling it incorrectly. + * If that happens, this function calls PyErr_WriteUnraisable() to clear the exception, + * which is what you would have done anyway. */ PyObject *aws_py_weakref_get_ref(PyObject *ref); From 9277ac232e55394831ca0ec14ca0a9b4d4ad8539 Mon Sep 17 00:00:00 2001 From: Waqar Ahmed Khan Date: Tue, 24 Sep 2024 16:22:13 -0700 Subject: [PATCH 46/46] fix warning --- source/module.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/source/module.c b/source/module.c index ff5bb4f28..197ce20f1 100644 --- a/source/module.c +++ b/source/module.c @@ -520,7 +520,7 @@ PyObject *aws_py_weakref_get_ref(PyObject *ref) { /* If Python >= 3.13 */ #if PY_VERSION_HEX >= 0x030D0000 /* Use PyWeakref_GetRef() (new in Python 3.13), which gets you: - /* a new strong reference, + * a new strong reference, * or NULL because ref is dead, * or -1 because you called it wrong */ PyObject *obj = NULL;