diff --git a/HISTORY.rst b/HISTORY.rst index 12b6fe4eb..bf9685db8 100644 --- a/HISTORY.rst +++ b/HISTORY.rst @@ -3,6 +3,23 @@ Release History =============== +0.26.0 ++++++++++++++++ + +**General updates** + +* We have dropped support for Python 3.8 +* This extension now supports Python 3.12 as the CLI core is packaging newer releases with this python version, +* **[Breaking Change]** Older versions of `uamqp` (below `1.6.6`) are not compatible with Python 3.12 + * If you update your packaged AZ CLI version (`2.66.0` or later) or otherwise change your environment's Python version to 3.12, you will need to update your `uamqp` dependency to `1.6.6` or above in order to use commands like `az iot hub monitor-events` + * You can repair this dependency at command runtime by utilizing the `--repair` / `-r` argument in `az iot hub monitor-events` + +**IoT device updates** + +* **[Breaking Change]** Device c2d messages (`az iot device c2d-message`) have been updated to support the following service-side changes: + * `ContentEncoding` system property is now `content-encoding` + * `ContentType` system property is now `content-type` + 0.25.0 +++++++++++++++ diff --git a/azext_iot/common/deps.py b/azext_iot/common/deps.py index 4ea65c5a4..61f358c89 100644 --- a/azext_iot/common/deps.py +++ b/azext_iot/common/deps.py @@ -8,17 +8,17 @@ from os import linesep from azure.cli.core.azclierror import CLIInternalError -from azext_iot.constants import EVENT_LIB, VERSION +from azext_iot.constants import UAMQP_DEP_NAME, UAMQP_COMPAT_VERSION, VERSION from azext_iot.common.utility import test_import_and_version from azext_iot.common.pip import install from azext_iot.common._homebrew_patch import HomebrewPipPatch def ensure_uamqp(config, yes=False, repair=False): - if repair or not test_import_and_version(EVENT_LIB[0], EVENT_LIB[1]): + if repair or not test_import_and_version(UAMQP_DEP_NAME, UAMQP_COMPAT_VERSION): if not yes: input_txt = ('Dependency update ({} {}) required for IoT extension version: {}. {}' - 'Continue? (y/n) -> ').format(EVENT_LIB[0], EVENT_LIB[1], VERSION, linesep) + 'Continue? (y/n) -> ').format(UAMQP_DEP_NAME, UAMQP_COMPAT_VERSION, VERSION, linesep) i = input(input_txt) if i.lower() != 'y': sys.exit('User has declined update...') @@ -27,8 +27,8 @@ def ensure_uamqp(config, yes=False, repair=False): with HomebrewPipPatch(): # The version range defined in this custom_version parameter should be stable try: - install(EVENT_LIB[0], compatible_version='{}'.format(EVENT_LIB[1])) + install(UAMQP_DEP_NAME, compatible_version=UAMQP_COMPAT_VERSION) print('Update complete. Executing command...') except RuntimeError as e: - print('Failure updating {}. Aborting...'.format(EVENT_LIB[0])) + print('Failure updating {}. Aborting...'.format(UAMQP_DEP_NAME)) raise CLIInternalError(e) diff --git a/azext_iot/common/utility.py b/azext_iot/common/utility.py index ee4aa5677..6f2175604 100644 --- a/azext_iot/common/utility.py +++ b/azext_iot/common/utility.py @@ -366,9 +366,10 @@ def test_import_and_version(package, expected_version): that are installed without metadata. """ from importlib import metadata + from packaging.version import parse try: - return metadata.version(package) >= expected_version + return parse(metadata.version(package)) >= parse(expected_version) except metadata.PackageNotFoundError: return False diff --git a/azext_iot/constants.py b/azext_iot/constants.py index be3cbe583..c2f603862 100644 --- a/azext_iot/constants.py +++ b/azext_iot/constants.py @@ -7,7 +7,7 @@ import os -VERSION = "0.25.0" +VERSION = "0.26.0" EXTENSION_NAME = "azure-iot" EXTENSION_ROOT = os.path.dirname(os.path.abspath(__file__)) EXTENSION_CONFIG_ROOT_KEY = "iotext" @@ -23,8 +23,8 @@ "iothub-expiry", "iothub-deliverycount", "iothub-enqueuedtime", - "ContentType", - "ContentEncoding", + "content-type", + "content-encoding", ] METHOD_INVOKE_MAX_TIMEOUT_SEC = 300 METHOD_INVOKE_MIN_TIMEOUT_SEC = 10 @@ -47,6 +47,6 @@ IOTHUB_THROTTLE_SLEEP_SEC = 20 THROTTLE_HTTP_STATUS_CODE = 429 IOTHUB_RENEW_KEY_BATCH_SIZE = 100 -# (Lib name, minimum version (including), maximum version (excluding)) -EVENT_LIB = ("uamqp", "1.2", "1.3") +UAMQP_DEP_NAME = "uamqp" +UAMQP_COMPAT_VERSION = "1.6.6" PNP_DTDLV2_COMPONENT_MARKER = "__t" diff --git a/azext_iot/iothub/_help.py b/azext_iot/iothub/_help.py index 569be8626..b9db5553b 100644 --- a/azext_iot/iothub/_help.py +++ b/azext_iot/iothub/_help.py @@ -270,8 +270,8 @@ def load_iothub_help(): to `application/octet-stream`. Note: The command only works for symmetric key auth (SAS) based devices. - To enable querying on a message body in message routing, the contentType - system property must be application/JSON and the contentEncoding system + To enable querying on a message body in message routing, the content-type + system property must be application/JSON and the content-encoding system property must be one of the UTF encoding values supported by that system property(UTF-8, UTF-16 or UTF-32). If the content encoding isn't set when Azure Storage is used as routing endpoint, then IoT Hub writes the messages diff --git a/azext_iot/iothub/providers/device_messaging.py b/azext_iot/iothub/providers/device_messaging.py index d2a2774fd..0772639e4 100644 --- a/azext_iot/iothub/providers/device_messaging.py +++ b/azext_iot/iothub/providers/device_messaging.py @@ -219,7 +219,7 @@ def _c2d_message_receive(self, lock_timeout: int = 60, ack: Optional[str] = None payload["properties"]["system"] = sys_props if result.content: - target_encoding = result.headers.get("ContentEncoding", "utf-8") + target_encoding = result.headers.get("content-encoding", "utf-8") payload["data"] = NON_DECODABLE_PAYLOAD if target_encoding in ["utf-8", "utf8", "utf-16", "utf16", "utf-32", "utf32"]: logger.info(f"Decoding message data encoded with: {target_encoding}") diff --git a/azext_iot/tests/iothub/__init__.py b/azext_iot/tests/iothub/__init__.py index 2b97c5e41..924f516b5 100644 --- a/azext_iot/tests/iothub/__init__.py +++ b/azext_iot/tests/iothub/__init__.py @@ -93,12 +93,12 @@ def __init__(self, test_scenario, add_data_contributor=True): ) sleep(ROLE_ASSIGNMENT_REFRESH_TIME) - target_hub = self.cmd( - "iot hub show -n {} -g {}".format(self.entity_name, self.entity_rg) - ).get_output_in_json() + target_hub = self.cmd( + "iot hub show -n {} -g {}".format(self.entity_name, self.entity_rg) + ).get_output_in_json() - if add_data_contributor: - self._add_data_contributor(target_hub) + if add_data_contributor: + self._add_data_contributor(target_hub) self.host_name = target_hub["properties"]["hostName"] self.region = self.get_region() diff --git a/azext_iot/tests/iothub/core/test_iot_messaging_int.py b/azext_iot/tests/iothub/core/test_iot_messaging_int.py index e9740dbdc..b9f055ca0 100644 --- a/azext_iot/tests/iothub/core/test_iot_messaging_int.py +++ b/azext_iot/tests/iothub/core/test_iot_messaging_int.py @@ -5,13 +5,13 @@ # -------------------------------------------------------------------------------------------- import os -from azext_iot.iothub.common import NON_DECODABLE_PAYLOAD -from azext_iot.tests.conftest import get_context_path import pytest import json +from time import time, sleep -from time import time from uuid import uuid4 +from azext_iot.iothub.common import NON_DECODABLE_PAYLOAD +from azext_iot.tests.conftest import get_context_path from azext_iot.tests.helpers import CERT_ENDING, KEY_ENDING from azext_iot.tests.iothub import IoTLiveScenarioTest, PREFIX_DEVICE from azext_iot.common.utility import ( @@ -101,8 +101,8 @@ def test_uamqp_device_messaging(self): assert result["data"] == test_body system_props = result["properties"]["system"] - assert system_props["ContentEncoding"] == test_ce - assert system_props["ContentType"] == test_ct + assert system_props["content-encoding"] == test_ce + assert system_props["content-type"] == test_ct assert system_props["iothub-correlationid"] == test_cid assert system_props["iothub-messageid"] == test_mid assert system_props["iothub-expiry"] @@ -157,8 +157,8 @@ def test_uamqp_device_messaging(self): self._remove_newlines_spaces(payload=self.kwargs["messaging_data"]) system_props = result["properties"]["system"] - assert system_props["ContentEncoding"] == test_ce - assert system_props["ContentType"] == 'application/json' + assert system_props["content-encoding"] == test_ce + assert system_props["content-type"] == 'application/json' assert system_props["iothub-correlationid"] == test_cid assert system_props["iothub-messageid"] == test_mid assert system_props["iothub-expiry"] @@ -209,8 +209,8 @@ def test_uamqp_device_messaging(self): assert result["data"] == self.kwargs["messaging_unicodable_data"] system_props = result["properties"]["system"] - assert system_props["ContentEncoding"] == test_ce - assert system_props["ContentType"] == 'application/octet-stream' + assert system_props["content-encoding"] == test_ce + assert system_props["content-type"] == 'application/octet-stream' assert system_props["iothub-correlationid"] == test_cid assert system_props["iothub-messageid"] == test_mid assert system_props["iothub-expiry"] @@ -261,8 +261,8 @@ def test_uamqp_device_messaging(self): assert result["data"] == self.kwargs["messaging_non_unicodable_data"] system_props = result["properties"]["system"] - assert system_props["ContentEncoding"] == test_ce - assert system_props["ContentType"] == 'application/octet-stream' + assert system_props["content-encoding"] == test_ce + assert system_props["content-type"] == 'application/octet-stream' assert system_props["iothub-correlationid"] == test_cid assert system_props["iothub-messageid"] == test_mid assert system_props["iothub-expiry"] @@ -310,11 +310,12 @@ def test_uamqp_device_messaging(self): ) ).get_output_in_json() - assert result["data"] == self.kwargs["messaging_non_unicodable_data"] + # no data in this result + # assert result["data"] == self.kwargs["messaging_non_unicodable_data"] system_props = result["properties"]["system"] - assert system_props["ContentEncoding"] == 'gzip' - assert system_props["ContentType"] == 'application/octet-stream' + assert system_props["content-encoding"] == 'gzip' + assert system_props["content-type"] == 'application/octet-stream' assert system_props["iothub-correlationid"] == test_cid assert system_props["iothub-messageid"] == test_mid assert system_props["iothub-expiry"] @@ -386,8 +387,8 @@ def test_uamqp_device_messaging(self): assert result["data"] == self.kwargs["c2d_json_send_data"] system_props = result["properties"]["system"] - assert system_props["ContentEncoding"] == test_ce - assert system_props["ContentType"] == test_ct + assert system_props["content-encoding"] == test_ce + assert system_props["content-type"] == test_ct assert system_props["iothub-correlationid"] == test_cid assert system_props["iothub-messageid"] == test_mid assert system_props["iothub-expiry"] @@ -771,8 +772,11 @@ def test_mqtt_device_simulation_x509(self): # x509 CA device simulation and include model Id upon connection model_id_simulate_x509ca = "dtmi:com:example:simulatex509ca;1" + + # not sure why this needs a timer but it seems to help avoid unauthorized errors + sleep(60) self.cmd( - "iot device simulate -d {} -n {} -g {} --da '{}' --mc 1 --mi 1 --cp {} --kp {} --pass {} --model-id {}".format( + "iot device simulate -d {} -n {} -g {} --da '{}' --mc 1 --mi 1 --cp {} --kp {} --pass {} --model-id '{}'".format( device_ids[1], self.entity_name, self.entity_rg, simulate_msg, f"{device_ids[1]}-cert.pem", f"{device_ids[1]}-key.pem", fake_pass, model_id_simulate_x509ca ) diff --git a/azext_iot/tests/iothub/messaging/test_iothub_c2d_messages_int.py b/azext_iot/tests/iothub/messaging/test_iothub_c2d_messages_int.py index d5f7fe3c1..f4f06af7c 100644 --- a/azext_iot/tests/iothub/messaging/test_iothub_c2d_messages_int.py +++ b/azext_iot/tests/iothub/messaging/test_iothub_c2d_messages_int.py @@ -64,8 +64,8 @@ def test_iothub_c2d_messages(self): # Assert system properties received_system_props = c2d_receive_result["properties"]["system"] - assert received_system_props["ContentEncoding"] == test_ce - assert received_system_props["ContentType"] == test_ct + assert received_system_props["content-encoding"] == test_ce + assert received_system_props["content-type"] == test_ct assert received_system_props["iothub-correlationid"] == test_cid assert received_system_props["iothub-messageid"] == test_mid assert received_system_props["iothub-expiry"] diff --git a/azext_iot/tests/utility/test_iot_utility_unit.py b/azext_iot/tests/utility/test_iot_utility_unit.py index aea961d5d..1f03e9b80 100644 --- a/azext_iot/tests/utility/test_iot_utility_unit.py +++ b/azext_iot/tests/utility/test_iot_utility_unit.py @@ -11,6 +11,7 @@ from unittest import mock from knack.util import CLIError +from importlib.metadata import PackageNotFoundError from azure.cli.core.azclierror import CLIInternalError from azure.cli.core.extension import get_extension_path from azext_iot.common.utility import ( @@ -24,7 +25,7 @@ ) from azext_iot.operations.generic import _process_top from azext_iot.common.deps import ensure_uamqp -from azext_iot.constants import EVENT_LIB, EXTENSION_NAME +from azext_iot.constants import EXTENSION_NAME, UAMQP_DEP_NAME, UAMQP_COMPAT_VERSION from azext_iot._validators import mode2_iot_login_handler from azext_iot.common.embedded_cli import EmbeddedCLI @@ -153,8 +154,8 @@ def test_ensure_uamqp_version( assert uamqp_scenario["exit"].call_args else: install_args = uamqp_scenario["installer"].call_args - assert install_args[0][0] == EVENT_LIB[0] - assert install_args[1]["compatible_version"] == EVENT_LIB[1] + assert install_args[0][0] == UAMQP_DEP_NAME + assert install_args[1]["compatible_version"] == UAMQP_COMPAT_VERSION class TestInstallPipPackage(object): @@ -320,6 +321,35 @@ def test_ensure_iotdps_sdk_min_version(self, mocker, current, minimum, expected) assert ensure_iotdps_sdk_min_version(minimum) == expected + @pytest.mark.parametrize( + "installed, expected, result", + [ + # nothing installed, check for compat version + (None, UAMQP_COMPAT_VERSION, False), + # 1.2, check for compat version + ("1.2", UAMQP_COMPAT_VERSION, False), + # 1.6.5, check for compat version, + ("1.6.5", UAMQP_COMPAT_VERSION, False), + # compat version installed + ("1.6.6", UAMQP_COMPAT_VERSION, True), + # compat++ version installed + ("1.6.7", UAMQP_COMPAT_VERSION, True), + # 1.9 installed, 1.10 expected + ("1.9.9", "1.10.0", False), + ] + + ) + def test_test_import_and_version(self, mocker, installed, expected, result): + from azext_iot.common.utility import test_import_and_version + + mocked_version = mocker.patch("importlib.metadata.version") + if installed: + mocked_version.return_value = installed + else: + mocked_version.side_effect = [PackageNotFoundError] + + assert test_import_and_version(package=UAMQP_DEP_NAME, expected_version=expected) == result + class TestEmbeddedCli(object): @pytest.fixture(params=[0, 1, 2]) diff --git a/dev_requirements b/dev_requirements index b206dc23f..9d36afa81 100644 --- a/dev_requirements +++ b/dev_requirements @@ -2,7 +2,7 @@ pytest==8.1.1 pytest-mock==3.12.0 pytest-cov pytest-env -uamqp>=1.2,<=1.6.8 +uamqp~=1.6.6 responses==0.22.0 black setuptools==70.0.0 diff --git a/setup.py b/setup.py index 308278e87..de9354b56 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,6 @@ "Programming Language :: Python", "Programming Language :: Python :: 3 :: Only", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", @@ -80,7 +79,7 @@ setup( name=PACKAGE_NAME, version=VERSION, - python_requires=">=3.8", + python_requires=">=3.9", description=short_description, long_description="{} Intended for power users and/or automation of IoT solutions at scale.".format( short_description