From 0bc1a70c83c66b312df470728c2c0556472843c9 Mon Sep 17 00:00:00 2001 From: Mike Gray Date: Fri, 9 Aug 2024 11:33:38 -0500 Subject: [PATCH 1/5] feat(performance): configurable filter The extra deserialization and serialization involved in filtering messages adds overhead to every message. By making it configurable, we could get some performance gains. --- README.md | 8 +++-- ovos_messagebus/event_handler.py | 52 ++++++++++++++++++++------------ 2 files changed, 39 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 890cfa6..1356abc 100644 --- a/README.md +++ b/README.md @@ -21,8 +21,12 @@ under mycroft.conf "ssl": false, // in mycroft-core all skills share a bus, this allows malicious skills // to manipulate it and affect other skills, this option ensures each skill - // gets it's own websocket connection - "shared_connection": true + // gets its own websocket connection + "shared_connection": true, + // filter out messages of certain types + "filter": false, + // which messages to filter if filter is enabled + "filter_ogs": ["gui.status.request", "gui.page.upload"] } } ``` \ No newline at end of file diff --git a/ovos_messagebus/event_handler.py b/ovos_messagebus/event_handler.py index e54824d..b11f00a 100644 --- a/ovos_messagebus/event_handler.py +++ b/ovos_messagebus/event_handler.py @@ -35,31 +35,45 @@ def __init__(self, application, request, **kwargs): def on(self, event_name, handler): self.emitter.on(event_name, handler) + @property + def filter(self) -> bool: + return Configuration().get("websocket", {}).get("filter", False) + + @property + def filter_ogs(self) -> list: + return Configuration().get("websocket", {}).get("filter_ogs", ["gui.status.request", "gui.page.upload"]) + @property def max_message_size(self) -> int: return Configuration().get("websocket", {}).get("max_msg_size", 10) * 1024 * 1024 def on_message(self, message): + if not self.filter: + try: + self.emitter.emit(message) + except Exception as e: + LOG.exception(e) + traceback.print_exc(file=sys.stdout) + pass + else: + try: + deserialized_message = Message.deserialize(message) + except Exception: + return + + if deserialized_message.msg_type not in self.filter_ogs: + LOG.debug(deserialized_message.msg_type + + f' source: {deserialized_message.context.get("source", [])}' + + f' destination: {deserialized_message.context.get("destination", [])}\n' + f'SESSION: {SessionManager.get(deserialized_message).serialize()}') - try: - deserialized_message = Message.deserialize(message) - except Exception: - return - - filter_ogs = ["gui.status.request", "gui.page.upload"] - if deserialized_message.msg_type not in filter_ogs: - LOG.debug(deserialized_message.msg_type + - f' source: {deserialized_message.context.get("source", [])}' + - f' destination: {deserialized_message.context.get("destination", [])}\n' - f'SESSION: {SessionManager.get(deserialized_message).serialize()}') - - try: - self.emitter.emit(deserialized_message.msg_type, - deserialized_message) - except Exception as e: - LOG.exception(e) - traceback.print_exc(file=sys.stdout) - pass + try: + self.emitter.emit(deserialized_message.msg_type, + deserialized_message) + except Exception as e: + LOG.exception(e) + traceback.print_exc(file=sys.stdout) + pass for client in client_connections: client.write_message(message) From 23c490521f9f5f63dc162bf39beaae6c1f7b7e44 Mon Sep 17 00:00:00 2001 From: Mike Gray Date: Fri, 9 Aug 2024 11:37:48 -0500 Subject: [PATCH 2/5] adjust workflows --- .github/workflows/build_tests.yml | 2 +- .github/workflows/install_tests.yml | 8 ++--- .github/workflows/unit_tests.yml | 45 +++++++++++++++-------------- 3 files changed, 28 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build_tests.yml b/.github/workflows/build_tests.yml index f5a9411..cfa61fa 100644 --- a/.github/workflows/build_tests.yml +++ b/.github/workflows/build_tests.yml @@ -13,4 +13,4 @@ jobs: uses: neongeckocom/.github/.github/workflows/python_build_tests.yml@master with: test_pipaudit: true - pipaudit_ignored: "GHSA-r9hx-vwmv-q579 PYSEC-2022-43012 GHSA-j8r2-6x86-q33q" \ No newline at end of file + pipaudit_ignored: "GHSA-r9hx-vwmv-q579 PYSEC-2022-43012 GHSA-j8r2-6x86-q33q GHSA-9wx4-h78v-vm56" diff --git a/.github/workflows/install_tests.yml b/.github/workflows/install_tests.yml index 4aaabea..d40fecc 100644 --- a/.github/workflows/install_tests.yml +++ b/.github/workflows/install_tests.yml @@ -11,12 +11,12 @@ jobs: strategy: max-parallel: 2 matrix: - python-version: [ 3.7, 3.8, 3.9, "3.10" ] + python-version: [3.8, 3.9, "3.10", "3.11", "3.12"] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Setup Python - uses: actions/setup-python@v1 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install Build Tools @@ -31,4 +31,4 @@ jobs: python setup.py bdist_wheel - name: Install package run: | - pip install .[all] \ No newline at end of file + pip install .[all] diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 9eaa17f..cd38eb0 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -4,29 +4,29 @@ on: branches: - dev paths-ignore: - - 'ovos_messagebus/version.py' - - 'examples/**' - - '.github/**' - - '.gitignore' - - 'LICENSE' - - 'CHANGELOG.md' - - 'MANIFEST.in' - - 'readme.md' - - 'scripts/**' + - "ovos_messagebus/version.py" + - "examples/**" + - ".github/**" + - ".gitignore" + - "LICENSE" + - "CHANGELOG.md" + - "MANIFEST.in" + - "readme.md" + - "scripts/**" push: branches: - master paths-ignore: - - 'ovos_messagebus/version.py' - - 'requirements/**' - - 'examples/**' - - '.github/**' - - '.gitignore' - - 'LICENSE' - - 'CHANGELOG.md' - - 'MANIFEST.in' - - 'readme.md' - - 'scripts/**' + - "ovos_messagebus/version.py" + - "requirements/**" + - "examples/**" + - ".github/**" + - ".gitignore" + - "LICENSE" + - "CHANGELOG.md" + - "MANIFEST.in" + - "readme.md" + - "scripts/**" workflow_dispatch: jobs: @@ -34,12 +34,12 @@ jobs: strategy: max-parallel: 2 matrix: - python-version: [ 3.7, 3.8, 3.9] + python-version: [3.8, 3.9, "3.10", "3.11", "3.12"] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 - name: Set up python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} - name: Install System Dependencies @@ -50,6 +50,7 @@ jobs: - name: Install core repo run: | pip install . + # TODO: Implement unit tests # - name: Install test dependencies # run: | From 9306a40d7cc94608bed501745497d73ec93d9c53 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 10 Sep 2024 21:53:25 -0500 Subject: [PATCH 3/5] Update README.md --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 1356abc..4dc9288 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ messagebus service, the nervous system of OpenVoiceOS ## Alternative implementations - [OVOS Bus Server](https://github.com/OpenVoiceOS/ovos-bus-server/) - Alternative C++ messagebus server implementation using WebSocket++ - +- [OVOS Rust Messagebus](https://github.com/OscillateLabsLLC/ovos-rust-messagebus) - Alternative Rust messagebus server implementation # Configuration @@ -26,7 +26,7 @@ under mycroft.conf // filter out messages of certain types "filter": false, // which messages to filter if filter is enabled - "filter_ogs": ["gui.status.request", "gui.page.upload"] + "filter_logs": ["gui.status.request", "gui.page.upload"] } } -``` \ No newline at end of file +``` From 165db655efe643b27c8e215a8897482321be3d46 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 10 Sep 2024 21:54:48 -0500 Subject: [PATCH 4/5] correct typo --- ovos_messagebus/event_handler.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ovos_messagebus/event_handler.py b/ovos_messagebus/event_handler.py index b11f00a..3ae8031 100644 --- a/ovos_messagebus/event_handler.py +++ b/ovos_messagebus/event_handler.py @@ -40,8 +40,8 @@ def filter(self) -> bool: return Configuration().get("websocket", {}).get("filter", False) @property - def filter_ogs(self) -> list: - return Configuration().get("websocket", {}).get("filter_ogs", ["gui.status.request", "gui.page.upload"]) + def filter_logs(self) -> list: + return Configuration().get("websocket", {}).get("filter_logs", ["gui.status.request", "gui.page.upload"]) @property def max_message_size(self) -> int: @@ -61,7 +61,7 @@ def on_message(self, message): except Exception: return - if deserialized_message.msg_type not in self.filter_ogs: + if deserialized_message.msg_type not in self.filter_logs: LOG.debug(deserialized_message.msg_type + f' source: {deserialized_message.context.get("source", [])}' + f' destination: {deserialized_message.context.get("destination", [])}\n' From a4d884a3127614aeec873c0bbfae227402c5ed56 Mon Sep 17 00:00:00 2001 From: Mike Date: Tue, 10 Sep 2024 22:07:32 -0500 Subject: [PATCH 5/5] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 4dc9288..911da52 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ under mycroft.conf // to manipulate it and affect other skills, this option ensures each skill // gets its own websocket connection "shared_connection": true, - // filter out messages of certain types + // filter out messages of certain types from the bus logs "filter": false, // which messages to filter if filter is enabled "filter_logs": ["gui.status.request", "gui.page.upload"]