From 60982b754ce426312978ce9f04ec10aea40d6429 Mon Sep 17 00:00:00 2001 From: Ed Younis Date: Sat, 31 Aug 2024 13:36:28 -0400 Subject: [PATCH 01/10] Protect runtime outgoing threads better --- bqskit/runtime/base.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/bqskit/runtime/base.py b/bqskit/runtime/base.py index d10996e49..bcc9f03fc 100644 --- a/bqskit/runtime/base.py +++ b/bqskit/runtime/base.py @@ -410,7 +410,13 @@ def send_outgoing(self) -> None: if outgoing[0].closed: continue - outgoing[0].send((outgoing[1], outgoing[2])) + try: + outgoing[0].send((outgoing[1], outgoing[2])) + except (EOFError, ConnectionResetError): + self.handle_disconnect(outgoing[0]) + _logger.warning('Connection reset while sending message.') + continue + if _logger.isEnabledFor(logging.DEBUG): to = self.get_to_string(outgoing[0]) _logger.debug(f'Sent message {outgoing[1].name} to {to}.') From 41c6ff03c8221ba526fd0e6136321e23974a9cd2 Mon Sep 17 00:00:00 2001 From: Ed Younis Date: Sat, 31 Aug 2024 14:07:03 -0400 Subject: [PATCH 02/10] Removed small memory leak --- bqskit/runtime/detached.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/bqskit/runtime/detached.py b/bqskit/runtime/detached.py index 8740c7170..7ef5e8a75 100644 --- a/bqskit/runtime/detached.py +++ b/bqskit/runtime/detached.py @@ -256,8 +256,19 @@ def handle_disconnect(self, conn: Connection) -> None: """Disconnect a client connection from the runtime.""" super().handle_disconnect(conn) tasks = self.clients.pop(conn) + for task_id in tasks: self.handle_cancel_comp_task(task_id) + + tasks_to_pop = [] + for (task, (tid, other_conn)) in self.tasks.items(): + if other_conn == conn: + tasks_to_pop.append((task_id, tid)) + + for task_id, tid in tasks_to_pop: + self.tasks.pop(task_id) + self.mailbox_to_task_dict.pop(tid) + _logger.info('Unregistered client.') def handle_new_comp_task( @@ -386,6 +397,9 @@ def handle_error(self, error_payload: tuple[int, str]) -> None: raise RuntimeError(error_payload) tid = error_payload[0] + if tid not in self.mailbox_to_task_dict: + return # Silently discard errors from cancelled tasks + conn = self.tasks[self.mailbox_to_task_dict[tid]][1] self.outgoing.put((conn, RuntimeMessage.ERROR, error_payload[1])) # TODO: Broadcast cancel to all tasks with compilation task id tid @@ -398,6 +412,9 @@ def handle_error(self, error_payload: tuple[int, str]) -> None: def handle_log(self, log_payload: tuple[int, LogRecord]) -> None: """Forward logs to appropriate client.""" tid = log_payload[0] + if tid not in self.mailbox_to_task_dict: + return # Silently discard logs from cancelled tasks + conn = self.tasks[self.mailbox_to_task_dict[tid]][1] self.outgoing.put((conn, RuntimeMessage.LOG, log_payload[1])) From ae093679afd226e3d4e224dfc2de4c691a587647 Mon Sep 17 00:00:00 2001 From: Ed Younis Date: Sat, 31 Aug 2024 16:23:29 -0400 Subject: [PATCH 03/10] Allow nonserializable logs through runtime --- bqskit/compiler/compiler.py | 14 ++++++++--- bqskit/runtime/detached.py | 2 +- bqskit/runtime/worker.py | 11 +++++++- tests/runtime/test_logging.py | 47 +++++++++++++++++++++++++++++++++++ 4 files changed, 69 insertions(+), 5 deletions(-) diff --git a/bqskit/compiler/compiler.py b/bqskit/compiler/compiler.py index 8f58f3c35..58452b91d 100644 --- a/bqskit/compiler/compiler.py +++ b/bqskit/compiler/compiler.py @@ -4,6 +4,7 @@ import atexit import functools import logging +import pickle import signal import subprocess import sys @@ -439,9 +440,16 @@ def _recv_handle_log_error(self) -> tuple[RuntimeMessage, Any]: msg, payload = self.conn.recv() if msg == RuntimeMessage.LOG: - logger = logging.getLogger(payload.name) - if logger.isEnabledFor(payload.levelno): - logger.handle(payload) + record = pickle.loads(payload) + if isinstance(record, logging.LogRecord): + logger = logging.getLogger(record.name) + if logger.isEnabledFor(record.levelno): + logger.handle(record) + else: + name, levelno, msg = record + logger = logging.getLogger(name) + logger.log(levelno, msg) + elif msg == RuntimeMessage.ERROR: raise RuntimeError(payload) diff --git a/bqskit/runtime/detached.py b/bqskit/runtime/detached.py index 7ef5e8a75..a67768734 100644 --- a/bqskit/runtime/detached.py +++ b/bqskit/runtime/detached.py @@ -409,7 +409,7 @@ def handle_error(self, error_payload: tuple[int, str]) -> None: # still cancel here incase the client catches the error and # resubmits a job. - def handle_log(self, log_payload: tuple[int, LogRecord]) -> None: + def handle_log(self, log_payload: tuple[int, bytes]) -> None: """Forward logs to appropriate client.""" tid = log_payload[0] if tid not in self.mailbox_to_task_dict: diff --git a/bqskit/runtime/worker.py b/bqskit/runtime/worker.py index 1684f7dbc..fd6b2030e 100644 --- a/bqskit/runtime/worker.py +++ b/bqskit/runtime/worker.py @@ -3,6 +3,7 @@ import argparse import logging +import pickle import os import signal import sys @@ -225,7 +226,15 @@ def record_factory(*args: Any, **kwargs: Any) -> logging.LogRecord: record.msg += con_str record.msg += ']' tid = active_task.comp_task_id - self._conn.send((RuntimeMessage.LOG, (tid, record))) + try: + serial = pickle.dumps(record) + except (pickle.PicklingError, TypeError): + serial = pickle.dumps(( + record.name, + record.levelno, + record.getMessage(), + )) + self._conn.send((RuntimeMessage.LOG, (tid, serial))) return record logging.setLogRecordFactory(record_factory) diff --git a/tests/runtime/test_logging.py b/tests/runtime/test_logging.py index bee4bfb0a..ad5778f8e 100644 --- a/tests/runtime/test_logging.py +++ b/tests/runtime/test_logging.py @@ -2,6 +2,7 @@ from __future__ import annotations import logging +import pickle from io import StringIO import pytest @@ -142,6 +143,52 @@ def test_using_external_logging(server_compiler: Compiler) -> None: logger.removeHandler(handler) logger.setLevel(logging.WARNING) +class ExternalWithArgsPass(BasePass): + async def run(self, circuit, pass_data): + logging.getLogger("dummy2").debug('int %d', 1) + +def test_external_logging_with_args(server_compiler: Compiler) -> None: + logger = logging.getLogger('dummy2') + logger.setLevel(logging.DEBUG) + handler = logging.StreamHandler(StringIO()) + handler.setLevel(logging.DEBUG) + logger.addHandler(handler) + server_compiler.compile(Circuit(1), [ExternalWithArgsPass()]) + log = handler.stream.getvalue() + assert 'int 1' in log + logger.removeHandler(handler) + logger.setLevel(logging.WARNING) + + +class NonSerializable: + def __reduce__(self): + raise pickle.PicklingError("This class is not serializable") + def __str__(self): + return "NonSerializable" + + +class ExternalWithNonSerializableArgsPass(BasePass): + async def run(self, circuit, pass_data): + logging.getLogger("dummy2").debug( + 'NonSerializable %s', + NonSerializable() + ) + + +def test_external_logging_with_nonserializable_args( + server_compiler: Compiler +) -> None: + logger = logging.getLogger('dummy2') + logger.setLevel(logging.DEBUG) + handler = logging.StreamHandler(StringIO()) + handler.setLevel(logging.DEBUG) + logger.addHandler(handler) + server_compiler.compile(Circuit(1), [ExternalWithNonSerializableArgsPass()]) + log = handler.stream.getvalue() + assert 'NonSerializable NonSerializable' in log + logger.removeHandler(handler) + logger.setLevel(logging.WARNING) + @pytest.mark.parametrize('level', [-1, 0, 1, 2, 3, 4]) def test_limiting_nested_calls_enable_logging( From e3d557e18b57b758befe5922907a176c5a165b1b Mon Sep 17 00:00:00 2001 From: Ed Younis Date: Sat, 31 Aug 2024 16:30:19 -0400 Subject: [PATCH 04/10] Added layer info to successful log on synthesis --- bqskit/passes/synthesis/leap.py | 6 ++++-- bqskit/passes/synthesis/qfast.py | 2 +- bqskit/passes/synthesis/qsearch.py | 8 +++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/bqskit/passes/synthesis/leap.py b/bqskit/passes/synthesis/leap.py index da7475fae..30ec70487 100644 --- a/bqskit/passes/synthesis/leap.py +++ b/bqskit/passes/synthesis/leap.py @@ -196,7 +196,7 @@ async def synthesize( # Evalute initial layer if best_dist < self.success_threshold: - _logger.debug('Successful synthesis.') + _logger.debug('Successful synthesis with 0 layers.') return initial_layer # Main loop @@ -222,7 +222,9 @@ async def synthesize( dist = self.cost.calc_cost(circuit, utry) if dist < self.success_threshold: - _logger.debug('Successful synthesis.') + _logger.debug( + f'Successful synthesis with {layer + 1} layers.' + ) if self.store_partial_solutions: data['psols'] = psols return circuit diff --git a/bqskit/passes/synthesis/qfast.py b/bqskit/passes/synthesis/qfast.py index 9a72bd79c..e4e036fb1 100644 --- a/bqskit/passes/synthesis/qfast.py +++ b/bqskit/passes/synthesis/qfast.py @@ -164,7 +164,7 @@ async def synthesize( if dist < self.success_threshold: self.finalize(circuit, utry, instantiate_options) - _logger.info('Successful synthesis.') + _logger.info(f'Successful synthesis with {depth} layers.') return circuit # Expand or restrict head diff --git a/bqskit/passes/synthesis/qsearch.py b/bqskit/passes/synthesis/qsearch.py index 13276ad82..c657bc702 100644 --- a/bqskit/passes/synthesis/qsearch.py +++ b/bqskit/passes/synthesis/qsearch.py @@ -171,7 +171,7 @@ async def synthesize( # Evalute initial layer if best_dist < self.success_threshold: - _logger.debug('Successful synthesis.') + _logger.debug('Successful synthesis with 0 layers.') return initial_layer # Main loop @@ -197,7 +197,9 @@ async def synthesize( dist = self.cost.calc_cost(circuit, utry) if dist < self.success_threshold: - _logger.debug('Successful synthesis.') + _logger.debug( + f'Successful synthesis with {layer + 1} layers.' + ) if self.store_partial_solutions: data['psols'] = psols return circuit @@ -210,7 +212,7 @@ async def synthesize( ) best_dist = dist best_circ = circuit - best_layer = layer + best_layer = layer + 1 if self.store_partial_solutions: if layer not in psols: From 8cb016551087605b4456187a1a2dbf5f8f7dbe3a Mon Sep 17 00:00:00 2001 From: Ed Younis Date: Sat, 31 Aug 2024 16:33:04 -0400 Subject: [PATCH 05/10] pre-commit --- bqskit/compiler/compiler.py | 1 - bqskit/passes/synthesis/leap.py | 2 +- bqskit/passes/synthesis/qsearch.py | 2 +- bqskit/runtime/detached.py | 3 +-- bqskit/runtime/worker.py | 2 +- tests/runtime/test_logging.py | 24 ++++++++++++++---------- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/bqskit/compiler/compiler.py b/bqskit/compiler/compiler.py index 58452b91d..1a94ef9bb 100644 --- a/bqskit/compiler/compiler.py +++ b/bqskit/compiler/compiler.py @@ -450,7 +450,6 @@ def _recv_handle_log_error(self) -> tuple[RuntimeMessage, Any]: logger = logging.getLogger(name) logger.log(levelno, msg) - elif msg == RuntimeMessage.ERROR: raise RuntimeError(payload) diff --git a/bqskit/passes/synthesis/leap.py b/bqskit/passes/synthesis/leap.py index 30ec70487..f05300eef 100644 --- a/bqskit/passes/synthesis/leap.py +++ b/bqskit/passes/synthesis/leap.py @@ -223,7 +223,7 @@ async def synthesize( if dist < self.success_threshold: _logger.debug( - f'Successful synthesis with {layer + 1} layers.' + f'Successful synthesis with {layer + 1} layers.', ) if self.store_partial_solutions: data['psols'] = psols diff --git a/bqskit/passes/synthesis/qsearch.py b/bqskit/passes/synthesis/qsearch.py index c657bc702..9cad4fc44 100644 --- a/bqskit/passes/synthesis/qsearch.py +++ b/bqskit/passes/synthesis/qsearch.py @@ -198,7 +198,7 @@ async def synthesize( if dist < self.success_threshold: _logger.debug( - f'Successful synthesis with {layer + 1} layers.' + f'Successful synthesis with {layer + 1} layers.', ) if self.store_partial_solutions: data['psols'] = psols diff --git a/bqskit/runtime/detached.py b/bqskit/runtime/detached.py index a67768734..ea32afbd6 100644 --- a/bqskit/runtime/detached.py +++ b/bqskit/runtime/detached.py @@ -8,7 +8,6 @@ import time import uuid from dataclasses import dataclass -from logging import LogRecord from multiprocessing.connection import Connection from multiprocessing.connection import Listener from threading import Thread @@ -263,7 +262,7 @@ def handle_disconnect(self, conn: Connection) -> None: tasks_to_pop = [] for (task, (tid, other_conn)) in self.tasks.items(): if other_conn == conn: - tasks_to_pop.append((task_id, tid)) + tasks_to_pop.append((task_id, tid)) for task_id, tid in tasks_to_pop: self.tasks.pop(task_id) diff --git a/bqskit/runtime/worker.py b/bqskit/runtime/worker.py index fd6b2030e..e61b13009 100644 --- a/bqskit/runtime/worker.py +++ b/bqskit/runtime/worker.py @@ -3,8 +3,8 @@ import argparse import logging -import pickle import os +import pickle import signal import sys import time diff --git a/tests/runtime/test_logging.py b/tests/runtime/test_logging.py index ad5778f8e..4c8694439 100644 --- a/tests/runtime/test_logging.py +++ b/tests/runtime/test_logging.py @@ -4,6 +4,7 @@ import logging import pickle from io import StringIO +from typing import Any import pytest @@ -143,9 +144,11 @@ def test_using_external_logging(server_compiler: Compiler) -> None: logger.removeHandler(handler) logger.setLevel(logging.WARNING) + class ExternalWithArgsPass(BasePass): - async def run(self, circuit, pass_data): - logging.getLogger("dummy2").debug('int %d', 1) + async def run(self, circuit: Circuit, data: PassData) -> None: + logging.getLogger('dummy2').debug('int %d', 1) + def test_external_logging_with_args(server_compiler: Compiler) -> None: logger = logging.getLogger('dummy2') @@ -161,22 +164,23 @@ def test_external_logging_with_args(server_compiler: Compiler) -> None: class NonSerializable: - def __reduce__(self): - raise pickle.PicklingError("This class is not serializable") - def __str__(self): - return "NonSerializable" + def __reduce__(self) -> str | tuple[Any, ...]: + raise pickle.PicklingError('This class is not serializable') + + def __str__(self) -> str: + return 'NonSerializable' class ExternalWithNonSerializableArgsPass(BasePass): - async def run(self, circuit, pass_data): - logging.getLogger("dummy2").debug( + async def run(self, circuit: Circuit, data: PassData) -> None: + logging.getLogger('dummy2').debug( 'NonSerializable %s', - NonSerializable() + NonSerializable(), ) def test_external_logging_with_nonserializable_args( - server_compiler: Compiler + server_compiler: Compiler, ) -> None: logger = logging.getLogger('dummy2') logger.setLevel(logging.DEBUG) From 0b557fb4388986a706a6d99806f8fe836bc40a90 Mon Sep 17 00:00:00 2001 From: Ed Younis Date: Sat, 31 Aug 2024 16:35:18 -0400 Subject: [PATCH 06/10] Fixed pass list docs --- bqskit/passes/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bqskit/passes/__init__.py b/bqskit/passes/__init__.py index 9a386f6fb..3a9882ee4 100644 --- a/bqskit/passes/__init__.py +++ b/bqskit/passes/__init__.py @@ -138,6 +138,10 @@ These passes either perform upper-bound error analysis of the PAM process. +.. autosummary:: + :toctree: autogen + :recursive: + TagPAMBlockDataPass CalculatePAMErrorsPass UnTagPAMBlockDataPass From 38e52364b5923f9c21c0f9fc94a35045fcfa3c5a Mon Sep 17 00:00:00 2001 From: Ed Younis Date: Sat, 31 Aug 2024 16:46:10 -0400 Subject: [PATCH 07/10] Attempt to fix #245 --- docs/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index f48262b93..ccb995325 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -75,8 +75,8 @@ # 'IntervalLike': 'bqskit.ir.IntervalLike', # } # napoleon_type_aliases = autodoc_type_aliases -autodoc_typehints = 'description' -autodoc_typehints_description_target = 'documented' +autodoc_typehints = 'both' +autodoc_typehints_description_target = 'all' autoclass_content = 'class' nbsphinx_output_prompt = 'Out[%s]:' From d1b9c38a48f652f760c19f0612b4a5845bafc833 Mon Sep 17 00:00:00 2001 From: Ed Younis Date: Sat, 31 Aug 2024 17:02:05 -0400 Subject: [PATCH 08/10] Second attempt --- docs/conf.py | 5 +++-- docs/requirements.txt | 1 + 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/conf.py b/docs/conf.py index ccb995325..45036da82 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -38,6 +38,7 @@ 'myst_parser', 'jupyter_sphinx', 'nbsphinx', + 'sphinx_autodoc_typehints', ] # Add any paths that contain templates here, relative to this directory. @@ -75,8 +76,8 @@ # 'IntervalLike': 'bqskit.ir.IntervalLike', # } # napoleon_type_aliases = autodoc_type_aliases -autodoc_typehints = 'both' -autodoc_typehints_description_target = 'all' +autodoc_typehints = 'description' +autodoc_typehints_description_target = 'documented' autoclass_content = 'class' nbsphinx_output_prompt = 'Out[%s]:' diff --git a/docs/requirements.txt b/docs/requirements.txt index 45bb175e7..a88d4eaa7 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -2,6 +2,7 @@ Sphinx>=4.5.0 sphinx-autodoc-typehints>=1.12.0 sphinx-rtd-theme>=1.0.0 sphinx-togglebutton>=0.2.3 +sphinx-autodoc-typehints>=2.3.0 sphinxcontrib-applehelp>=1.0.2 sphinxcontrib-devhelp>=1.0.2 sphinxcontrib-htmlhelp>=2.0.0 From de21e49a0b61f53902f2237dd796edc99c13155f Mon Sep 17 00:00:00 2001 From: Ed Younis Date: Sat, 31 Aug 2024 17:09:29 -0400 Subject: [PATCH 09/10] Manual fix --- bqskit/compiler/compiler.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/bqskit/compiler/compiler.py b/bqskit/compiler/compiler.py index 1a94ef9bb..6127c3745 100644 --- a/bqskit/compiler/compiler.py +++ b/bqskit/compiler/compiler.py @@ -313,7 +313,15 @@ def submit( return task.task_id def status(self, task_id: uuid.UUID) -> CompilationStatus: - """Retrieve the status of the specified task.""" + """ + Retrieve the status of the specified task. + + Args: + task_id (uuid.UUID): The ID of the task to check. + + Returns: + CompilationStatus: The status of the task. + """ msg, payload = self._send_recv(RuntimeMessage.STATUS, task_id) if msg != RuntimeMessage.STATUS: raise RuntimeError(f'Unexpected message type: {msg}.') From f061beff9ec6270047bb40ddb9d0fc458b68d27f Mon Sep 17 00:00:00 2001 From: Ed Younis Date: Sat, 31 Aug 2024 17:14:10 -0400 Subject: [PATCH 10/10] Fixed DiagonalSynthesis Doc Issue --- bqskit/passes/__init__.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bqskit/passes/__init__.py b/bqskit/passes/__init__.py index 3a9882ee4..389d42ec5 100644 --- a/bqskit/passes/__init__.py +++ b/bqskit/passes/__init__.py @@ -28,7 +28,7 @@ :toctree: autogen :recursive: - DiagonalSynthesisPass + WalshDiagonalSynthesisPass LEAPSynthesisPass QSearchSynthesisPass QFASTDecompositionPass @@ -289,6 +289,7 @@ from bqskit.passes.search.heuristics.astar import AStarHeuristic from bqskit.passes.search.heuristics.dijkstra import DijkstraHeuristic from bqskit.passes.search.heuristics.greedy import GreedyHeuristic +from bqskit.passes.synthesis.diagonal import WalshDiagonalSynthesisPass from bqskit.passes.synthesis.leap import LEAPSynthesisPass from bqskit.passes.synthesis.pas import PermutationAwareSynthesisPass from bqskit.passes.synthesis.qfast import QFASTDecompositionPass @@ -326,7 +327,7 @@ 'ScanPartitioner', 'QuickPartitioner', 'SynthesisPass', - 'DiagonalSynthesisPass', + 'WalshDiagonalSynthesisPass', 'LEAPSynthesisPass', 'QSearchSynthesisPass', 'QFASTDecompositionPass',