From 005b0923bd0b4c14849944c00a8f17fa352df250 Mon Sep 17 00:00:00 2001 From: janezd Date: Thu, 26 Oct 2023 15:10:44 +0200 Subject: [PATCH 1/3] Network Explorer: Replace (crashing) threads with ConcurrentMixin --- .../network/network/layout/__init__.py | 25 ++- orangecontrib/network/widgets/OWNxExplorer.py | 154 +++++++----------- 2 files changed, 72 insertions(+), 107 deletions(-) diff --git a/orangecontrib/network/network/layout/__init__.py b/orangecontrib/network/network/layout/__init__.py index c4a7410d..0d3ce54c 100644 --- a/orangecontrib/network/network/layout/__init__.py +++ b/orangecontrib/network/network/layout/__init__.py @@ -13,31 +13,27 @@ def fruchterman_reingold(pos, k, init_temp=0.05, sample_ratio=None, - callback_step=None, callback=None): - - last_perc = 0 - - def run_iterations(n_iterations, temperatures): - nonlocal last_perc + callback_interval=0.5, callback=None): + def run_iterations(n_iterations, temperatures, callback=callback): + last_call = time.perf_counter() for iteration in range(n_iterations): if sample_ratio is not None: np.random.shuffle(sample) fruchterman_reingold_step(pos, sample[:sample_size], edge_src, edge_dst, edge_weights, k, temperatures[iteration], disp) - if callback is not None and iteration % callback_step == 0: - perc = iteration / n_iterations * 100 - if not callback(pos, perc - last_perc): - return False - last_perc = perc - return True + if callback and \ + (now := time.perf_counter()) - last_call > callback_interval: + progress = iteration / n_iterations * 100 + callback(pos, progress) + last_call = now n_nodes = len(pos) edge_weights = edges.data if weighted else np.empty(0) - edge_src, edge_dst = edges.row, edges.col, + edge_src, edge_dst = edges.row.astype(np.int32), edges.col.astype(np.int32) disp = np.empty((n_nodes, 2)) @@ -50,8 +46,7 @@ def run_iterations(n_iterations, temperatures): temperatures = np.linspace(init_temp, 0.01, TEST_ITERATIONS) start_time = time.perf_counter() - if not run_iterations(TEST_ITERATIONS, temperatures): - return + run_iterations(TEST_ITERATIONS, temperatures, callback=None) elapsed_time = time.perf_counter() - start_time iterations = int(allowed_time / (elapsed_time / TEST_ITERATIONS)) diff --git a/orangecontrib/network/widgets/OWNxExplorer.py b/orangecontrib/network/widgets/OWNxExplorer.py index a6d5d8fd..c3955622 100644 --- a/orangecontrib/network/widgets/OWNxExplorer.py +++ b/orangecontrib/network/widgets/OWNxExplorer.py @@ -1,12 +1,13 @@ import numpy as np import scipy.sparse as sp -from AnyQt.QtCore import QTimer, QSize, Qt, Signal, QObject, QThread +from AnyQt.QtCore import QTimer, QSize, Qt import Orange from Orange.data import Table, Domain, StringVariable -from Orange.widgets import gui, widget +from Orange.widgets import gui from Orange.widgets.settings import Setting, SettingProvider +from Orange.widgets.utils.concurrent import ConcurrentWidgetMixin from Orange.widgets.utils.plot import OWPlotGUI from Orange.widgets.visualize.utils.widget import OWDataProjectionWidget from Orange.widgets.widget import Input, Output @@ -14,11 +15,30 @@ from orangecontrib.network.network.base import Network from orangecontrib.network.network.layout import fruchterman_reingold from orangecontrib.network.widgets.graphview import GraphView +from orangewidget.widget import Message, Msg FR_ALLOWED_TIME = 30 -class OWNxExplorer(OWDataProjectionWidget): +def run(positions, edges, observe_weights, init_temp, k, state): + def update(positions, progress): + state.set_progress_value(progress) + if not large_graph: + state.set_partial_result(positions) + if state.is_interruption_requested(): + raise Exception # pylint: disable=broad-exception-raised + + nnodes = positions.shape[0] + large_graph = nnodes + edges.shape[0] > 30000 + sample_ratio = None if nnodes < 1000 else 1000 / nnodes + fruchterman_reingold( + positions, edges, observe_weights, + FR_ALLOWED_TIME, k, init_temp, sample_ratio, + callback=update) + return positions + + +class OWNxExplorer(OWDataProjectionWidget, ConcurrentWidgetMixin): name = "Network Explorer" description = "Visually explore the network and its properties." icon = "icons/NetworkExplorer.svg" @@ -36,8 +56,8 @@ class Outputs(OWDataProjectionWidget.Outputs): distances = Output("Distance matrix", Orange.misc.DistMatrix) UserAdviceMessages = [ - widget.Message("Double clicks select connected components", - widget.Message.Information), + Message("Double clicks select connected components", + Message.Information), ] GRAPH_CLASS = GraphView @@ -54,16 +74,15 @@ class Outputs(OWDataProjectionWidget.Outputs): alpha_value = 255 # Override the setting from parent class Warning(OWDataProjectionWidget.Warning): - distance_matrix_mismatch = widget.Msg( + distance_matrix_mismatch = Msg( "Distance matrix size doesn't match the number of network nodes " "and will be ignored.") - no_graph_found = widget.Msg("Node data is given, graph data is missing.") + no_graph_found = Msg("Node data is given, graph data is missing.") class Error(OWDataProjectionWidget.Error): - data_size_mismatch = widget.Msg( + data_size_mismatch = Msg( "Length of the data does not match the number of nodes.") - network_too_large = widget.Msg("Network is too large to visualize.") - single_node_graph = widget.Msg("I don't do single-node graphs today.") + network_too_large = Msg("Network is too large to visualize.") def __init__(self): # These are already needed in super().__init__() @@ -77,7 +96,8 @@ def __init__(self): self.mark_mode = 0 self.mark_text = "" - super().__init__() + OWDataProjectionWidget.__init__(self) + ConcurrentWidgetMixin.__init__(self) self.network = None self.node_data = None @@ -85,10 +105,6 @@ def __init__(self): self.edges = None self.positions = None - self._optimizer = None - self._animation_thread = None - self._stop_optimization = False - self.marked_nodes = None self.searchStringTimer = QTimer(self) self.searchStringTimer.timeout.connect(self.update_marks) @@ -99,6 +115,7 @@ def sizeHint(self): return QSize(800, 600) def _add_controls(self): + # pylint: disable=attribute-defined-outside-init self.gui = OWPlotGUI(self) self._add_info_box() self.gui.point_properties_box(self.controlArea) @@ -108,6 +125,7 @@ def _add_controls(self): self.controls.attr_label.activated.connect(self.on_change_label_attr) def _add_info_box(self): + # pylint: disable=attribute-defined-outside-init info = gui.vBox(self.controlArea, box="Layout") gui.label( info, self, @@ -137,6 +155,7 @@ def _add_info_box(self): callback=self.improve) def _add_effects_box(self): + # pylint: disable=attribute-defined-outside-init gbox = self.gui.create_gridbox(self.controlArea, box="Widths and Sizes") self.gui.add_widget(self.gui.PointSize, gbox) gbox.layout().itemAtPosition(1, 0).widget().setText("Node Size:") @@ -167,6 +186,7 @@ def _add_effects_box(self): gui.hSlider(None, self, "graph.alpha_value") def _add_mark_box(self): + # pylint: disable=attribute-defined-outside-init hbox = gui.hBox(None, box=True) self.mainArea.layout().addWidget(hbox) vbox = gui.hBox(hbox) @@ -345,7 +365,7 @@ def update_selection_buttons(self): self.btselect.show() selection = self.graph.get_selection() - if not len(selection) or np.max(selection) == 0: + if len(selection) == 0 or np.max(selection) == 0: self.btadd.hide() self.btgroup.hide() elif np.max(selection) == 1: @@ -433,7 +453,6 @@ def set_actual_data(): self.closeContext() self.Error.data_size_mismatch.clear() self.Warning.no_graph_found.clear() - self._invalid_data = False if network is None: if self.node_data is not None: self.Warning.no_graph_found() @@ -442,7 +461,6 @@ def set_actual_data(): if self.node_data is not None: if len(self.node_data) != n_nodes: self.Error.data_size_mismatch() - self._invalid_data = True self.data = None else: self.data = self.node_data @@ -500,7 +518,7 @@ def set_checkboxes(value): elif len(set(self.edges.data)) == 1: set_checkboxes(False) - self.stop_optimization_and_wait() + self.cancel() set_actual_data() super()._handle_subset_data() if self.positions is None: @@ -527,7 +545,7 @@ def randomize(self): def set_random_positions(self): if self.network is None: - self.position = None + self.positions = None else: self.positions = np.random.uniform(size=(self.number_of_nodes, 2)) @@ -593,8 +611,7 @@ def set_buttons(self, running): self.randomize_button.setHidden(running) def stop_relayout(self): - self._stop_optimization = True - self.set_buttons(running=False) + self.cancel() def restart(self): self.relayout(restart=True) @@ -602,89 +619,41 @@ def restart(self): def improve(self): self.relayout(restart=False) - # TODO: Stop relayout if new data is received def relayout(self, restart): if self.edges is None: return if restart or self.positions is None: self.set_random_positions() - self.progressbar = gui.ProgressBar(self, 100) - self.set_buttons(running=True) - self._stop_optimization = False Simplifications = self.graph.Simplifications self.graph.set_simplifications( Simplifications.NoDensity + Simplifications.NoLabels * (len(self.graph.labels) > 20) + Simplifications.NoEdgeLabels * (len(self.graph.edge_labels) > 20) - + Simplifications.NoEdges * (self.number_of_edges > 30000)) - - large_graph = self.number_of_nodes + self.number_of_edges > 30000 - - class LayoutOptimizer(QObject): - update = Signal(np.ndarray, float) - done = Signal(np.ndarray) - stopped = Signal() - - def __init__(self, widget): - super().__init__() - self.widget = widget - - def send_update(self, positions, progress): - if not large_graph: - self.update.emit(np.array(positions), progress) - return not self.widget._stop_optimization - - def run(self): - widget = self.widget - edges = widget.edges - nnodes = widget.number_of_nodes - init_temp = 0.05 if restart else 0.2 - k = widget.layout_density / 10 / np.sqrt(nnodes) - sample_ratio = None if nnodes < 1000 else 1000 / nnodes - fruchterman_reingold( - widget.positions, edges, widget.observe_weights, - FR_ALLOWED_TIME, k, init_temp, sample_ratio, - callback_step=4, callback=self.send_update) - self.done.emit(widget.positions) - self.stopped.emit() - - def update(positions, progress): - self.progressbar.advance(progress) - self.positions = positions - self.graph.update_coordinates() - - def done(positions): - self.positions = positions - self.set_buttons(running=False) - self.graph.set_simplifications( - self.graph.Simplifications.NoSimplifications) - self.graph.update_coordinates() - self.progressbar.finish() - - def thread_finished(): - self._optimizer = None - self._animation_thread = None - - self._optimizer = LayoutOptimizer(self) - self._animation_thread = QThread() - self._optimizer.update.connect(update) - self._optimizer.done.connect(done) - self._optimizer.stopped.connect(self._animation_thread.quit) - self._optimizer.moveToThread(self._animation_thread) - self._animation_thread.started.connect(self._optimizer.run) - self._animation_thread.finished.connect(thread_finished) - self._animation_thread.start() - - def stop_optimization_and_wait(self): - if self._animation_thread is not None: - self._stop_optimization = True - self._animation_thread.quit() - self._animation_thread.wait() - self._animation_thread = None + + Simplifications.NoEdges * (self.number_of_edges > 1000)) + + init_temp = 0.05 if restart else 0.2 + k = self.layout_density / 10 / np.sqrt(self.number_of_nodes) + self.set_buttons(running=True) + self.start(run, self.positions, self.edges, self.observe_weights, init_temp, k) + + def cancel(self): + self.set_buttons(running=False) + super().cancel() + + def on_done(self, positions): # pylint: disable=arguments-renamed + self.positions = positions + self.set_buttons(running=False) + self.graph.set_simplifications( + self.graph.Simplifications.NoSimplifications) + self.graph.update_coordinates() + + def on_partial_result(self, positions): # pylint: disable=arguments-renamed + self.positions = positions + self.graph.update_coordinates() def onDeleteWidget(self): - self.stop_optimization_and_wait() + self.shutdown() super().onDeleteWidget() def send_report(self): @@ -711,6 +680,7 @@ def send_report(self): def main(): + # pylint: disable=import-outside-toplevel, unused-import from Orange.widgets.utils.widgetpreview import WidgetPreview from orangecontrib.network.network.readwrite \ import read_pajek, transform_data_to_orange_table From 6f492dced2c6255cfda9f7c55ee70729a58469a8 Mon Sep 17 00:00:00 2001 From: janezd Date: Sat, 28 Oct 2023 13:38:18 +0200 Subject: [PATCH 2/3] Network Explorer: Reduce number of slow repaints during optimization --- orangecontrib/network/widgets/OWNxExplorer.py | 142 ++++++++++++++++++ .../widgets/tests/test_OWNxExplorer.py | 9 +- 2 files changed, 150 insertions(+), 1 deletion(-) diff --git a/orangecontrib/network/widgets/OWNxExplorer.py b/orangecontrib/network/widgets/OWNxExplorer.py index c3955622..a51ea7a4 100644 --- a/orangecontrib/network/widgets/OWNxExplorer.py +++ b/orangecontrib/network/widgets/OWNxExplorer.py @@ -1,3 +1,8 @@ +import time +import functools +from weakref import WeakKeyDictionary +from typing import Union, Optional, Callable + import numpy as np import scipy.sparse as sp @@ -20,6 +25,139 @@ FR_ALLOWED_TIME = 30 +# This decorator doesn't belong here. When Orange 3.37 is released +# (hopefully with https://github.com/biolab/orange3/pull/6612), this code +# should be removed and the decorator imported from Orange.util. + +# This should look like decorator, not a class, pylint: disable=invalid-name +class allot: + """ + Decorator that allows a function only a specified portion of time per call. + + Usage: + + ``` + @allot(0.2, overflow=of) + def f(x): + ... + ``` + + The above function is allotted 0.2 second per second. If it runs for 0.2 s, + all subsequent calls in the next second (after the start of the call) are + ignored. If it runs for 0.1 s, subsequent calls in the next 0.5 s are + ignored. If it runs for a second, subsequent calls are ignored for 5 s. + + An optional overflow function can be given as a keyword argument + `overflow`. This function must have the same signature as the wrapped + function and is called instead of the original when the call is blocked. + + If the overflow function is not given, the wrapped function must not return + result. This is because without the overflow function, the wrapper has no + value to return when the call is skipped. + + The decorator adds a method `call` to force the call, e.g. by calling + f.call(5), in the above case. The used up time still counts for the + following (non-forced) calls. + + The decorator also adds two attributes: + + - f.last_call_duration is the duration of the last call (in seconds) + - f.no_call_before contains the time (time.perf_counter) when the next + call will be made. + + The decorator can be used for functions and for methods. + + A non-parametrized decorator doesn't block any calls and only adds + last_call_duration, so that it can be used for timing. + """ + def __new__(cls: type, arg: Union[None, float, Callable], *, + overflow: Optional[Callable] = None, + _bound_methods: Optional[WeakKeyDictionary] = None): + self = super().__new__(cls) + + if arg is None or isinstance(arg, float): + # Parametrized decorator + if arg is not None: + assert arg > 0 + + def set_func(func): + self.__init__(func, + overflow=overflow, + _bound_methods=_bound_methods) + self.allotted_time = arg + return self + + return set_func + + else: + # Non-parametrized decorator + self.allotted_time = None + return self + + def __init__(self, + func: Callable, *, + overflow: Optional[Callable] = None, + _bound_methods: Optional[WeakKeyDictionary] = None): + assert callable(func) + self.func = func + self.overflow = overflow + functools.update_wrapper(self, func) + + self.no_call_before = 0 + self.last_call_duration = None + + # Used by __get__; see a comment there + if _bound_methods is None: + self.__bound_methods = WeakKeyDictionary() + else: + self.__bound_methods = _bound_methods + + # If we are wrapping a method, __get__ is called to bind it. + # Create a wrapper for each instance and store it, so that each instance's + # method gets its share of time. + def __get__(self, inst, cls): + if inst is None: + return self + + if inst not in self.__bound_methods: + # __bound_methods caches bound methods per instance. This is not + # done for perfoamnce. Bound methods can be rebound, even to + # different instances or even classes, e.g. + # >>> x = f.__get__(a, A) + # >>> y = x.__get__(b, B) + # >>> z = x.__get__(a, A) + # After this, we want `x is z`, there shared caching. This looks + # bizarre, but let's keep it safe. At least binding to the same + # instance, f.__get__(a, A),__get__(a, A), sounds reasonably + # possible. + cls = type(self) + bound_overflow = self.overflow and self.overflow.__get__(inst, cls) + decorator = cls( + self.allotted_time, + overflow=bound_overflow, + _bound_methods=self.__bound_methods) + self.__bound_methods[inst] = decorator(self.func.__get__(inst, cls)) + + return self.__bound_methods[inst] + + def __call__(self, *args, **kwargs): + if time.perf_counter() < self.no_call_before: + if self.overflow is None: + return None + return self.overflow(*args, **kwargs) + return self.call(*args, **kwargs) + + def call(self, *args, **kwargs): + start = time.perf_counter() + result = self.func(*args, **kwargs) + self.last_call_duration = time.perf_counter() - start + if self.allotted_time is not None: + if self.overflow is None: + assert result is None, "skippable function cannot return a result" + self.no_call_before = start + self.last_call_duration / self.allotted_time + return result + + def run(positions, edges, observe_weights, init_temp, k, state): def update(positions, progress): state.set_progress_value(progress) @@ -648,6 +786,7 @@ def on_done(self, positions): # pylint: disable=arguments-renamed self.graph.Simplifications.NoSimplifications) self.graph.update_coordinates() + @allot(0.02) def on_partial_result(self, positions): # pylint: disable=arguments-renamed self.positions = positions self.graph.update_coordinates() @@ -687,9 +826,12 @@ def main(): from os.path import join, dirname network = read_pajek(join(dirname(dirname(__file__)), 'networks', 'leu_by_genesets.net')) + # network = read_pajek( + # join(dirname(dirname(__file__)), 'networks', 'dicty_publication.net')) #network = read_pajek(join(dirname(dirname(__file__)), 'networks', 'davis.net')) #transform_data_to_orange_table(network) WidgetPreview(OWNxExplorer).run(set_graph=network) + if __name__ == "__main__": main() diff --git a/orangecontrib/network/widgets/tests/test_OWNxExplorer.py b/orangecontrib/network/widgets/tests/test_OWNxExplorer.py index b2cd7db6..3048d96c 100644 --- a/orangecontrib/network/widgets/tests/test_OWNxExplorer.py +++ b/orangecontrib/network/widgets/tests/test_OWNxExplorer.py @@ -3,9 +3,10 @@ import numpy as np -from orangecontrib.network.widgets.tests.utils import NetworkTest +import Orange from orangewidget.tests.utils import simulate +from orangecontrib.network.widgets.tests.utils import NetworkTest from orangecontrib.network import Network from orangecontrib.network.widgets.OWNxExplorer import OWNxExplorer @@ -23,6 +24,12 @@ def test_minimum_size(self): # Disable this test from the base test class pass + @unittest.skipIf(Orange.__version__ < "3.38", "3.36 is not released yet") + def test_remove_allot(self): + self.fail( + "If https://github.com/biolab/orange3/pull/6612 is merged and released, " + "import allot from Orange.util and remove the class from the add-on.") + class TestOWNxExplorerWithLayout(TestOWNxExplorer): def test_empty_network(self): From d041e731b7b25a1a42433ddd8a03e517d59a9839 Mon Sep 17 00:00:00 2001 From: janezd Date: Tue, 13 Feb 2024 11:44:58 +0100 Subject: [PATCH 3/3] Network Explorer: Fix Qt6 incompatibility (float width) --- orangecontrib/network/widgets/graphview.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/orangecontrib/network/widgets/graphview.py b/orangecontrib/network/widgets/graphview.py index 236aaff7..638a49bc 100644 --- a/orangecontrib/network/widgets/graphview.py +++ b/orangecontrib/network/widgets/graphview.py @@ -14,22 +14,27 @@ class PlotVarWidthCurveItem(pg.PlotCurveItem): def __init__(self, directed, *args, **kwargs): self.directed = directed - self.widths = kwargs.pop("widths", None) + self.__setWidths(kwargs.pop("widths", None)) self.setPen(kwargs.pop("pen", pg.mkPen(0.0))) self.sizes = kwargs.pop("size", None) self.coss = self.sins = None super().__init__(*args, **kwargs) def setWidths(self, widths): - self.widths = widths + self.__setWidths(widths) self.update() + def __setWidths(self, widths): + if widths is not None: + widths = np.ceil(widths).astype(int) + self.widths = widths + def setPen(self, pen): self.pen = pen self.pen.setCapStyle(Qt.RoundCap) def setData(self, *args, **kwargs): - self.widths = kwargs.pop("widths", self.widths) + self.__setWidths(kwargs.pop("widths", self.widths)) self.setPen(kwargs.pop("pen", self.pen)) self.sizes = kwargs.pop("size", self.sizes) super().setData(*args, **kwargs)