Live plotting

qcodes/__init__.py

@@ -4,6 +4,15 @@
 from qcodes.station import Station
 from qcodes.loops import get_bg, halt_bg, Loop, Task, Wait
 
+# hack for code that should only be imported into the main (notebook) thread
+# see: http://stackoverflow.com/questions/15411967
+# in particular, importing matplotlib in the side processes takes a long
+# time and spins up other processes in order to try and get a front end
+import sys as _sys
+if 'ipy' in repr(_sys.stdout):
+    from qcodes.plots.matplotlib import MatPlot
+    from qcodes.plots.pyqtgraph import QtPlot
+
 from qcodes.data.manager import get_data_manager
 from qcodes.data.data_set import DataMode, DataSet
 from qcodes.data.data_array import DataArray

qcodes/data/data_array.py

@@ -1,8 +1,6 @@
 import numpy as np
 import collections
 
-import qcodes
-
 
 class DataArray(object):
     '''
@@ -37,6 +35,10 @@ def __init__(self, parameter=None, name=None, label=None, array_id=None,
         self.size = size
         self._preset = False
 
+        # store a reference up to the containing DataSet
+        # this also lets us make sure a DataArray is only in one DataSet
+        self._data_set = None
+
         self.data = None
         if preset_data is not None:
             self.init_data(preset_data)
@@ -47,6 +49,18 @@ def __init__(self, parameter=None, name=None, label=None, array_id=None,
         self.last_saved_index = None
         self.modified_range = None
 
+    @property
+    def data_set(self):
+        return self._data_set
+
+    @data_set.setter
+    def data_set(self, new_data_set):
+        if (self._data_set is not None and
+                new_data_set is not None and
+                self._data_set != new_data_set):
+            raise RuntimeError('A DataArray can only be part of one DataSet')
+        self._data_set = new_data_set
+
     def nest(self, size, action_index=None, set_array=None):
         '''
         nest this array inside a new outer loop
@@ -162,6 +176,13 @@ def __getattr__(self, key):
 
         return getattr(self.data, key)
 
+    def __len__(self):
+        '''
+        must be explicitly delegated to, because len() will look for this
+        attribute to already exist
+        '''
+        return len(self.data)
+
     def _flat_index(self, indices, index_fill):
         indices = indices + index_fill[len(indices):]
         return np.ravel_multi_index(tuple(zip(indices)), self.size)[0]

qcodes/data/data_set.py

@@ -168,6 +168,8 @@ def sync(self):
         '''
         synchronize this data set with a possibly newer version either
         in storage or on the DataServer, depending on its mode
+
+        returns: boolean, is this DataSet live on the server
         '''
         # TODO: sync implies bidirectional... and it could be!
         # we should keep track of last sync timestamp and last modification
@@ -179,7 +181,10 @@ def sync(self):
             # LOCAL DataSet - just read it in
             # TODO: compare timestamps to know if we need to read?
             self.read()
-            return
+            return False
+            # TODO - for remote live plotting, maybe set some timestamp
+            # threshold and call it static after it's been dormant a long time?
+            # I'm thinking like a minute, or ten? Maybe it's configurable?
 
         with self.data_manager.query_lock:
             if self.is_on_server:
@@ -198,22 +203,35 @@ def sync(self):
                     # but the DataSet is still on the server,
                     # so we got the data, and don't need to read.
                     self.mode = DataMode.LOCAL
-                return
+                    return False
+                return True
             else:
                 # this DataSet *thought* it was on the server, but it wasn't,
                 # so we haven't synced yet and need to read from storage
                 self.mode = DataMode.LOCAL
                 self.read()
+                return False
 
     def add_array(self, data_array):
         '''
         add one DataArray to this DataSet
+
+        note: DO NOT just set data_set.arrays[id] = data_array
+        because this will not check for overriding, nor set the
+        reference back to this DataSet. It would also allow you to
+        load the array in with different id than it holds itself.
+
         '''
+        # TODO: mask self.arrays so you *can't* set it directly
+
         if data_array.array_id in self.arrays:
             raise ValueError('array_id {} already exists in this '
                              'DataSet'.format(data_array.array_id))
         self.arrays[data_array.array_id] = data_array
 
+        # back-reference to the DataSet
+        data_array.data_set = self
+
     def _clean_array_ids(self, arrays):
         '''
         replace action_indices tuple with compact string array_ids

qcodes/data/format.py

@@ -283,7 +283,7 @@ def read_one_file(self, data_set, f, ids_read):
                 set_array = DataArray(label=labels[i], array_id=array_id,
                                       set_arrays=set_arrays, size=set_size)
                 set_array.init_data()
-                arrays[array_id] = set_array
+                data_set.add_array(set_array)
 
             set_arrays = set_arrays + (set_array, )
             ids_read.add(array_id)
@@ -300,7 +300,7 @@ def read_one_file(self, data_set, f, ids_read):
                 data_array = DataArray(label=labels[i], array_id=array_id,
                                        set_arrays=set_arrays, size=size)
                 data_array.init_data()
-                arrays[array_id] = data_array
+                data_set.add_array(data_array)
             data_arrays.append(data_array)
             ids_read.add(array_id)
 

qcodes/plots/base.py

@@ -0,0 +1,180 @@
+'''
+Live plotting in Jupyter notebooks
+'''
+from IPython.display import display
+
+from qcodes.widgets.widgets import HiddenUpdateWidget
+
+
+class BasePlot(object):
+    '''
+    create an auto-updating plot connected to a Jupyter notebook
+
+    interval: period in seconds between update checks
+        default 1
+
+    data_keys: which keys in trace config can contain data
+        that we should look for updates in.
+        default 'xyz' (treated as a sequence) but add more if
+        for example marker size or color can contain data
+    '''
+    def __init__(self, interval=1, data_keys='xyz'):
+        self.data_keys = data_keys
+        self.traces = []
+        self.data_updaters = set()
+
+        self.interval = interval
+        self.update_widget = HiddenUpdateWidget(self.update, interval)
+        display(self.update_widget)
+
+    def add(self, *args, updater=None, **kwargs):
+        '''
+        add one trace to this plot
+
+        args: optional way to provide x/y/z data without keywords
+            If the last one is 1D, may be `y` or `x`, `y`
+            If the last one is 2D, may be `z` or `x`, `y`, `z`
+
+        updater: a callable (with no args) that updates the data in this trace
+            if omitted, we will look for DataSets referenced in this data, and
+            call their sync methods.
+
+        kwargs: after inserting info found in args and possibly in set_arrays
+            into `x`, `y`, and optionally `z`, these are passed along to
+            self.add_to_plot
+        '''
+        self.expand_trace(args, kwargs)
+        self.add_to_plot(**kwargs)
+        self.add_updater(updater, kwargs)
+
+    def add_to_plot(self, **kwargs):
+        '''
+        add a trace the plot itself (typically called by self.add,
+        which incorporates args into kwargs, so the subclass doesn't
+        need to worry about this). Data will be in `x`, `y`, and optionally
+        `z`
+
+        should be implemented by a subclass, and each call should append
+        a dictionary to self.traces, containing at least {'config': kwargs}
+        '''
+        raise NotImplementedError
+
+    def add_updater(self, updater, plot_config):
+        if updater is not None:
+            self.data_updaters.add(updater)
+        else:
+            for key in self.data_keys:
+                data_array = plot_config.get(key, '')
+                if hasattr(data_array, 'data_set'):
+                    self.data_updaters.add(data_array.data_set.sync)
+
+        if self.data_updaters:
+            self.update_widget.interval = self.interval
+
+    def get_default_title(self):
+        '''
+        the default title for a plot is just a list of DataSet locations
+        '''
+        title_parts = []
+        for trace in self.traces:
+            config = trace['config']
+            for part in self.data_keys:
+                data_array = config.get(part, '')
+                if hasattr(data_array, 'data_set'):
+                    location = data_array.data_set.location
+                    if location not in title_parts:
+                        title_parts.append(location)
+        return ', '.join(title_parts)
+
+    def get_label(self, data_array):
+        '''
+        look for a label, falling back on name.
+        '''
+        return (getattr(data_array, 'label', '') or
+                getattr(data_array, 'name', ''))
+
+    def expand_trace(self, args, kwargs):
+        '''
+        the x, y (and potentially z) data for a trace may be provided
+        as positional rather than keyword args. The allowed forms are
+        [y] or [x, y] if the last arg is 1D, and
+        [z] or [x, y, z] if the last arg is 2D
+
+        also, look in the main data array (`z` if it exists, else `y`)
+        for set_arrays that can provide the `x` (and potentially `y`) data
+
+        even if we allow data in other attributes (like marker size/color)
+        by providing a different self.data_keys, set_arrays will only
+        contribute x from y, or x & y from z, so we don't use data_keys here
+        '''
+        if args:
+            if hasattr(args[-1][0], '__len__'):
+                # 2D (or higher... but ignore this for now)
+                # this test works for both numpy arrays and bare sequences
+                axletters = 'xyz'
+                ndim = 2
+            else:
+                axletters = 'xy'
+                ndim = 1
+
+            if len(args) not in (1, len(axletters)):
+                raise ValueError('{}D data needs 1 or {} unnamed args'.format(
+                    ndim, len(axletters)))
+
+            arg_axletters = axletters[-len(args):]
+
+            for arg, arg_axletters in zip(args, arg_axletters):
+                if arg_axletters in kwargs:
+                    raise ValueError(arg_axletters + ' data provided twice')
+                kwargs[arg_axletters] = arg
+
+        # reset axletters, we may or may not have found them above
+        axletters = 'xyz' if 'z' in kwargs else 'xy'
+        main_data = kwargs[axletters[-1]]
+        if hasattr(main_data, 'set_arrays'):
+            num_axes = len(axletters) - 1
+            # things will probably fail if we try to plot arrays of the
+            # wrong dimension... but we'll give it a shot anyway.
+            set_arrays = main_data.set_arrays[-num_axes:]
+            # for 2D: y is outer loop, which is earlier in set_arrays,
+            # and x is the inner loop... is this the right convention?
+            set_axletters = reversed(axletters[:-1])
+            for axletter, set_array in zip(set_axletters, set_arrays):
+                if axletter not in kwargs:
+                    kwargs[axletter] = set_array
+
+    def update(self):
+        '''
+        update the data in this plot, using the updaters given with
+        MatPlot.add() or in the included DataSets, then include this in
+        the plot
+
+        this is a wrapper routine that the update widget calls,
+        inside this we call self.update() which should be subclassed
+        '''
+        any_updates = False
+        for updater in self.data_updaters:
+            updates = updater()
+            if updates is not False:
+                any_updates = True
+
+        self.update_plot()
+
+        # once all updaters report they're finished (by returning exactly
+        # False) we stop updating the plot.
+        if any_updates is False:
+            self.halt()
+
+    def update_plot(self):
+        '''
+        update the plot itself (typically called by self.update).
+        should be implemented by a subclass
+        '''
+        raise NotImplementedError
+
+    def halt(self):
+        '''
+        stop automatic updates to this plot, by canceling its update widget
+        '''
+        if hasattr(self, 'update_widget'):
+            self.update_widget.halt()