Merge pull request #813 from finos/numpy-client

Client mode supports dataframes, np.ndarray, structured and recarray

cpp/perspective/src/cpp/emscripten.cpp

@@ -778,7 +778,7 @@ namespace binding {
             }
 
             double fval = item.as<double>();
-            if (isnan(fval)) {
+            if (!is_update && isnan(fval)) {
                 std::cout << "Promoting to string" << std::endl;
                 tbl.promote_column(name, DTYPE_STR, i, false);
                 col = tbl.get_column(name);
@@ -823,13 +823,13 @@ namespace binding {
                     // float value in an inferred column. Would not be needed if the type
                     // inference checked the entire column/we could reset parsing.
                     double fval = item.as<double>();
-                    if (fval > 2147483647 || fval < -2147483648) {
+                    if (!is_update && (fval > 2147483647 || fval < -2147483648)) {
                         std::cout << "Promoting to float" << std::endl;
                         tbl.promote_column(name, DTYPE_FLOAT64, i, true);
                         col = tbl.get_column(name);
                         type = DTYPE_FLOAT64;
                         col->set_nth(i, fval);
-                    } else if (isnan(fval)) {
+                    } else if (!is_update && isnan(fval)) {
                         std::cout << "Promoting to string" << std::endl;
                         tbl.promote_column(name, DTYPE_STR, i, false);
                         col = tbl.get_column(name);

packages/perspective-jupyterlab/src/ts/view.ts

@@ -124,11 +124,18 @@ export class PerspectiveView extends DOMWidgetView {
      */
     _handle_message(msg: PerspectiveJupyterMessage) {
         // If in client-only mode (no Table on the python widget), message.data is an object containing "data" and "options". 
-        if (this.pWidget.client === true && msg.data["data"]) {
-            if (msg.data["cmd"] === "update") {
-                this.pWidget._update(msg.data["data"]);
-            } else {
-                this.pWidget.load(msg.data["data"], msg.data["options"]);
+        if (this.pWidget.client === true) {
+            if(msg.data["data"]) {
+                // either an update or a load
+                if (msg.data["cmd"] === "update") {
+                    this.pWidget._update(msg.data["data"]);
+                } else if (msg.data["cmd"] === "replace") {
+                    this.pWidget.replace(msg.data["data"]);
+                } else {
+                    this.pWidget.load(msg.data["data"], msg.data["options"]);
+                }
+            } else if (msg.data["cmd"] == "clear") {
+                this.pWidget.clear();
             }
         } else {
             // Make a deep copy of each message - widget views share the same comm, so mutations on `msg` affect subsequent message handlers.

packages/perspective-phosphor/src/ts/widget.ts

@@ -153,6 +153,23 @@ export class PerspectiveWidget extends Widget {
         this.viewer.update(data);
     }
 
+    /**
+     * Removes all rows from the viewer's table. Does not reset viewer state.
+     */
+    clear(): void {
+        this.viewer.clear();
+    }
+
+    /**
+     * Replaces the data of the viewer's table with new data. New data must conform
+     * to the schema of the Table.
+     * 
+     * @param data
+     */
+    replace(data: TableData): void {
+        this.viewer.replace(data);
+    }
+
     get table(): Table {
         return this.viewer.table;
     }

packages/perspective-viewer/index.d.ts

@@ -7,6 +7,8 @@ declare module '@finos/perspective-viewer' {
         update(data: TableData): void;
         notifyResize(): void;
         delete(): Promise<void>;
+        clear() : void;
+        replace(data: TableData) : void;
         flush(): Promise<void>;
         toggleConfig(): void;
         save(): PerspectiveViewerOptions;

packages/perspective-viewer/src/js/viewer.js

@@ -633,7 +633,6 @@ class PerspectiveViewer extends ActionElement {
      * Reset's this element's view state and attributes to default.  Does not
      * delete this element's `perspective.table` or otherwise modify the data
      * state.
-     *
      */
     reset() {
         this.removeAttribute("row-pivots");

python/perspective/perspective/core/__init__.py

@@ -11,5 +11,6 @@
 from .exception import PerspectiveError  # noqa: F401
 from .manager import PerspectiveManager  # noqa: F401
 from .tornado_handler import PerspectiveTornadoHandler  # noqa: F401
+from .sort import Sort  # noqa: F401
 from .viewer import PerspectiveViewer  # noqa: F401
 from .widget import PerspectiveWidget  # noqa: F401

python/perspective/perspective/core/viewer.py

@@ -144,6 +144,30 @@ def update(self, data):
         '''
         self.table.update(data)
 
+    def clear(self):
+        '''Clears the rows of this viewer's `Table`.'''
+        if self.table is not None:
+            self.table.clear()
+
+    def replace(self, data):
+        '''Replaces the rows of this viewer's `Table` with new data.
+
+        Args:
+            data : new data to set into the table - must conform to the table's schema.
+        '''
+        if self.table is not None:
+            self.table.replace(data)
+
+    def reset(self):
+        '''Resets the viewer's attributes and state, but does not delete or modify the underlying `Table`.'''
+        self.row_pivots = []
+        self.column_pivots = []
+        self.filters = []
+        self.sort = []
+        self.aggregates = {}
+        self.columns = []
+        self.plugin = "hypergrid"
+
     def _new_view(self):
         '''Create a new View, and assign its name to the viewer.
 

python/perspective/perspective/core/widget.py

@@ -50,11 +50,15 @@ def _serialize(data):
             # serialize schema values to string
             if isinstance(v, type):
                 return {k: _type_to_string(data[k]) for k in data}
+            elif isinstance(v, numpy.ndarray):
+                return {k: data[k].tolist() for k in data}
             else:
                 return data
-    elif isinstance(data, numpy.recarray):
-        # flatten numpy record arrays
-        columns = [data[col] for col in data.dtype.names]
+    elif isinstance(data, numpy.ndarray):
+        # structured or record array
+        if not isinstance(data.dtype.names, tuple):
+            raise NotImplementedError("Data should be dict of numpy.ndarray or a structured array.")
+        columns = [data[col].tolist() for col in data.dtype.names]
         return dict(zip(data.dtype.names, columns))
     elif isinstance(data, pandas.DataFrame) or isinstance(data, pandas.Series):
         # take flattened dataframe and make it serializable
@@ -71,16 +75,6 @@ def _serialize(data):
         raise NotImplementedError("Cannot serialize a dataset of `{0}`.".format(str(type(data))))
 
 
-class DateTimeStringEncoder(json.JSONEncoder):
-
-    def default(self, obj):
-        '''Create a stringified representation of a datetime object.'''
-        if isinstance(obj, datetime.datetime):
-            return obj.strftime("%Y-%m-%d %H:%M:%S.%f")
-        else:
-            return super(DateTimeStringEncoder, self).default(obj)
-
-
 class PerspectiveWidget(Widget, PerspectiveViewer):
     '''`PerspectiveWidget` allows for Perspective to be used in the form of a JupyterLab IPython widget.
 
@@ -231,6 +225,25 @@ def update(self, data):
         else:
             super(PerspectiveWidget, self).update(data)
 
+    def clear(self):
+        '''Clears the widget's underlying `Table` - does not function in client mode.'''
+        if self.client is False:
+            super(PerspectiveWidget, self).clear()
+
+    def replace(self, data):
+        '''Replaces the widget's `Table` with new data conforming to the same schema. Does not clear
+        user-set state. If in client mode, serializes the data and sends it to the browser.'''
+        if self.client is True:
+            if isinstance(data, pandas.DataFrame) or isinstance(data, pandas.Series):
+                data, _ = deconstruct_pandas(data)
+            d = _serialize(data)
+            self.post({
+                "cmd": "replace",
+                "data": d
+            })
+        else:
+            super(PerspectiveWidget, self).replace(data)
+
     def post(self, msg, id=None):
         '''Post a serialized message to the `PerspectiveJupyterClient` in the front end.
 

python/perspective/perspective/src/fill.cpp

@@ -235,13 +235,13 @@ _fill_col_numeric(t_data_accessor accessor, t_data_table& tbl,
                 // float value in an inferred column. Would not be needed if the type
                 // inference checked the entire column/we could reset parsing.
                 double fval = item.cast<double>();
-                if (fval > 2147483647 || fval < -2147483648) {
+                if (!is_update && (fval > 2147483647 || fval < -2147483648)) {
                     WARN("Promoting %s to float from int32", name);
                     tbl.promote_column(name, DTYPE_FLOAT64, i, true);
                     col = tbl.get_column(name);
                     type = DTYPE_FLOAT64;
                     col->set_nth(i, fval);
-                } else if (isnan(fval)) {
+                } else if (!is_update && isnan(fval)) {
                     WARN("Promoting column %s to string from int32", name);
                     tbl.promote_column(name, DTYPE_STR, i, false);
                     col = tbl.get_column(name);
@@ -254,7 +254,7 @@ _fill_col_numeric(t_data_accessor accessor, t_data_table& tbl,
             } break;
             case DTYPE_INT64: {
                 double fval = item.cast<double>();
-                if (isnan(fval)) {
+                if (!is_update && isnan(fval)) {
                     WARN("Promoting %s to string from int64", name);
                     tbl.promote_column(name, DTYPE_STR, i, false);
                     col = tbl.get_column(name);
@@ -271,7 +271,7 @@ _fill_col_numeric(t_data_accessor accessor, t_data_table& tbl,
             case DTYPE_FLOAT64: {
                 bool is_float = py::isinstance<py::float_>(item);
                 bool is_numpy_nan = is_float && npy_isnan(item.cast<double>());
-                if (!is_float || is_numpy_nan) {
+                if (!is_update && (!is_float || is_numpy_nan)) {
                     WARN("Promoting column %s to string from float64", name);
                     tbl.promote_column(name, DTYPE_STR, i, false);
                     col = tbl.get_column(name);

python/perspective/perspective/table/table.py

@@ -57,7 +57,7 @@ def clear(self):
         self._table.reset_gnode(self._gnode_id)
 
     def replace(self, data):
-        '''Replaces all rows in the Table with the new data.
+        '''Replaces all rows in the Table with the new data that conforms to the Table's schema.
 
         Args:
             data (dict|list|dataframe) the new data with which to fill the Table

python/perspective/perspective/tests/core/test_viewer.py

@@ -183,4 +183,42 @@ def test_viewer_update_dict_partial(self):
         assert viewer.table.view().to_dict() == {
             "a": [1, 2, 3],
             "b": [8, 9, 10]
-        }
+        }
+
+    # clear
+
+    def test_viewer_clear(self):
+        table = Table({"a": [1, 2, 3]})
+        viewer = PerspectiveViewer()
+        viewer.load(table)
+        viewer.clear()
+        assert viewer.table.size() == 0
+        assert viewer.table.schema() == {
+            "a": int
+        }
+
+    # replace
+
+    def test_viewer_replace(self):
+        table = Table({"a": [1, 2, 3]})
+        viewer = PerspectiveViewer()
+        viewer.load(table)
+        viewer.replace({"a": [4, 5, 6]})
+        assert viewer.table.size() == 3
+        assert viewer.table.schema() == {
+            "a": int
+        }
+        assert viewer.table.view().to_dict() == {
+            "a": [4, 5, 6]
+        }
+
+    # reset
+
+    def test_viewer_reset(self):
+        table = Table({"a": [1, 2, 3]})
+        viewer = PerspectiveViewer(plugin="x_bar", filters=[["a", "==", 2]])
+        viewer.load(table)
+        assert viewer.filters == [["a", "==", 2]]
+        viewer.reset()
+        assert viewer.plugin == "hypergrid"
+        assert viewer.filters == []

python/perspective/perspective/tests/core/test_widget.py

@@ -7,6 +7,7 @@
 #
 import six
 import numpy as np
+import pandas as pd
 from datetime import date, datetime
 from pytest import raises
 from perspective import PerspectiveError, PerspectiveWidget, Table
@@ -65,11 +66,52 @@ def test_widget_pass_options_invalid(self):
     # client-only mode
 
     def test_widget_client(self):
-        data = {"a": np.arange(0, 50)}
+        data = {"a": [i for i in range(50)]}
         widget = PerspectiveWidget(data, client=True)
         assert widget.table is None
         assert widget._data == data
 
+    def test_widget_client_np(self):
+        data = {"a": np.arange(0, 50)}
+        widget = PerspectiveWidget(data, client=True)
+        assert widget.table is None
+        assert widget._data == {
+            "a": [i for i in range(50)]
+        }
+
+    def test_widget_client_df(self):
+        data = pd.DataFrame({
+            "a": np.arange(10),
+            "b": [True for i in range(10)],
+            "c": [str(i) for i in range(10)]
+        })
+        widget = PerspectiveWidget(data, client=True)
+        assert widget.table is None
+        assert widget._data == {
+            "index": [i for i in range(10)],
+            "a": [i for i in range(10)],
+            "b": [True for i in range(10)],
+            "c": [str(i) for i in range(10)]
+        }
+
+    def test_widget_client_np_structured_array(self):
+        data = np.array([(1, 2), (3, 4)], dtype=[("a", "int64"), ("b", "int64")])
+        widget = PerspectiveWidget(data, client=True)
+        assert widget.table is None
+        assert widget._data == {
+            "a": [1, 3],
+            "b": [2, 4]
+        }
+
+    def test_widget_client_np_recarray(self):
+        data = np.array([(1, 2), (3, 4)], dtype=[("a", "int64"), ("b", "int64")]).view(np.recarray)
+        widget = PerspectiveWidget(data, client=True)
+        assert widget.table is None
+        assert widget._data == {
+            "a": [1, 3],
+            "b": [2, 4]
+        }
+
     def test_widget_client_schema(self):
         widget = PerspectiveWidget({
             "a": int,
@@ -114,4 +156,6 @@ def test_widget_client_update(self):
         widget = PerspectiveWidget(data, client=True)
         widget.update(data)
         assert widget.table is None
-        assert widget._data == data
+        assert widget._data == {
+            "a": [i for i in range(50)]
+        }

python/perspective/perspective/tests/table/test_table.py

@@ -404,5 +404,3 @@ def test_table_replace(self):
         tbl = Table(data)
         tbl.replace(data2)
         assert tbl.view().to_records() == data2
-
-