Merge pull request #547 from jpmorganchase/remote-arrow

Remote arrow
finos · Apr 19, 2019 · 52d134e · 52d134e
2 parents 529e9b5 + 965363e
commit 52d134e
Show file tree

Hide file tree

Showing 8 changed files with 109 additions and 14 deletions.
diff --git a/docs/md/usage.md b/docs/md/usage.md
@@ -432,21 +432,30 @@ const fs = require("fs");
 // module's directory.
 const host = new WebSocketHost({ assets: [__dirname], port: 8080 });
 
-// Read an arrow file from the file system and load it as a named data source.
+// Read an arrow file from the file system and load it as a named table.
 const arr = fs.readFileSync(__dirname + "/superstore.arrow");
-host.open("data_source_one", table(arr));
+const tbl = table(arr);
+host.host_table("table_one", tbl);
+
+// Or host a view 
+const view = tbl.view({filter: [["State", "==", "Texas"]]});
+host.host_view("view_one", view);
 ```
 
 In the browser:
 
 ```javascript
-var elem = document.getElementsByTagName("perspective-viewer")[0];
+const elem = document.getElementsByTagName("perspective-viewer")[0];
 
 // Bind to the server's worker instead of instantiating a Web Worker.
-var worker = perspective.worker(window.location.origin.replace("http", "ws"));
+const worker = perspective.worker(window.location.origin.replace("http", "ws"));
 
 // Bind the viewer to the preloaded data source.
-elem.load(worker.open("data_source_one"));
+elem.load(worker.open_table("table_one"));
+
+// Or load data from a view
+const arrow = await worker.open_view("view_one").to_arrow();
+elem.load(arrow);
 ```
 
 `<perspective-viewer>` instances bound in this way are otherwise no different

diff --git a/examples/git_history/chained.html b/examples/git_history/chained.html
@@ -0,0 +1,52 @@
+<!--
+   
+   Copyright (c) 2017, the Perspective Authors.
+   
+   This file is part of the Perspective library, distributed under the terms of
+   the Apache License 2.0.  The full license can be found in the LICENSE file.
+
+-->
+
+<!DOCTYPE html>
+<html>
+
+<head>
+
+    <meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1, minimum-scale=1, user-scalable=no">
+
+    <script src="perspective.view.js"></script>
+    <script src="hypergrid.plugin.js"></script>
+    <script src="highcharts.plugin.js"></script>
+
+    <script src="perspective.js"></script>
+
+    <link rel='stylesheet' href="index.css">
+    <link rel='stylesheet' href="material.css">
+
+</head>
+
+<body>
+
+    <perspective-viewer 
+        id="view1"
+        view="y_area" 
+        row-pivots='["week_bucket(Date)"]' 
+        column-pivots='["Email"]'
+        columns='["Hash"]'
+        computed-columns='[{"name":"week_bucket(Date)","inputs":["Date"],"func":"week_bucket"}]'>
+
+    </perspective-viewer>
+
+    <script>
+        window.addEventListener('WebComponentsReady', async function() {
+            var elem = document.getElementById('view1');
+            var worker = perspective.worker(window.location.origin.replace('http', 'ws'));
+            let view = worker.open_view('data_source_one');
+            let arrow = await view.to_arrow();
+            elem.load(perspective.worker().table(arrow));
+        });
+    </script>
+
+</body>
+
+</html>
diff --git a/examples/git_history/server.js b/examples/git_history/server.js
@@ -18,5 +18,6 @@ function execute(command, callback) {
 
 execute(`git log --date=iso --pretty=format:'"%h","%an","%aD","%s","%ae"'`, log => {
     const host = new WebSocketHost({assets: [__dirname]});
-    host.open("data_source_one", table("Hash,Name,Date,Message,Email\n" + log));
+    const tbl = table("Hash,Name,Date,Message,Email\n" + log);
+    host.host_view("data_source_one", tbl.view());
 });
diff --git a/packages/perspective-cli/src/html/index.html b/packages/perspective-cli/src/html/index.html
@@ -49,7 +49,7 @@
         window.addEventListener('WebComponentsReady', function() {
             var elem = document.getElementById('view1');
             var worker = perspective.worker(window.location.origin.replace('http', 'ws'));
-            elem.load(worker.open('data_source_one'));
+            elem.load(worker.open_table('data_source_one'));
         });
     </script>
 </body>

diff --git a/packages/perspective-cli/src/js/index.js b/packages/perspective-cli/src/js/index.js
@@ -74,7 +74,7 @@ async function host(filename, options) {
     } else {
         file = await read_stdin();
     }
-    server.open("data_source_one", table(file));
+    server.host_table("data_source_one", table(file));
     if (options.open) {
         open_browser(options.port);
     }

diff --git a/packages/perspective/src/js/api.js b/packages/perspective/src/js/api.js
@@ -84,6 +84,13 @@ function view(worker, table_name, config) {
     bindall(this);
 }
 
+function proxy_view(worker, name) {
+    this._worker = worker;
+    this._name = name;
+}
+
+proxy_view.prototype = view.prototype;
+
 view.prototype.get_config = async_queue("get_config");
 
 view.prototype.to_json = async_queue("to_json");
@@ -251,10 +258,14 @@ worker.prototype.send = function() {
     throw new Error("post() not implemented");
 };
 
-worker.prototype.open = function(name) {
+worker.prototype.open_table = function(name) {
     return new proxy_table(this, name);
 };
 
+worker.prototype.open_view = function(name) {
+    return new proxy_view(this, name);
+};
+
 let _initialized = false;
 
 worker.prototype._handle = function(e) {

diff --git a/packages/perspective/src/js/perspective.node.js b/packages/perspective/src/js/perspective.node.js
@@ -186,14 +186,24 @@ class WebSocketHost extends module.exports.Host {
         });
     }
 
-    post(msg) {
-        this.REQS[msg.id].send(JSON.stringify(msg));
+    post(msg, transferable) {
+        if (transferable) {
+            msg.is_transferable = true;
+            this.REQS[msg.id].send(JSON.stringify(msg));
+            this.REQS[msg.id].send(transferable[0]);
+        } else {
+            this.REQS[msg.id].send(JSON.stringify(msg));
+        }
         delete this.REQS[msg.id];
     }
 
-    open(name, table) {
+    host_table(name, table) {
         this._tables[name] = table;
-        table.view({aggregate: []});
+        table.view({columns: []});
+    }
+
+    host_view(name, view) {
+        this._views[name] = view;
     }
 
     close() {

diff --git a/packages/perspective/src/js/perspective.parallel.js b/packages/perspective/src/js/perspective.parallel.js
@@ -116,14 +116,26 @@ class WebSocketWorker extends worker {
             pingTimeout: 15000,
             pingMsg: "heartbeat"
         });
+        this._ws.ws.binaryType = "arraybuffer";
         this._ws.onopen = () => {
             this.send({id: -1, cmd: "init"});
         };
         this._ws.onmessage = msg => {
             if (msg.data === "heartbeat") {
                 return;
             }
-            this._handle({data: JSON.parse(msg.data)});
+            if (this._pending_arrow) {
+                this._handle({data: {id: this._pending_arrow, data: msg.data}});
+                delete this._pending_arrow;
+            } else {
+                msg = JSON.parse(msg.data);
+                if (msg.is_transferable) {
+                    console.warn("Arrow transfer detected!");
+                    this._pending_arrow = msg.id;
+                } else {
+                    this._handle({data: msg});
+                }
+            }
         };
     }