finos · texodus · Jul 14, 2022 · Jul 4, 2022 · Jul 4, 2022 · Jul 10, 2022
diff --git a/cpp/perspective/src/cpp/arrow_csv.cpp b/cpp/perspective/src/cpp/arrow_csv.cpp
@@ -78,7 +78,8 @@ namespace apachearrow {
     public:
         bool
         operator()(const char* s, size_t length, arrow::TimeUnit::type out_unit,
-            int64_t* out, bool* out_zone_offset_present = NULLPTR) const override {
+            int64_t* out,
+            bool* out_zone_offset_present = NULLPTR) const override {
             size_t endptr;
             std::string val(s, s + length);
             int64_t value
@@ -142,7 +143,8 @@ namespace apachearrow {
     public:
         bool
         operator()(const char* s, size_t length, arrow::TimeUnit::type unit,
-            int64_t* out, bool* out_zone_offset_present = NULLPTR) const override {
+            int64_t* out,
+            bool* out_zone_offset_present = NULLPTR) const override {
 
             if (!arrow::internal::ParseTimestampISO8601(s, length, unit, out)) {
                 if (s[length - 1] == 'Z') {
@@ -247,6 +249,7 @@ namespace apachearrow {
         auto convert_options = arrow::csv::ConvertOptions::Defaults();
 
         read_options.use_threads = false;
+        parse_options.newlines_in_values = true;
 
         if (is_update) {
             convert_options.column_types = std::move(schema);

diff --git a/examples/blocks/gists.json b/examples/blocks/gists.json
@@ -3,11 +3,12 @@
     "olympics": "efd4a857aca9a52ab6cddbb6e1f701c9",
     "custom": "c42f3189699bd29cf20bbe7dce767b07",
     "editable": "45b868833c9f456bd39a51e606412c5d",
+    "evictions": "fc0cdabe27dc121a1a4038545a9e9b23",
     "streaming": "9bec2f8041471bafc2c56db2272a9381",
     "csv": "02d8fd10aef21b19d6165cf92e43e668",
     "iex": "eb151fdd9f98bde987538cbc20e003f6",
     "citibike": "bc8d7e6f72e09c9dbd7424b4332cacad",
     "movies": "6b4dcebf65db4ebe4fe53a6de5ea0b48",
     "fractal": "5485f6b630b08d38218822e507f09f21",
     "covid": "e074d7d9e5783e680d35f565d2b4b32e"
-}
+}
diff --git a/examples/blocks/src/citibike/index.html b/examples/blocks/src/citibike/index.html
@@ -8,6 +8,7 @@
         <script src="/node_modules/@finos/perspective-workspace/dist/umd/perspective-workspace.js"></script>
         <script src="/node_modules/@finos/perspective-viewer-datagrid/dist/umd/perspective-viewer-datagrid.js"></script>
         <script src="/node_modules/@finos/perspective-viewer-d3fc/dist/umd/perspective-viewer-d3fc.js"></script>
+        <script src="/node_modules/@finos/perspective-viewer-openlayers/dist/umd/perspective-viewer-openlayers.js"></script>
         <script src="/node_modules/@finos/perspective/dist/umd/perspective.js"></script>
 
         <script src="citibike.js"></script>

diff --git a/examples/blocks/src/csv/index.html b/examples/blocks/src/csv/index.html
@@ -7,6 +7,7 @@
         <script src="/node_modules/@finos/perspective-viewer/dist/umd/perspective-viewer.js"></script>
         <script src="/node_modules/@finos/perspective-viewer-datagrid/dist/umd/perspective-viewer-datagrid.js"></script>
         <script src="/node_modules/@finos/perspective-viewer-d3fc/dist/umd/perspective-viewer-d3fc.js"></script>
+        <script src="/node_modules/@finos/perspective-viewer-openlayers/dist/umd/perspective-viewer-openlayers.js"></script>
 
         <script src="csv.js"></script>
 

diff --git a/examples/blocks/src/evictions/.block b/examples/blocks/src/evictions/.block
@@ -0,0 +1 @@
+license: apache-2.0
diff --git a/examples/blocks/src/evictions/README.md b/examples/blocks/src/evictions/README.md
@@ -0,0 +1 @@
+Demo of [Perspective](https://github.com/finos/perspective), using SF eviciton data from 1997-present provided by [DataSF](https://data.sfgov.org/Housing-and-Buildings/Eviction-Notices/5cei-gny5).
diff --git a/examples/blocks/src/evictions/index.html b/examples/blocks/src/evictions/index.html
@@ -0,0 +1,123 @@
+<!--
+
+   Copyright (c) 2017, the Perspective Authors.
+
+   This file is part of the Perspective library, distributed under the terms of
+   the Apache License 2.0.  The full license can be found in the LICENSE file.
+
+-->
+
+<!DOCTYPE html>
+<html>
+    <head>
+        <meta
+            name="viewport"
+            content="width=device-width,initial-scale=1,maximum-scale=1,minimum-scale=1,user-scalable=no"
+        />
+
+        <script
+            type="module"
+            src="/node_modules/@finos/perspective/dist/cdn/perspective.js"
+        ></script>
+        <script
+            type="module"
+            src="/node_modules/@finos/perspective-viewer/dist/cdn/perspective-viewer.js"
+        ></script>
+        <script
+            type="module"
+            src="/node_modules/@finos/perspective-viewer-datagrid/dist/cdn/perspective-viewer-datagrid.js"
+        ></script>
+        <script
+            type="module"
+            src="/node_modules/@finos/perspective-viewer-d3fc/dist/cdn/perspective-viewer-d3fc.js"
+        ></script>
+        <script
+            type="module"
+            src="/node_modules/@finos/perspective-viewer-openlayers/dist/cdn/perspective-viewer-openlayers.js"
+        ></script>
+        <script src="https://cdn.jsdelivr.net/npm/[email protected]/umd/index.min.js"></script>
+
+        <link
+            rel="stylesheet"
+            crossorigin="anonymous"
+            href="/node_modules/@finos/perspective-viewer/dist/css/themes.css"
+        />
+
+        <script type="module">
+            import {worker} from "/node_modules/@finos/perspective/dist/cdn/perspective.js";
+
+            const WORKER = worker();
+            const URL = `https://data.sfgov.org/resource/5cei-gny5.csv?$limit=50000`;
+
+            async function get_evictions() {
+                const arrow = localStorage.getItem(URL);
+                if (arrow !== null) {
+                    console.log("Using cached data from localStorage");
+                    try {
+                        const buffer = fflate.strToU8(arrow, true);
+                        return await WORKER.table(
+                            fflate.decompressSync(buffer).buffer
+                        );
+                    } catch (e) {
+                        console.error("Can't load cached data", e);
+                        localStorage.clear();
+                    }
+                }
+
+                console.log("Downloading data from data.sfgov.org");
+                const resp = await fetch(
+                    URL
+                    // "5cei-gny5.csv"
+                    // "evictions.all.arrow"
+                );
+                const csv = await resp.text();
+                const table = await WORKER.table(csv);
+                (async () => {
+                    console.log("Caching data");
+                    const view = await table.view();
+                    const arrow = await view.to_arrow();
+                    try {
+                        const x = fflate.compressSync(new Uint8Array(arrow));
+                        const zipped = fflate.strFromU8(x, true);
+                        localStorage.setItem(URL, zipped);
+                    } catch (e) {
+                        console.error(
+                            "Can't cache data from data.sfgov.org",
+                            e
+                        );
+                    }
+                })();
+
+                return table;
+            }
+
+            async function load() {
+                const el =
+                    document.getElementsByTagName("perspective-viewer")[0];
+
+                const evictions = get_evictions();
+                const layout_json = await fetch("layout.json");
+                const layout = await layout_json.json();
+
+                el.load(await evictions);
+                el.restore(layout);
+            }
+
+            load();
+        </script>
+
+        <style>
+            perspective-viewer {
+                position: absolute;
+                top: 0;
+                left: 0;
+                bottom: 0;
+                right: 0;
+            }
+        </style>
+    </head>
+
+    <body>
+        <perspective-viewer editable> </perspective-viewer>
+    </body>
+</html>
diff --git a/examples/blocks/src/evictions/layout.json b/examples/blocks/src/evictions/layout.json
@@ -0,0 +1,18 @@
+{
+    "plugin": "Map Scatter",
+    "plugin_config": {},
+    "settings": true,
+    "theme": "Material Dark",
+    "group_by": ["bucket lon", "bucket lat"],
+    "split_by": ["neighborhood"],
+    "columns": ["bucket lon", "bucket lat", null, null, null],
+    "filter": [["lon", "is not null", null]],
+    "sort": [],
+    "expressions": [
+        "// lon\nvar x[2];\nindexof(\"shape\", ' .+?( )', x);\nvar y := substring(\"shape\", 7, x[1] - 7);\nfloat(y)",
+        "// lat\nvar x[2];\nindexof(\"shape\", ' .+?( )', x);\nvar y := substring(\"shape\", x[0], length(\"shape\") - x[1]);\nfloat(y)",
+        "// bucket lon\nvar x[2];\nindexof(\"shape\", ' .+?( )', x);\nvar y := substring(\"shape\", 7, x[1] - 7);\nbucket(float(y), 0.0025) + 0.00125",
+        "// bucket lat\nvar x[2];\nindexof(\"shape\", ' .+?( )', x);\nvar y := substring(\"shape\", x[0], length(\"shape\") - x[1]);\nbucket(float(y), 0.0025) + 0.00125"
+    ],
+    "aggregates": {"lon": "avg", "bucket lon": "avg", "bucket lat": "avg"}
+}
diff --git a/packages/perspective-jupyterlab/package.json b/packages/perspective-jupyterlab/package.json
@@ -35,6 +35,7 @@
         "@finos/perspective-viewer": "^1.5.1",
         "@finos/perspective-viewer-d3fc": "^1.5.1",
         "@finos/perspective-viewer-datagrid": "^1.5.1",
+        "@finos/perspective-viewer-openlayers": "^1.5.1",
         "@jupyter-widgets/base": "^4.1.0",
         "@jupyterlab/application": "^3.3.2",
         "@lumino/application": "^1.27.0",

diff --git a/packages/perspective-jupyterlab/src/js/index.js b/packages/perspective-jupyterlab/src/js/index.js
@@ -17,6 +17,7 @@ export * from "./widget";
 import "../less/index.less";
 import "@finos/perspective-viewer-datagrid";
 import "@finos/perspective-viewer-d3fc";
+import "@finos/perspective-viewer-openlayers";
 import {perspectiveRenderers} from "./renderer";
 import {PerspectiveJupyterPlugin} from "./plugin";
 

diff --git a/packages/perspective-viewer-openlayers/build.js b/packages/perspective-viewer-openlayers/build.js
@@ -0,0 +1,32 @@
+const {NodeModulesExternal} = require("@finos/perspective-build/external");
+const {InlineCSSPlugin} = require("@finos/perspective-build/inline_css");
+const {UMDLoader} = require("@finos/perspective-build/umd");
+const {build} = require("@finos/perspective-build/build");
+
+const BUILD = [
+    {
+        entryPoints: ["src/js/plugin/plugin.js"],
+        plugins: [InlineCSSPlugin(), NodeModulesExternal()],
+        format: "esm",
+        outfile: "dist/esm/perspective-viewer-openlayers.js",
+    },
+    {
+        entryPoints: ["src/js/plugin/plugin.js"],
+        globalName: "perspective_openlayers",
+        plugins: [InlineCSSPlugin(), UMDLoader()],
+        format: "cjs",
+        outfile: "dist/umd/perspective-viewer-openlayers.js",
+    },
+    {
+        entryPoints: ["src/js/plugin/plugin.js"],
+        plugins: [InlineCSSPlugin()],
+        format: "esm",
+        outfile: "dist/cdn/perspective-viewer-openlayers.js",
+    },
+];
+
+async function build_all() {
+    await Promise.all(BUILD.map(build)).catch(() => process.exit(1));
+}
+
+build_all();
diff --git a/packages/perspective-viewer-openlayers/package.json b/packages/perspective-viewer-openlayers/package.json
@@ -0,0 +1,30 @@
+{
+    "name": "@finos/perspective-viewer-openlayers",
+    "version": "1.5.1",
+    "main": "src/js/plugin/plugin.js",
+    "files": [
+        "dist/**/*"
+    ],
+    "author": "",
+    "license": "Apache-2.0",
+    "scripts": {
+        "bench": "npm-run-all bench:build bench:run",
+        "bench:build": ":",
+        "bench:run": ":",
+        "build": "node build.js",
+        "test:build": "cpy \"test/html/*\" dist/umd",
+        "test:run": "jest --rootDir=. --config=../../tools/perspective-test/jest.config.js --color",
+        "test": "npm-run-all test:build test:run",
+        "watch": ":",
+        "clean": "rimraf dist",
+        "clean:screenshots": "rimraf \"test/screenshots/**/*.@(failed|diff).png\""
+    },
+    "dependencies": {
+        "@finos/perspective": "1.5.1",
+        "@finos/perspective-viewer": "1.5.1",
+        "d3": "^7.1.1",
+        "gradient-parser": "1.0.2",
+        "less": "^4.1.0",
+        "ol": "^5.3.2"
+    }
+}
diff --git a/packages/perspective-viewer-openlayers/src/js/data/data.js b/packages/perspective-viewer-openlayers/src/js/data/data.js
@@ -0,0 +1,54 @@
+/******************************************************************************
+ *
+ * Copyright (c) 2017, the Perspective Authors.
+ *
+ * This file is part of the Perspective library, distributed under the terms of
+ * the Apache License 2.0.  The full license can be found in the LICENSE file.
+ *
+ */
+
+export function getMapData(config) {
+    const points = [];
+    // const cols = config.columns.map(x => )
+    config.data.forEach((row, i) => {
+        // Exclude "total" rows that don't have all values
+        const groupCount = row.__ROW_PATH__ ? row.__ROW_PATH__.length : 0;
+        if (groupCount < config.group_by.length) {
+            return;
+        }
+
+        // Get the group from the row path
+        const group = row.__ROW_PATH__ ? row.__ROW_PATH__.join("|") : `${i}`;
+        const rowPoints = {};
+
+        // Split the rest of the row into a point for each category
+        Object.keys(row)
+            .filter((key) => key !== "__ROW_PATH__" && row[key] !== null)
+            .forEach((key) => {
+                const split = key.split("|");
+                const category =
+                    split.length > 1
+                        ? split.slice(0, split.length - 1).join("|")
+                        : "__default__";
+                rowPoints[category] = rowPoints[category] || {group, row};
+                rowPoints[category][split[split.length - 1]] = row[key];
+            });
+
+        // Add the points for this row to the data set
+        Object.keys(rowPoints).forEach((key) => {
+            const rowPoint = rowPoints[key];
+            const cols = config.real_columns.map((c) =>
+                c ? rowPoint[c] : null
+            );
+
+            points.push({
+                cols,
+                group: rowPoint.group,
+                row: rowPoint.row,
+                category: key,
+            });
+        });
+    });
+
+    return points;
+}