ARROW-1990: [JS] C++ Refactor, Add DataFrame

js/package.json

@@ -12,6 +12,7 @@
     "clean": "gulp clean",
     "debug": "gulp debug",
     "perf": "node ./perf/index.js",
+    "create:perfdata": "python ./test/data/tables/generate.py ./test/data/tables/tracks.arrow",
     "release": "./npm-release.sh",
     "clean:all": "run-p clean clean:testdata",
     "clean:testdata": "gulp clean:testdata",

js/perf/index.js

@@ -16,21 +16,21 @@
 // under the License.
 
 // Use the ES5 UMD target as perf baseline
-// const { Table, readVectors } = require('../targets/es5/umd');
-// const { Table, readVectors } = require('../targets/es5/cjs');
-const { Table, readVectors } = require('../targets/es2015/umd');
-// const { Table, readVectors } = require('../targets/es2015/cjs');
+// const { col, Table, readVectors } = require('../targets/es5/umd');
+// const { col, Table, readVectors } = require('../targets/es5/cjs');
+// const { col, Table, readVectors } = require('../targets/es2015/umd');
+const { col, Table, readVectors } = require('../targets/es2015/cjs');
 
 const config = require('./config');
 const Benchmark = require('benchmark');
 
 const suites = [];
 
 for (let { name, buffers} of config) {
-    const parseSuite = new Benchmark.Suite(`Parse ${name}`, { async: true });
-    const sliceSuite = new Benchmark.Suite(`Slice ${name} vectors`, { async: true });
-    const iterateSuite = new Benchmark.Suite(`Iterate ${name} vectors`, { async: true });
-    const getByIndexSuite = new Benchmark.Suite(`Get ${name} values by index`, { async: true });
+    const parseSuite = new Benchmark.Suite(`Parse "${name}"`, { async: true });
+    const sliceSuite = new Benchmark.Suite(`Slice "${name}" vectors`, { async: true });
+    const iterateSuite = new Benchmark.Suite(`Iterate "${name}" vectors`, { async: true });
+    const getByIndexSuite = new Benchmark.Suite(`Get "${name}" values by index`, { async: true });
     parseSuite.add(createFromTableTest(name, buffers));
     parseSuite.add(createReadVectorsTest(name, buffers));
     for (const vector of Table.from(buffers).columns) {
@@ -41,6 +41,25 @@ for (let { name, buffers} of config) {
     suites.push(getByIndexSuite, iterateSuite, sliceSuite, parseSuite);
 }
 
+for (let {name, buffers, countBys, counts} of require('./table_config')) {
+    const table = Table.from(buffers);
+
+    const dfCountBySuite = new Benchmark.Suite(`DataFrame Count By "${name}"`, { async: true });
+    for (countBy of countBys) {
+        dfCountBySuite.add(createDataFrameCountByTest(table, countBy));
+    }
+
+    const dfFilterCountSuite = new Benchmark.Suite(`DataFrame Filter-Scan Count "${name}"`, { async: true });
+    const dfDirectCountSuite = new Benchmark.Suite(`DataFrame Direct Count "${name}"`, { async: true });
+
+    for (test of counts) {
+        dfFilterCountSuite.add(createDataFrameFilterCountTest(table, test.col, test.test, test.value))
+        dfDirectCountSuite.add(createDataFrameDirectCountTest(table, test.col, test.test, test.value))
+    }
+
+    suites.push(dfCountBySuite, dfFilterCountSuite, dfDirectCountSuite)
+}
+
 console.log('Running apache-arrow performance tests...\n');
 
 run();
@@ -109,3 +128,80 @@ function createGetByIndexTest(vector) {
         }
     };
 }
+
+function createDataFrameDirectCountTest(table, column, test, value) {
+    let colidx = table.columns.findIndex((c)=>c.name === column);
+
+    if (test == 'gteq') {
+        op = function () {
+            sum = 0;
+            for (let batch = -1; ++batch < table.lengths.length;) {
+                const length = table.lengths[batch];
+
+                // load batches
+                const columns = table.batches[batch];
+
+                // yield all indices
+                for (let idx = -1; ++idx < length;) {
+                    sum += (columns[colidx].get(idx) >= value);
+                }
+            }
+        }
+    } else if (test == 'eq') {
+        op = function() {
+            sum = 0;
+            for (let batch = -1; ++batch < table.lengths.length;) {
+                const length = table.lengths[batch];
+
+                // load batches
+                const columns = table.batches[batch]
+
+                // yield all indices
+                for (let idx = -1; ++idx < length;) {
+                    sum += (columns[colidx].get(idx) == value);
+                }
+            }
+        }
+    } else {
+        throw new Error(`Unrecognized test "${test}"`);
+    }
+
+    return {
+        async: true,
+        name: `name: '${column}', length: ${table.length}, type: ${table.columns[colidx].type}, test: ${test}, value: ${value}`,
+        fn: op
+    };
+}
+
+function createDataFrameCountByTest(table, column) {
+    let colidx = table.columns.findIndex((c)=>c.name === column);
+
+    return {
+        async: true,
+        name: `name: '${column}', length: ${table.length}, type: ${table.columns[colidx].type}`,
+        fn() {
+            table.countBy(column);
+        }
+    };
+}
+
+function createDataFrameFilterCountTest(table, column, test, value) {
+    let colidx = table.columns.findIndex((c)=>c.name === column);
+    let df;
+
+    if (test == 'gteq') {
+        df = table.filter(col(column).gteq(value));
+    } else if (test == 'eq') {
+        df = table.filter(col(column).eq(value));
+    } else {
+        throw new Error(`Unrecognized test "${test}"`);
+    }
+
+    return {
+        async: true,
+        name: `name: '${column}', length: ${table.length}, type: ${table.columns[colidx].type}, test: ${test}, value: ${value}`,
+        fn() {
+            df.count();
+        }
+    };
+}

js/perf/table_config.js

@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+const fs = require('fs');
+const path = require('path');
+const glob = require('glob');
+
+const config = [];
+const filenames = glob.sync(path.resolve(__dirname, `../test/data/tables/`, `*.arrow`));
+
+countBys = {
+    "tracks": ['origin', 'destination']
+}
+counts = {
+    "tracks": [
+        {col: 'lat',    test: 'gteq', value: 0        },
+        {col: 'lng',    test: 'gteq', value: 0        },
+        {col: 'origin', test:   'eq', value: 'Seattle'},
+    ]
+}
+
+for (const filename of filenames) {
+    const { name } = path.parse(filename);
+    if (name in counts) {
+        config.push({
+            name,
+            buffers: [fs.readFileSync(filename)],
+            countBys: countBys[name],
+            counts: counts[name],
+        });
+    }
+}
+
+module.exports = config;

js/src/Arrow.externs.ts

@@ -50,6 +50,20 @@ Table.prototype.key;
 Table.prototype.select;
 /** @type {?} */
 Table.prototype.toString;
+/** @type {?} */
+Table.prototype.lengths;
+/** @type {?} */
+Table.prototype.batches;
+/** @type {?} */
+Table.prototype.countBy;
+/** @type {?} */
+Table.prototype.scan;
+/** @type {?} */
+Table.prototype.get;
+
+let CountByResult = function() {};
+/** @type {?} */
+CountByResult.prototype.asJSON;
 
 let Vector = function() {};
 /** @type {?} */
@@ -82,3 +96,11 @@ let DictionaryVector = function() {};
 DictionaryVector.prototype.getKey;
 /** @type {?} */
 DictionaryVector.prototype.getValue;
+
+let Col = function() {};
+/** @type {?} */
+Col.prototype.gteq;
+/** @type {?} */
+Col.prototype.lteq;
+/** @type {?} */
+Col.prototype.eq;

js/src/Arrow.ts

@@ -15,7 +15,8 @@
 // specific language governing permissions and limitations
 // under the License.
 
-import { Table } from './vector/table';
+import { Table, TableRow, CountByResult } from './table';
+import { lit, col, Col, Value } from './predicate';
 import { Vector } from './vector/vector';
 import { Utf8Vector } from './vector/utf8';
 import { DictionaryVector } from './vector/dictionary';
@@ -53,7 +54,9 @@ Table['fromAsync'] = Table.fromAsync;
 BoolVector['pack'] = BoolVector.pack;
 
 export { read, readAsync };
-export { Table, Vector, StructRow };
+export { Table, TableRow, CountByResult };
+export { lit, col, Col, Value };
+export { Vector, StructRow };
 export { Uint64, Int64, Int128 };
 export { NumericVectorConstructor } from './vector/numeric';
 export { List, TypedArray, TypedArrayConstructor } from './vector/types';
@@ -89,9 +92,13 @@ try {
     const Arrow = eval('exports');
     if (typeof Arrow === 'object') {
         // string indexers tell closure compiler not to rename these properties
+        Arrow['lit'] = lit;
+        Arrow['col'] = col;
+        Arrow['Col'] = Col;
         Arrow['read'] = read;
-        Arrow['readAsync'] = readAsync;
+        Arrow['Value'] = Value;
         Arrow['Table'] = Table;
+        Arrow['readAsync'] = readAsync;
         Arrow['Vector'] = Vector;
         Arrow['StructRow'] = StructRow;
         Arrow['BoolVector'] = BoolVector;
@@ -115,6 +122,7 @@ try {
         Arrow['Float32Vector'] = Float32Vector;
         Arrow['Float64Vector'] = Float64Vector;
         Arrow['DecimalVector'] = DecimalVector;
+        Arrow['CountByResult'] = CountByResult;
         Arrow['TimestampVector'] = TimestampVector;
         Arrow['DictionaryVector'] = DictionaryVector;
         Arrow['FixedSizeListVector'] = FixedSizeListVector;

js/src/bin/arrow2csv.ts

@@ -97,8 +97,8 @@ files.forEach((source) => {
     printTable(table);
 });
 
-function printTable(table: Arrow.Table<any>) {
-    let header = [...table.columns.map((_, i) => table.key(i))].map(stringify);
+function printTable(table: Arrow.Table) {
+    let header = [...table.columns.map((c) => c.name)].map(stringify);
     let maxColumnWidths = header.map(x => x.length);
     // Pass one to convert to strings and count max column widths
     for (let i = -1, n = table.length - 1; ++i < n;) {
@@ -132,4 +132,4 @@ function stringify(x: any) {
                                       : `${x}`;
 }
 
-})();
+})();