Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add support for javascript defined custom columns #88

Merged
merged 8 commits into from
Apr 16, 2018
Merged

Add support for javascript defined custom columns #88

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
87b27f8
Initial approach to computed columns
nmichaud Apr 11, 2018
7719088
Handle null values
nmichaud Apr 11, 2018
0bce36e
Move computed columns into helper function
nmichaud Apr 11, 2018
a5a7d26
Add benchmarks
nmichaud Apr 11, 2018
9c4fe32
Support both integer and float computed columns
nmichaud Apr 12, 2018
65e445d
Add web worker api
nmichaud Apr 12, 2018
5502786
Transitivity of adding computed columns
nmichaud Apr 12, 2018
c033ddb
Add tests for constructed columns
nmichaud Apr 12, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
203 changes: 198 additions & 5 deletions packages/perspective/src/cpp/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,10 @@ using namespace perspective;
using namespace emscripten;


typedef std::codecvt_utf8<wchar_t> utf8convert_type;
typedef std::codecvt_utf8_utf16<wchar_t> utf16convert_type;


/******************************************************************************
*
* Data Loading
Expand Down Expand Up @@ -396,7 +400,6 @@ _fill_col<std::string>(val dcol, t_col_sptr col, t_bool is_arrow)
for (auto i = 0; i < nrows; ++i)
{
std::wstring welem = dcol[i].as<std::wstring>();
typedef std::codecvt_utf8_utf16<wchar_t> utf16convert_type;
std::wstring_convert<utf16convert_type, wchar_t> converter;
std::string elem = converter.to_bytes(welem);
col->set_nth(i, elem);
Expand Down Expand Up @@ -618,6 +621,24 @@ make_gnode(t_table_sptr table)
return gnode;
}


/**
* Copies the internal table from a gnode
*
* Params
* ------
*
* Returns
* -------
* A table.
*/
t_table_sptr
clone_gnode_table(t_gnode_sptr gnode)
{
// This creates a copy of the table
return t_table_sptr(gnode->_get_pkeyed_table());
}

/**
*
*
Expand Down Expand Up @@ -740,9 +761,8 @@ sort(t_ctx2_sptr ctx2, val j_sortby)
*
*/
val
scalar_to_val(const t_tscalvec& scalars, t_uint32 idx)
scalar_to_val(const t_tscalar scalar)
{
auto scalar = scalars[idx];
switch (scalar.get_dtype())
{
case DTYPE_BOOL:
Expand Down Expand Up @@ -787,13 +807,178 @@ scalar_to_val(const t_tscalvec& scalars, t_uint32 idx)
case DTYPE_STR:
default:
{
typedef std::codecvt_utf8<wchar_t> utf8convert_type;
std::wstring_convert<utf8convert_type, wchar_t> converter;
return val(converter.from_bytes(scalar.to_string()));
}
}
}

val
scalar_vec_to_val(const t_tscalvec& scalars, t_uint32 idx)
{
return scalar_to_val(scalars[idx]);
}

val
get_column_data(t_table_sptr table,
t_str colname)
{
val arr = val::array();
auto col = table->get_column(colname);
for (auto idx = 0; idx < col->size(); ++idx)
{
arr.set(idx, scalar_to_val(col->get_scalar(idx)));
}
return arr;
}

void set_column_nth(t_column* col, t_uindex idx, val value) {

// Check if the value is a javascript null
if (value.isNull()) {
col->set_valid(idx, false);
return;
}

switch (col->get_dtype())
{
case DTYPE_BOOL:
{
col->set_nth<t_bool>(idx, value.as<t_bool>(), true);
break;
}
case DTYPE_FLOAT64:
{
col->set_nth<t_float64>(idx, value.as<t_float64>(), true);
break;
}
case DTYPE_FLOAT32:
{
col->set_nth<t_float32>(idx, value.as<t_float32>(), true);
break;
}
case DTYPE_UINT32:
{
col->set_nth<t_uint32>(idx, value.as<t_uint32>(), true);
break;
}
case DTYPE_UINT64:
{
col->set_nth<t_uint64>(idx, value.as<t_uint64>(), true);
break;
}
case DTYPE_INT32:
{
col->set_nth<t_int32>(idx, value.as<t_int32>(), true);
break;
}
case DTYPE_INT64:
{
col->set_nth<t_int64>(idx, value.as<t_int64>(), true);
break;
}
case DTYPE_STR:
{
std::wstring welem = value.as<std::wstring>();

std::wstring_convert<utf16convert_type, wchar_t> converter;
std::string elem = converter.to_bytes(welem);
col->set_nth(idx, elem, true);
break;
}
case DTYPE_UINT8:
case DTYPE_UINT16:
case DTYPE_INT8:
case DTYPE_INT16:
case DTYPE_TIME:
default:
{
// Other types not implemented
}
}
}

/**
* Helper function for computed columns
*
* Params
* ------
*
*
* Returns
* -------
*
*/

void
table_add_computed_column(t_table_sptr table, t_str name, t_dtype dtype,
val func, val inputs) {

// Get list of input column names
auto icol_names = vecFromJSArray<std::string>(inputs);

// Get t_column* for all input columns
t_colcptrvec icols;
for (const auto& cc : icol_names) {
icols.push_back(table->_get_column(cc));
}

int arity = icols.size();

// Add new column
t_column* out = table->add_column(name, dtype, true);

val i1 = val::undefined(),
i2 = val::undefined(),
i3 = val::undefined(),
i4 = val::undefined();

t_uindex size = table->size();
for (t_uindex ridx = 0; ridx < size; ++ridx) {
val value = val::null();

switch (arity) {
case 0: {
value = func();
break;
}
case 1: {
i1 = scalar_to_val(icols[0]->get_scalar(ridx));
value = func(i1);
break;
}
case 2: {
i1 = scalar_to_val(icols[0]->get_scalar(ridx));
i2 = scalar_to_val(icols[1]->get_scalar(ridx));
value = func(i1, i2);
break;
}
case 3: {
i1 = scalar_to_val(icols[0]->get_scalar(ridx));
i2 = scalar_to_val(icols[1]->get_scalar(ridx));
i3 = scalar_to_val(icols[2]->get_scalar(ridx));
value = func(i1, i2, i3);
break;
}
case 4: {
i1 = scalar_to_val(icols[0]->get_scalar(ridx));
i2 = scalar_to_val(icols[1]->get_scalar(ridx));
i3 = scalar_to_val(icols[2]->get_scalar(ridx));
i4 = scalar_to_val(icols[3]->get_scalar(ridx));
value = func(i1, i2, i3, i4);
break;
}
default: {
// Don't handle other arity values
break;
}
}

set_column_nth(out, ridx, value);
}

}

/**
*
*
Expand All @@ -813,7 +998,7 @@ get_data(T ctx, t_uint32 start_row, t_uint32 end_row, t_uint32 start_col, t_uint
val arr = val::array();
for (auto idx = 0; idx < slice.size(); ++idx)
{
arr.set(idx, scalar_to_val(slice, idx));
arr.set(idx, scalar_to_val(slice[idx]));
}
return arr;
}
Expand Down Expand Up @@ -846,10 +1031,14 @@ main(int argc, char** argv)

EMSCRIPTEN_BINDINGS(perspective)
{
class_<t_column>("t_column")
.smart_ptr<std::shared_ptr<t_column>>("shared_ptr<t_column>")
.function<void>("set_scalar", &t_column::set_scalar);

class_<t_table>("t_table")
.constructor<t_schema, t_uindex>()
.smart_ptr<std::shared_ptr<t_table>>("shared_ptr<t_table>")
.function<t_column*>("add_column", &t_table::add_column, allow_raw_pointers())
.function<void>("pprint", &t_table::pprint)
.function<unsigned long>(
"size",
Expand Down Expand Up @@ -1086,11 +1275,15 @@ EMSCRIPTEN_BINDINGS(perspective)
function("sort", &sort);
function("make_table", &make_table);
function("make_gnode", &make_gnode);
function("clone_gnode_table", &clone_gnode_table);
function("fill", &fill, allow_raw_pointers());
function("make_context_zero", &make_context_zero);
function("make_context_one", &make_context_one);
function("make_context_two", &make_context_two);
function("scalar_to_val", &scalar_to_val);
function("scalar_vec_to_val", &scalar_vec_to_val);
function("table_add_computed_column", &table_add_computed_column);
function("set_column_nth", &set_column_nth, allow_raw_pointers());
function("get_data_zero", &get_data<t_ctx0_sptr>);
function("get_data_one", &get_data<t_ctx1_sptr>);
function("get_data_two", &get_data<t_ctx2_sptr>);
Expand Down
Loading