Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Null fix #233

Merged
merged 5 commits into from
Sep 12, 2018
Merged

Null fix #233

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
9633bba
WIP: null handling consistency fix
sc1f Sep 2, 2018
4eaa904
wip: add markers for null handling fixes
sc1f Sep 3, 2018
efa7f0c
Add support for clearing cells, removes valid and replaces with statu…
neilslinger Sep 10, 2018
e67b229
Additional tests for appending rows with null/partial values
neilslinger Sep 11, 2018
cdf4b1c
Added two test cases for specific update scenarios
texodus Sep 11, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
235 changes: 128 additions & 107 deletions packages/perspective/src/cpp/column.cpp
View file
Open in desktop

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions packages/perspective/src/cpp/context_one.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,7 +156,7 @@ t_ctx1::get_data(t_tvidx start_row,
agg_ridx,
agg_pridx);
if (!value.is_valid())
value.set(none);
value.set(none); // todo: fix null handling
tmpvalues[(ridx - ext.m_srow) * ncols + 1 + aggidx].set(
value);
}
Expand Down Expand Up @@ -543,7 +543,7 @@ t_ctx1::pprint() const
agg_ridx,
agg_pridx);
if (!value.is_valid())
value.set(none);
value.set(none); // todo: fix null handling

std::cout << value << ", ";
}
Expand Down
1 change: 1 addition & 0 deletions packages/perspective/src/cpp/context_zero.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,7 @@ t_ctx0::get_data(t_tvidx start_row,
{
auto v = out_data[ridx - ext.m_srow];

// todo: fix null handling
if (!v.is_valid())
v.set(none);

Expand Down
17 changes: 10 additions & 7 deletions packages/perspective/src/cpp/gnode_state.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -293,20 +293,23 @@ t_gstate::update_history(const t_table* tbl)
idx < loop_end;
++idx)
{

t_bool is_valid = fcolumn->is_valid(idx);
t_uindex stableidx = stableidx_vec[idx];

if (!is_valid)
continue;
if (!is_valid) {
t_bool is_cleared = fcolumn->is_cleared(idx);
if (is_cleared) {
scolumn->clear(stableidx);
}
continue;
}

const t_uint8* op_ptr = op_col->get_nth<t_uint8>(idx);
t_op op = static_cast<t_op>(*op_ptr);

if (op == OP_DELETE)
continue;

t_uindex stableidx = stableidx_vec[idx];

switch (fcolumn->get_dtype())
{
case DTYPE_NONE:
Expand Down Expand Up @@ -712,8 +715,8 @@ t_gstate::_get_pkeyed_table(const t_schema& schema,
auto op_col = rval->get_column("psp_op").get();

op_col->raw_fill<t_uint8>(OP_INSERT);
op_col->valid_raw_fill(true);
pkey_col->valid_raw_fill(true);
op_col->valid_raw_fill();
pkey_col->valid_raw_fill();

std::vector<std::pair<t_tscalar, t_uindex>> order(enable_pkeyed_table_mask_fix ? sz : m_mapping.size());
if( enable_pkeyed_table_mask_fix )
Expand Down
59 changes: 42 additions & 17 deletions packages/perspective/src/cpp/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,7 @@ template<typename T>
void
_fill_col(val dcol, t_col_sptr col, t_bool is_arrow)
{
// iterates through dcol, sets them on the c++ column
t_uindex nrows = col->size();

if (is_arrow) {
Expand All @@ -256,7 +257,13 @@ _fill_col(val dcol, t_col_sptr col, t_bool is_arrow)
} else {
for (auto i = 0; i < nrows; ++i)
{
if (dcol[i].isUndefined() || dcol[i].isNull()) continue;
if (dcol[i].isUndefined()) continue;

if (dcol[i].isNull()) {
col->unset(i);
continue;
}

auto elem = dcol[i].as<T>();
col->set_nth(i, elem);
}
Expand Down Expand Up @@ -307,6 +314,12 @@ _fill_col<t_time>(val dcol, t_col_sptr col, t_bool is_arrow)
for (auto i = 0; i < nrows; ++i)
{
if (dcol[i].isUndefined()) continue;

if (dcol[i].isNull()) {
col->unset(i);
continue;
}

auto elem = static_cast<t_int64>(dcol[i].as<t_float64>());
col->set_nth(i, elem);
}
Expand All @@ -332,6 +345,12 @@ _fill_col<t_bool>(val dcol, t_col_sptr col, t_bool is_arrow)
for (auto i = 0; i < nrows; ++i)
{
if (dcol[i].isUndefined()) continue;

if (dcol[i].isNull()) {
col->unset(i);
continue;
}

auto elem = dcol[i].as<t_bool>();
col->set_nth(i, elem);
}
Expand Down Expand Up @@ -389,6 +408,12 @@ _fill_col<std::string>(val dcol, t_col_sptr col, t_bool is_arrow)
for (auto i = 0; i < nrows; ++i)
{
if (dcol[i].isUndefined()) continue;

if (dcol[i].isNull()) {
col->unset(i);
continue;
}

std::wstring welem = dcol[i].as<std::wstring>();
std::wstring_convert<utf16convert_type, wchar_t> converter;
std::string elem = converter.to_bytes(welem);
Expand Down Expand Up @@ -461,11 +486,11 @@ _fill_data(t_table_sptr tbl,
_fill_col<t_float64>(dcol, col, is_arrow);
}
break;
case DTYPE_TIME:
{
_fill_col<t_time>(dcol, col, is_arrow);
}
break;
case DTYPE_TIME:
{
_fill_col<t_time>(dcol, col, is_arrow);
}
break;
case DTYPE_STR:
{
_fill_col<std::string>(dcol, col, is_arrow);
Expand All @@ -479,7 +504,7 @@ _fill_data(t_table_sptr tbl,
t_uint32 null_count = dcol["nullCount"].as<t_uint32>();

if (null_count == 0) {
col->valid_raw_fill(true);
col->valid_raw_fill();
} else {
val validity = dcol["nullBitmap"];
arrow::fill_col_valid(validity, col);
Expand Down Expand Up @@ -832,45 +857,45 @@ void set_column_nth(t_column* col, t_uindex idx, val value) {

// Check if the value is a javascript null
if (value.isNull()) {
col->set_valid(idx, false);
col->unset(idx);
return;
}

switch (col->get_dtype())
{
case DTYPE_BOOL:
{
col->set_nth<t_bool>(idx, value.as<t_bool>(), true);
col->set_nth<t_bool>(idx, value.as<t_bool>(), STATUS_VALID);
break;
}
case DTYPE_FLOAT64:
{
col->set_nth<t_float64>(idx, value.as<t_float64>(), true);
col->set_nth<t_float64>(idx, value.as<t_float64>(), STATUS_VALID);
break;
}
case DTYPE_FLOAT32:
{
col->set_nth<t_float32>(idx, value.as<t_float32>(), true);
col->set_nth<t_float32>(idx, value.as<t_float32>(), STATUS_VALID);
break;
}
case DTYPE_UINT32:
{
col->set_nth<t_uint32>(idx, value.as<t_uint32>(), true);
col->set_nth<t_uint32>(idx, value.as<t_uint32>(), STATUS_VALID);
break;
}
case DTYPE_UINT64:
{
col->set_nth<t_uint64>(idx, value.as<t_uint64>(), true);
col->set_nth<t_uint64>(idx, value.as<t_uint64>(), STATUS_VALID);
break;
}
case DTYPE_INT32:
{
col->set_nth<t_int32>(idx, value.as<t_int32>(), true);
col->set_nth<t_int32>(idx, value.as<t_int32>(), STATUS_VALID);
break;
}
case DTYPE_INT64:
{
col->set_nth<t_int64>(idx, value.as<t_int64>(), true);
col->set_nth<t_int64>(idx, value.as<t_int64>(), STATUS_VALID);
break;
}
case DTYPE_STR:
Expand All @@ -879,12 +904,12 @@ void set_column_nth(t_column* col, t_uindex idx, val value) {

std::wstring_convert<utf16convert_type, wchar_t> converter;
std::string elem = converter.to_bytes(welem);
col->set_nth(idx, elem, true);
col->set_nth(idx, elem, STATUS_VALID);
break;
}
case DTYPE_TIME:
{
col->set_nth<t_int64>(idx, static_cast<t_int64>(value.as<t_float64>()), true);
col->set_nth<t_int64>(idx, static_cast<t_int64>(value.as<t_float64>()), STATUS_VALID);
break;
}
case DTYPE_UINT8:
Expand Down
Loading