From 599c10168e1c8e71a9dc6fff9e0ee0d64a485526 Mon Sep 17 00:00:00 2001 From: "s.lewis.97@cantab.net" Date: Wed, 8 Aug 2018 00:24:11 -0400 Subject: [PATCH] Merge from fork --- .../src/cpp/context_grouped_pkey.cpp | 17 +-- packages/perspective/src/cpp/context_two.cpp | 107 +++++++++++++++++- packages/perspective/src/cpp/gnode.cpp | 6 +- packages/perspective/src/cpp/gnode_state.cpp | 29 ++++- packages/perspective/src/cpp/sparse_tree.cpp | 38 +++++-- packages/perspective/src/cpp/table.cpp | 12 +- packages/perspective/src/cpp/traversal.cpp | 2 +- .../src/cpp/tree_context_common.cpp | 5 +- packages/perspective/src/cpp/vocab.cpp | 24 +++- .../src/include/perspective/base.h | 1 - .../src/include/perspective/column.h | 51 +++++++-- .../src/include/perspective/context_common.h | 4 + .../perspective/context_grouped_pkey.h | 2 +- .../src/include/perspective/context_handle.h | 6 +- .../src/include/perspective/context_two.h | 5 + .../src/include/perspective/gnode.h | 2 +- .../src/include/perspective/gnode_state.h | 2 +- .../src/include/perspective/pool.h | 6 +- .../src/include/perspective/portable.h | 49 ++++---- .../src/include/perspective/sparse_tree.h | 2 +- .../src/include/perspective/table.h | 13 ++- .../src/include/perspective/traversal.h | 2 +- .../src/include/perspective/vocab.h | 2 + 23 files changed, 290 insertions(+), 97 deletions(-) diff --git a/packages/perspective/src/cpp/context_grouped_pkey.cpp b/packages/perspective/src/cpp/context_grouped_pkey.cpp index f6ab402ec6..666e65c1a9 100644 --- a/packages/perspective/src/cpp/context_grouped_pkey.cpp +++ b/packages/perspective/src/cpp/context_grouped_pkey.cpp @@ -947,15 +947,11 @@ t_ctx_grouped_pkey::notify(const t_table& flattened) // aggregates should be presized to be same size // as agg_indices void -t_ctx_grouped_pkey::get_aggregates(t_uindex nidx, +t_ctx_grouped_pkey::get_aggregates_for_sorting(t_uindex nidx, const t_idxvec& agg_indices, t_tscalvec& aggregates, t_ctx2 * ) const { - - const t_str& grouping_label_col = - m_config.get_grouping_label_column(); - for (t_uindex idx = 0, loop_end = agg_indices.size(); idx < loop_end; ++idx) @@ -964,16 +960,7 @@ t_ctx_grouped_pkey::get_aggregates(t_uindex nidx, if (which_agg < 0) { - if (m_has_label) - { - t_tscalvec pkeys; - auto iters = m_tree->get_pkeys_for_leaf(nidx); - aggregates[idx].set(m_state->get_value( - iters.first->m_pkey, grouping_label_col)); - } else - { - aggregates[idx].set(m_tree->get_value(nidx)); - } + aggregates[idx].set(m_tree->get_sortby_value(nidx)); } else { diff --git a/packages/perspective/src/cpp/context_two.cpp b/packages/perspective/src/cpp/context_two.cpp index d1c65f0588..2cc52fe34a 100644 --- a/packages/perspective/src/cpp/context_two.cpp +++ b/packages/perspective/src/cpp/context_two.cpp @@ -243,7 +243,7 @@ t_ctx2::get_ctraversal_indices() const } t_tscalvec -t_ctx2::get_data(t_tvidx start_row, +t_ctx2::get_data_old_path(t_tvidx start_row, t_tvidx end_row, t_tvidx start_col, t_tvidx end_col) const @@ -343,6 +343,111 @@ t_ctx2::get_data(t_tvidx start_row, return retval; } +t_tscalvec +t_ctx2::get_data(t_tvidx start_row, + t_tvidx end_row, + t_tvidx start_col, + t_tvidx end_col) const +{ + static bool const enable_getdata_fix = true; + + if( !enable_getdata_fix ) + return get_data_old_path( start_row, end_row, start_col, end_col ); + + auto ext = sanitize_get_data_extents( + *this, start_row, end_row, start_col, end_col); + + t_uidxpvec cells; + for (t_index ridx = ext.m_srow; ridx < ext.m_erow; ++ridx) + { + for (t_index cidx = ext.m_scol; cidx < ext.m_ecol; ++cidx) + { + cells.push_back(t_idxpair(ridx, cidx)); + } + } + + auto cells_info = resolve_cells(cells); + + t_index nrows = ext.m_erow - ext.m_srow; + t_index stride = ext.m_ecol - ext.m_scol; + t_tscalvec retval(nrows * stride); + + t_tscalar empty = mknone(); + + typedef std::pair t_aggpair; + std::map aggmap; + + for (t_uindex treeidx = 0, tree_loop_end = m_trees.size(); + treeidx < tree_loop_end; + ++treeidx) + { + auto aggtable = m_trees[treeidx]->get_aggtable(); + t_schema aggschema = aggtable->get_schema(); + + for (t_uindex aggidx = 0, + agg_loop_end = m_config.get_num_aggregates(); + aggidx < agg_loop_end; + ++aggidx) + { + const t_str& aggname = aggschema.m_columns[aggidx]; + + aggmap[t_aggpair(treeidx, aggidx)] = + aggtable->get_const_column(aggname).get(); + } + } + + const t_aggspecvec& aggspecs = m_config.get_aggregates(); + + for (t_index ridx = ext.m_srow; ridx < ext.m_erow; ++ridx) + { + if( ext.m_scol == 0 ) + { + retval[(ridx - ext.m_srow) * stride].set( + rtree()->get_value(m_rtraversal->get_tree_index(ridx))); + } + + for (t_index cidx = std::max( ext.m_scol, t_tvidx(1) ); cidx < ext.m_ecol; ++cidx) + { + t_index insert_idx = (ridx - ext.m_srow) * stride + (cidx - ext.m_scol); + const t_cellinfo& cinfo = cells_info[insert_idx]; + + if (cinfo.m_idx < 0) + { + retval[insert_idx].set(empty); + } + else + { + auto aggcol = aggmap[t_aggpair(cinfo.m_treenum, + cinfo.m_agg_index)]; + + t_ptidx p_idx = + m_trees[cinfo.m_treenum]->get_parent_idx( + cinfo.m_idx); + + t_uindex agg_ridx = + m_trees[cinfo.m_treenum]->get_aggidx(cinfo.m_idx); + + t_uindex agg_pridx = + p_idx == INVALID_INDEX + ? INVALID_INDEX + : m_trees[cinfo.m_treenum]->get_aggidx(p_idx); + + auto value = + extract_aggregate(aggspecs[cinfo.m_agg_index], + aggcol, + agg_ridx, + agg_pridx); + + if (!value.is_valid()) + value.set(empty); + + retval[insert_idx].set(value); + } + } + } + + return retval; +} void t_ctx2::sort_by(const t_sortsvec& sortby) { diff --git a/packages/perspective/src/cpp/gnode.cpp b/packages/perspective/src/cpp/gnode.cpp index bc3f4fd33f..f81a44b3d1 100644 --- a/packages/perspective/src/cpp/gnode.cpp +++ b/packages/perspective/src/cpp/gnode.cpp @@ -495,7 +495,7 @@ t_gnode::_process() t_colcptrvec fcolumns(flattened->num_columns()); t_uindex ncols = sschema.get_num_columns(); - + t_colcptrvec scolumns(ncols); t_colptrvec dcolumns(ncols); t_colptrvec pcolumns(ncols); @@ -1261,7 +1261,7 @@ t_gnode::notify_contexts(const t_table& flattened) t_sctxhvec ctxhvec(num_ctx); t_index ctxh_count = 0; - for (t_sctxhmap::iterator iter = m_contexts.begin(); + for (t_sctxhmap::const_iterator iter = m_contexts.begin(); iter != m_contexts.end(); ++iter) { @@ -1272,7 +1272,7 @@ t_gnode::notify_contexts(const t_table& flattened) auto notify_context_helper = [this, &ctxhvec, &flattened]( t_index ctxidx) { const t_ctx_handle& ctxh = ctxhvec[ctxidx]; - switch (ctxh.m_ctx_type) + switch (ctxh.get_type()) { case TWO_SIDED_CONTEXT: { diff --git a/packages/perspective/src/cpp/gnode_state.cpp b/packages/perspective/src/cpp/gnode_state.cpp index f3152b3d04..ade9ba1eca 100644 --- a/packages/perspective/src/cpp/gnode_state.cpp +++ b/packages/perspective/src/cpp/gnode_state.cpp @@ -155,10 +155,10 @@ t_gstate::update_history(const t_table* tbl) ++idx) { const t_str& cname = fschema.m_columns[idx]; - col_translation[count] = idx; - fcolumns[idx] = tbl->get_const_column(cname).get(); - ++count; - } + col_translation[count] = idx; + fcolumns[idx] = tbl->get_const_column(cname).get(); + ++count; + } t_colptrvec scolumns(ncols); @@ -762,10 +762,29 @@ t_gstate::_get_pkeyed_table(const t_schema& schema, if (get_pkey_dtype() == DTYPE_STR) { static const t_tscalar empty = get_interned_tscalar(""); + static bool const enable_pkeyed_table_vocab_reserve = true; t_uindex offset = has_pkey(empty) ? 0 : 1; - pkey_col->set_vocabulary(order); + size_t total_string_size = 0; + + if( enable_pkeyed_table_vocab_reserve ) + { + total_string_size += offset; + for (t_uindex idx = 0, loop_end = order.size(); + idx < loop_end; + ++idx) + { + total_string_size += strlen(order[idx].first.get_char_ptr()) + 1; + } + } + + // if the m_mapping is empty, get_pkey_dtype() may lie about our pkeys being strings + // don't try to reserve in this case + if( !order.size() ) + total_string_size = 0; + + pkey_col->set_vocabulary(order, total_string_size); auto base = pkey_col->get_nth(0); for (t_uindex idx = 0, loop_end = order.size(); diff --git a/packages/perspective/src/cpp/sparse_tree.cpp b/packages/perspective/src/cpp/sparse_tree.cpp index dd2ef8c237..7590b1784c 100644 --- a/packages/perspective/src/cpp/sparse_tree.cpp +++ b/packages/perspective/src/cpp/sparse_tree.cpp @@ -1011,18 +1011,26 @@ t_stree::update_aggs_from_static(const t_dtree_ctx& ctx, cols_topo_sorted.clear(); cols_topo_sorted.reserve(col_cnt); + static bool const enable_aggregate_reordering = true; + static bool const enable_fix_double_calculation = true; + std::unordered_set< t_column* > dst_visited; - auto push_column = [&](size_t idx) + auto push_column = [&]( size_t idx ) { - t_column* dst = agg_update_info.m_dst[idx]; - if (dst_visited.find(dst) != dst_visited.end()) + if ( enable_fix_double_calculation ) { - return; + t_column* dst = agg_update_info.m_dst[ idx ]; + if ( dst_visited.find( dst ) != dst_visited.end() ) + { + return; + } + dst_visited.insert( dst ); } - dst_visited.insert(dst); - cols_topo_sorted.push_back(idx); + cols_topo_sorted.push_back( idx ); }; + if ( enable_aggregate_reordering ) + { // Move scaled agg columns to the end // This does not handle case where scaled aggregate depends on other scaled aggregate // ( not sure if that is possible ) @@ -1040,6 +1048,15 @@ t_stree::update_aggs_from_static(const t_dtree_ctx& ctx, push_column(i); } } + } + else + { + // If backed out, use same column order as before ( not topo sorted ) + for ( size_t i = 0; i < col_cnt; ++i ) + { + push_column( i ); + } + } for (const auto& r : m_tree_unification_records) { @@ -1213,6 +1230,7 @@ t_stree::update_agg_table(t_uindex nidx, t_index nstrands, const t_gstate& gstate) { + static bool const enable_sticky_nan_fix = true; for (t_uindex idx : info.m_dst_topo_sorted) { const t_column* src = info.m_src[idx]; @@ -1231,8 +1249,7 @@ t_stree::update_agg_table(t_uindex nidx, t_tscalar dst_scalar = dst->get_scalar(dst_ridx); old_value.set(dst_scalar); new_value.set(dst_scalar.add(src_scalar)); - - if(old_value.is_nan()) + if( enable_sticky_nan_fix && old_value.is_nan() ) // is_nan returns false for non-float types { // if we previously had a NaN, add can't make it finite again; recalculate entire sum in case it is now finite auto pkeys = get_pkeys(nidx); @@ -1240,7 +1257,6 @@ t_stree::update_agg_table(t_uindex nidx, gstate.read_column(spec.get_dependencies()[0].name(), pkeys, values); new_value.set(std::accumulate(values.begin(), values.end(), t_float64(0))); } - dst->set_scalar(dst_ridx, new_value); } break; @@ -2337,7 +2353,7 @@ t_stree::get_aggcols(const t_idxvec& agg_indices) const // aggregates should be presized to be same size // as agg_indices void -t_stree::get_aggregates(t_uindex nidx, +t_stree::get_aggregates_for_sorting(t_uindex nidx, const t_idxvec& agg_indices, t_tscalvec& aggregates, t_ctx2 * ctx2) const @@ -2350,7 +2366,7 @@ t_stree::get_aggregates(t_uindex nidx, auto which_agg = agg_indices[idx]; if(which_agg < 0) { - aggregates[idx] = get_value(nidx); + aggregates[idx] = get_sortby_value(nidx); } else if( ctx2 || ( size_t(which_agg) >= m_aggcols.size() ) ) { diff --git a/packages/perspective/src/cpp/table.cpp b/packages/perspective/src/cpp/table.cpp index 1e4e68b413..446369749e 100644 --- a/packages/perspective/src/cpp/table.cpp +++ b/packages/perspective/src/cpp/table.cpp @@ -567,6 +567,7 @@ t_mask t_table::filter_cpp(t_filter_op combiner, const t_ftermvec& fterms_) const { + static bool const enable_interned_filtering = true; auto self = const_cast(this); auto fterms = fterms_; @@ -581,10 +582,9 @@ t_table::filter_cpp(t_filter_op combiner, indices[idx] = m_schema.get_colidx(fterms[idx].m_colname); columns[idx] = get_const_column(fterms[idx].m_colname).get(); fterms[idx].coerce_numeric(columns[idx]->get_dtype()); - auto op = fterms[idx].m_op; - t_tscalar& thr = fterms[idx].m_threshold; - if (fterms[idx].m_use_interned) + if (fterms[idx].m_use_interned && enable_interned_filtering) { + t_tscalar& thr = fterms[idx].m_threshold; auto col = self->get_column(fterms[idx].m_colname); auto interned = col->get_interned(thr.get_char_ptr()); thr.set(interned); @@ -611,7 +611,7 @@ t_table::filter_cpp(t_filter_op combiner, const auto& ft = fterms[cidx]; t_bool tval; - if (ft.m_use_interned) + if (ft.m_use_interned && enable_interned_filtering) { cell_val.set( *(columns[cidx]->get_nth(ridx))); @@ -623,7 +623,7 @@ t_table::filter_cpp(t_filter_op combiner, tval = ft(cell_val); } - if (!tval) + if (!cell_val.is_valid() || !tval) { pass = false; break; @@ -875,7 +875,7 @@ t_table::fill_expr_helper(const t_svec& icol_names, struct cmp_str { bool - operator()(const char* a, const char* b) + operator()(const char* a, const char* b) const { return std::strcmp(a, b) < 0; } diff --git a/packages/perspective/src/cpp/traversal.cpp b/packages/perspective/src/cpp/traversal.cpp index 32dddb25eb..6a354b5886 100644 --- a/packages/perspective/src/cpp/traversal.cpp +++ b/packages/perspective/src/cpp/traversal.cpp @@ -164,7 +164,7 @@ t_traversal::expand_node(const t_sortsvec& sortby, t_tvidx exp_idx, t_ctx2 * ctx iter != tchildren.end(); ++iter) { - m_tree->get_aggregates( + m_tree->get_aggregates_for_sorting( iter->m_idx, sortby_agg_indices, aggregates, ctx2); (*sortelems)[count] = t_mselem(aggregates, child_idx); ++count; diff --git a/packages/perspective/src/cpp/tree_context_common.cpp b/packages/perspective/src/cpp/tree_context_common.cpp index 14bd9a3ccb..4aef1b361a 100644 --- a/packages/perspective/src/cpp/tree_context_common.cpp +++ b/packages/perspective/src/cpp/tree_context_common.cpp @@ -137,7 +137,10 @@ notify_sparse_tree_common(t_table_sptr strands, if (!leaf_paths.empty() && traversal.get() && traversal->size() == 1) { - traversal->populate_root_children(tree); + if ( traversal->get_node( 0 ).m_expanded ) + { + traversal->populate_root_children( tree ); + } } else { diff --git a/packages/perspective/src/cpp/vocab.cpp b/packages/perspective/src/cpp/vocab.cpp index f487cfd7af..7197097847 100644 --- a/packages/perspective/src/cpp/vocab.cpp +++ b/packages/perspective/src/cpp/vocab.cpp @@ -46,12 +46,22 @@ void t_vocab::rebuild_map() { m_map.clear(); + m_map.reserve((size_t)m_vlenidx); for (t_uindex idx = 0; idx < m_vlenidx; ++idx) { m_map[unintern_c(idx)] = idx; } } +void +t_vocab::reserve(size_t total_string_size, size_t string_count) +{ + m_vlendata->reserve( total_string_size ); + m_extents->reserve( sizeof(t_uidxpair) * string_count ); + rebuild_map(); +} + + t_bool t_vocab::string_exists(const char* c, t_stridx& interned) const { @@ -83,10 +93,12 @@ t_vocab::get_interned(const char* s) bidx = m_vlendata->size(); eidx = bidx + len; const void* obase = m_vlendata->get_nth(0); + const void* oebase = m_extents->get_nth(0); m_vlendata->push_back(static_cast(s), len); m_extents->push_back(t_uidxpair(bidx, eidx)); const void* nbase = m_vlendata->get_nth(0); - if (obase == nbase) + const void* nebase = m_extents->get_nth(0); + if ((obase == nbase) && (oebase == nebase)) { m_map[unintern_c(idx)] = idx; } @@ -99,12 +111,14 @@ t_vocab::get_interned(const char* s) { idx = iter->second; } +#ifndef PSP_ENABLE_WASM #ifdef PSP_COLUMN_VERIFY if (t_str(s) == "") { PSP_VERBOSE_ASSERT(idx == 0, "Expected empty string to map to 0"); } +#endif #endif return idx; } @@ -135,7 +149,9 @@ t_vocab::init(t_bool from_recipe) { rebuild_map(); } - //get_interned(""); + #ifndef PSP_ENABLE_WASM + get_interned(""); + #endif //PSP_ENABLE_WASM } t_uindex @@ -154,13 +170,17 @@ t_vocab::verify() const rlookup[kv.second] = kv.first; } +#ifndef PSP_ENABLE_WASM auto zero = rlookup.find(t_stridx(0)); PSP_VERBOSE_ASSERT(zero != rlookup.end(), "0 Not found"); PSP_VERBOSE_ASSERT(t_str(zero->second) == "", "0 mapped to unknown"); +#endif std::unordered_set seen; +#ifndef PSP_ENABLE_WASM seen.insert(t_str("")); +#endif for (t_uindex idx = 1; idx < m_vlenidx; ++idx) { diff --git a/packages/perspective/src/include/perspective/base.h b/packages/perspective/src/include/perspective/base.h index 7fa8e54d43..a52dd3e7e1 100644 --- a/packages/perspective/src/include/perspective/base.h +++ b/packages/perspective/src/include/perspective/base.h @@ -50,7 +50,6 @@ const t_float64 PSP_TABLE_GROW_RATIO = 1.3; const t_index INVALID_INDEX = -1; -//#define PSP_PARALLEL_FOR #ifdef PSP_PARALLEL_FOR #define PSP_PSORT tbb::parallel_sort #else diff --git a/packages/perspective/src/include/perspective/column.h b/packages/perspective/src/include/perspective/column.h index 5cbbfc42c5..289ed68129 100644 --- a/packages/perspective/src/include/perspective/column.h +++ b/packages/perspective/src/include/perspective/column.h @@ -218,7 +218,7 @@ class PERSPECTIVE_EXPORT t_column // indices should be > 0 // scalars will be implicitly understood to be of dtype str template - void set_vocabulary(const VOCAB_T& vocab); + void set_vocabulary(const VOCAB_T& vocab, size_t total_size = 0); void copy_vocabulary(const t_column* other); @@ -493,6 +493,8 @@ template t_histogram t_column::build_histogram(t_uindex nbuckets, t_mask* mask) const { + static bool const enable_gethistogram_fix = true; + DATA_T histmin = std::numeric_limits::max(); DATA_T histmax = std::numeric_limits::min(); @@ -515,16 +517,24 @@ t_column::build_histogram(t_uindex nbuckets, t_mask* mask) const } } + t_histogram rval(nbuckets); + t_float64 range = histmax - histmin; t_float64 bucket_size = range / nbuckets; - t_histogram rval(nbuckets); - for (t_uindex idx = 0; idx < nbuckets; ++idx) { - rval.m_buckets[idx].m_begin.set(histmin + idx * bucket_size); - rval.m_buckets[idx].m_end.set(histmin + + if( enable_gethistogram_fix ) + { + rval.m_buckets[idx].m_begin.set(std::numeric_limits::max()); + rval.m_buckets[idx].m_end.set(std::numeric_limits::min()); + } + else + { + rval.m_buckets[idx].m_begin.set(histmin + idx * bucket_size); + rval.m_buckets[idx].m_end.set(histmin + (idx + 1) * bucket_size); + } } for (t_uindex idx = 0; idx < nrows; ++idx) @@ -542,10 +552,26 @@ t_column::build_histogram(t_uindex nbuckets, t_mask* mask) const continue; } - t_uindex buck_num = v == histmax - ? nbuckets - 1 - : static_cast(std::floor( - (v - histmin) / bucket_size)); + t_uindex buck_num; + if( enable_gethistogram_fix ) + { + + buck_num = v >= histmax + ? nbuckets - 1 + : static_cast(std::max( 0.0, std::floor( + (v - histmin) / bucket_size))); + + + rval.m_buckets[idx].m_begin.set( std::min(rval.m_buckets[buck_num].m_begin.get(), v ) ); + rval.m_buckets[idx].m_end.set( std::max(rval.m_buckets[buck_num].m_end.get(), v ) ); + } + else + { + buck_num = v == histmax + ? nbuckets - 1 + : static_cast(std::floor( + (v - histmin) / bucket_size)); + } rval.m_buckets[buck_num].m_count += 1; } @@ -561,12 +587,13 @@ t_column::raw_fill(DATA_T v) template void -t_column::set_vocabulary(const VOCAB_T& vocab) +t_column::set_vocabulary(const VOCAB_T& vocab, size_t total_size) { + if( total_size ) + m_vocab->reserve( total_size, vocab.size() + 1 ); + for (const auto& kv : vocab) - { m_vocab->get_interned(kv.first.get_char_ptr()); - } } template <> diff --git a/packages/perspective/src/include/perspective/context_common.h b/packages/perspective/src/include/perspective/context_common.h index 92b9564e36..6275d089fd 100644 --- a/packages/perspective/src/include/perspective/context_common.h +++ b/packages/perspective/src/include/perspective/context_common.h @@ -42,6 +42,10 @@ sanitize_get_data_extents(const CONTEXT_T& ctx, start_col = std::min(start_col, ncols); end_col = std::min(end_col, ncols); + start_col = std::max(t_tvidx(0), start_col); + end_col = std::max(t_tvidx(0), end_col); + end_col = std::max(start_col, end_col); + t_get_data_extents rval; rval.m_srow = start_row; rval.m_erow = end_row; diff --git a/packages/perspective/src/include/perspective/context_grouped_pkey.h b/packages/perspective/src/include/perspective/context_grouped_pkey.h index 8a93c12459..20cfd5b2ed 100644 --- a/packages/perspective/src/include/perspective/context_grouped_pkey.h +++ b/packages/perspective/src/include/perspective/context_grouped_pkey.h @@ -82,7 +82,7 @@ class PERSPECTIVE_EXPORT t_ctx_grouped_pkey // aggregates should be presized to be same size // as agg_indices - void get_aggregates(t_uindex nidx, + void get_aggregates_for_sorting(t_uindex nidx, const t_idxvec& agg_indices, t_tscalvec& aggregates, t_ctx2 *) const; diff --git a/packages/perspective/src/include/perspective/context_handle.h b/packages/perspective/src/include/perspective/context_handle.h index b085bbebef..d22e385f52 100644 --- a/packages/perspective/src/include/perspective/context_handle.h +++ b/packages/perspective/src/include/perspective/context_handle.h @@ -21,8 +21,12 @@ struct t_ctx_handle t_ctx_handle(); t_ctx_handle(void* ctx, t_ctx_type ctx_type); - t_str get_type_descr() const; + t_str get_type_descr() const; + t_ctx_type get_type() const { return m_ctx_type; } + template< typename _T > + _T* get() const { return static_cast<_T*>( m_ctx ); } + t_ctx_type m_ctx_type; void* m_ctx; }; diff --git a/packages/perspective/src/include/perspective/context_two.h b/packages/perspective/src/include/perspective/context_two.h index 96dda7d260..7559a2d0db 100644 --- a/packages/perspective/src/include/perspective/context_two.h +++ b/packages/perspective/src/include/perspective/context_two.h @@ -97,6 +97,11 @@ class PERSPECTIVE_EXPORT t_ctx2 : public t_ctxbase t_tvidx idx) const; using t_ctxbase::get_data; + t_tscalvec get_data_old_path(t_tvidx start_row, + t_tvidx end_row, + t_tvidx start_col, + t_tvidx end_col) const; + protected: t_cinfovec resolve_cells(const t_uidxpvec& cells) const; diff --git a/packages/perspective/src/include/perspective/gnode.h b/packages/perspective/src/include/perspective/gnode.h index f4c043dc85..20e1c69285 100644 --- a/packages/perspective/src/include/perspective/gnode.h +++ b/packages/perspective/src/include/perspective/gnode.h @@ -215,7 +215,7 @@ void t_gnode::notify_context(const t_table& flattened, const t_ctx_handle& ctxh) { - CTX_T* ctx = static_cast(ctxh.m_ctx); + CTX_T* ctx = ctxh.get(); const t_table& delta = *(m_oports[PSP_PORT_DELTA]->get_table().get()); const t_table& prev = diff --git a/packages/perspective/src/include/perspective/gnode_state.h b/packages/perspective/src/include/perspective/gnode_state.h index 307894e0fe..e504ffab38 100644 --- a/packages/perspective/src/include/perspective/gnode_state.h +++ b/packages/perspective/src/include/perspective/gnode_state.h @@ -112,7 +112,7 @@ class PERSPECTIVE_EXPORT t_gstate void reset(); const t_schema& get_port_schema() const; - t_uidxvec get_pkeys_idx(const t_tscalvec& pkeys) const; + t_uidxvec get_pkeys_idx(const t_tscalvec& pkeys) const; protected: t_dtype get_pkey_dtype() const; diff --git a/packages/perspective/src/include/perspective/pool.h b/packages/perspective/src/include/perspective/pool.h index 30d486b5f4..4628b498d3 100644 --- a/packages/perspective/src/include/perspective/pool.h +++ b/packages/perspective/src/include/perspective/pool.h @@ -119,8 +119,6 @@ class PERSPECTIVE_EXPORT t_pool std::vector m_gnodes; #ifdef PSP_ENABLE_PYTHON std::vector m_pynodes; - PyObject* m_update_delegate; - t_ctx_refmap m_ctx_refmap; #endif #ifdef PSP_ENABLE_WASM @@ -129,6 +127,10 @@ class PERSPECTIVE_EXPORT t_pool std::atomic_flag m_run; std::atomic m_data_remaining; std::atomic m_sleep; +#ifdef PSP_ENABLE_PYTHON + PyObject* m_update_delegate; + t_ctx_refmap m_ctx_refmap; +#endif std::atomic m_epoch; t_bool m_has_python_dep; }; diff --git a/packages/perspective/src/include/perspective/portable.h b/packages/perspective/src/include/perspective/portable.h index 44110aae4b..db06db446e 100644 --- a/packages/perspective/src/include/perspective/portable.h +++ b/packages/perspective/src/include/perspective/portable.h @@ -7,54 +7,45 @@ * */ -/****************************************************************************** - * - * Copyright (c) 2017, the Perspective Authors. - * - * This file is part of the Perspective library, distributed under the terms of - * the Apache License 2.0. The full license can be found in the LICENSE file. - * - */ - #pragma once +#include + +#ifdef PSP_ENABLE_PYTHON -#ifdef __unix -#define OPEN_MODE_BINARY 0 -#define OPEN_PMODE 0644 -#else -#define OPEN_MODE_BINARY O_BINARY -#define OPEN_PMODE ( FILE_SHARE_READ | FILE_SHARE_WRITE ) -#define strcasecmp stricmp -#endif - -#define CXX_FINAL final -#define CXX_OVERRIDE override +#include -#ifdef __GNUC__ -#define CXX_NOEXCEPT noexcept(true) #else -#define CXX_NOEXCEPT -#endif +// standalone build, no ASGUtils #if defined( __GNUC__ ) - #define __ALWAYS_INLINE__ __attribute__(( always_inline )) +#ifndef NOINLINE #define NOINLINE __attribute__(( noinline )) +#endif + +#ifndef NORETURN #define NORETURN __attribute__((__noreturn__)) void +#endif + #define UNUSED __attribute__(( unused )) #define PRAGMA_GCC( X_ ) _Pragma( #X_ ) #define PRAGMA_VC( X_ ) -#define ASG_BREAK() __asm__( "int3" ) - #else #define __ALWAYS_INLINE__ __forceinline + +// TODO: Resolve collision on NOINLINE and NORETURN with fxalib +#ifndef NOINLINE #define NOINLINE __declspec( noinline ) +#endif + +#ifndef NORETURN #define NORETURN __declspec( noreturn ) void +#endif + #define UNUSED #define PRAGMA_GCC( X_ ) #define PRAGMA_VC( X_ ) __pragma( X_ ) -#define ASG_BREAK() DebugBreak() #endif @@ -63,3 +54,5 @@ #define SUPPRESS_WARNINGS_VC( X_ ) PRAGMA_VC( warning( push ) ) PRAGMA_VC( warning( disable: X_ ) ) #define RESTORE_WARNINGS_VC() PRAGMA_VC( warning( pop ) ) + +#endif diff --git a/packages/perspective/src/include/perspective/sparse_tree.h b/packages/perspective/src/include/perspective/sparse_tree.h index 66b028a7de..456ddfb6d2 100644 --- a/packages/perspective/src/include/perspective/sparse_tree.h +++ b/packages/perspective/src/include/perspective/sparse_tree.h @@ -344,7 +344,7 @@ class PERSPECTIVE_EXPORT t_stree // aggregates should be presized to be same size // as agg_indices - void get_aggregates(t_uindex nidx, + void get_aggregates_for_sorting(t_uindex nidx, const t_idxvec& agg_indices, t_tscalvec& aggregates, t_ctx2 *) const; diff --git a/packages/perspective/src/include/perspective/table.h b/packages/perspective/src/include/perspective/table.h index e658ef8ea2..0a24f4a1a3 100644 --- a/packages/perspective/src/include/perspective/table.h +++ b/packages/perspective/src/include/perspective/table.h @@ -17,15 +17,22 @@ #include #include #include -#ifdef PSP_ENABLE_PYTHON -#include -#endif #ifdef PSP_PARALLEL_FOR #include #include #endif #include +#ifdef PSP_ENABLE_PYTHON + +namespace JITCompiler { + +class PspCompiledComputedColumn; + +} // namespace JITCompiler + +#endif + namespace perspective { diff --git a/packages/perspective/src/include/perspective/traversal.h b/packages/perspective/src/include/perspective/traversal.h index 60ebc4e647..496b1c83b3 100644 --- a/packages/perspective/src/include/perspective/traversal.h +++ b/packages/perspective/src/include/perspective/traversal.h @@ -190,7 +190,7 @@ t_traversal::sort_by(const t_config& config, { children_ptidx[i] = h_children[i].second; - src.get_aggregates(children_ptidx[i], + src.get_aggregates_for_sorting(children_ptidx[i], sortby_agg_indices, aggregates, ctx2); diff --git a/packages/perspective/src/include/perspective/vocab.h b/packages/perspective/src/include/perspective/vocab.h index 7f3244f3ea..e85a5e3146 100644 --- a/packages/perspective/src/include/perspective/vocab.h +++ b/packages/perspective/src/include/perspective/vocab.h @@ -65,6 +65,8 @@ class PERSPECTIVE_EXPORT t_vocab t_bool string_exists(const char* c, t_stridx& interned) const; t_uindex get_max_slen() const; + void reserve(size_t total_string_size, size_t string_count); + protected: // vlen interface t_uindex genidx();