diff --git a/cmake/ordered-map.txt.in b/cmake/ordered-map.txt.in new file mode 100644 index 0000000000..70518f03fe --- /dev/null +++ b/cmake/ordered-map.txt.in @@ -0,0 +1,16 @@ +cmake_minimum_required(VERSION 2.8.2) + +project(ordered-map-download NONE) + +include(ExternalProject) +ExternalProject_Add(ordered-map + GIT_REPOSITORY https://github.com/Tessil/ordered-map.git + GIT_TAG master + SOURCE_DIR "${CMAKE_BINARY_DIR}/ordered-map-src" + BINARY_DIR "${CMAKE_BINARY_DIR}/ordered-map-build" + CONFIGURE_COMMAND "" + BUILD_COMMAND "" + INSTALL_COMMAND "" + TEST_COMMAND "" + CMAKE_ARGS "-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE}" +) diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt index d6055a7769..e097ab0891 100644 --- a/cpp/perspective/CMakeLists.txt +++ b/cpp/perspective/CMakeLists.txt @@ -259,6 +259,7 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD) endif() psp_build_dep("hopscotch" "../../cmake/hopscotch.txt.in") +psp_build_dep("ordered-map" "../../cmake/ordered-map.txt.in") ##################### @@ -346,6 +347,7 @@ set (SOURCE_FILES src/cpp/utils.cpp src/cpp/update_task.cpp src/cpp/view.cpp + src/cpp/view_config.cpp src/cpp/vocab.cpp ) diff --git a/cpp/perspective/src/cpp/aggspec.cpp b/cpp/perspective/src/cpp/aggspec.cpp index 80b77e33f3..1174979fa1 100644 --- a/cpp/perspective/src/cpp/aggspec.cpp +++ b/cpp/perspective/src/cpp/aggspec.cpp @@ -23,27 +23,6 @@ t_col_name_type::t_col_name_type(const std::string& name, t_dtype type) t_aggspec::t_aggspec() {} -t_aggspec::t_aggspec(const t_aggspec_recipe& v) { - m_name = v.m_name; - m_disp_name = v.m_name; - m_agg = v.m_agg; - - for (const auto& d : v.m_dependencies) { - m_dependencies.push_back(d); - } - - for (const auto& d : v.m_odependencies) { - m_odependencies.push_back(d); - } - - m_sort_type = v.m_sort_type; - m_agg_one_idx = v.m_agg_one_idx; - m_agg_two_idx = v.m_agg_two_idx; - m_agg_one_weight = v.m_agg_one_weight; - m_agg_two_weight = v.m_agg_two_weight; - m_invmode = v.m_invmode; -} - t_aggspec::t_aggspec( const std::string& name, t_aggtype agg, const std::vector& dependencies) : m_name(name) @@ -407,29 +386,4 @@ t_aggspec::get_first_depname() const { return m_dependencies[0].name(); } -t_aggspec_recipe -t_aggspec::get_recipe() const { - t_aggspec_recipe rv; - rv.m_name = m_name; - rv.m_disp_name = m_name; - rv.m_agg = m_agg; - - for (const auto& d : m_dependencies) { - rv.m_dependencies.push_back(d.get_recipe()); - } - - for (const auto& d : m_odependencies) { - rv.m_odependencies.push_back(d.get_recipe()); - } - - rv.m_sort_type = m_sort_type; - rv.m_agg_one_idx = m_agg_one_idx; - rv.m_agg_two_idx = m_agg_two_idx; - rv.m_agg_one_weight = m_agg_one_weight; - rv.m_agg_two_weight = m_agg_two_weight; - rv.m_invmode = m_invmode; - - return rv; -} - } // end namespace perspective diff --git a/cpp/perspective/src/cpp/config.cpp b/cpp/perspective/src/cpp/config.cpp index 86ebf3df08..fc6c0eab04 100644 --- a/cpp/perspective/src/cpp/config.cpp +++ b/cpp/perspective/src/cpp/config.cpp @@ -12,78 +12,7 @@ namespace perspective { -t_config_recipe::t_config_recipe() - : m_child_pkey_column("psp_pkey") {} - -t_config::t_config() {} - -t_config::t_config(const t_config_recipe& r) - : m_detail_columns(r.m_detail_columns) - , m_totals(r.m_totals) - , m_combiner(r.m_combiner) - , m_handle_nan_sort(r.m_handle_nan_sort) - , m_parent_pkey_column(r.m_parent_pkey_column) - , m_child_pkey_column(r.m_child_pkey_column) - , m_grouping_label_column(r.m_grouping_label_column) - , m_fmode(r.m_fmode) - , m_filter_exprs(r.m_filter_exprs) - -{ - for (const auto& v : r.m_row_pivots) { - m_row_pivots.push_back(t_pivot(v)); - } - - for (const auto& v : r.m_col_pivots) { - m_col_pivots.push_back(t_pivot(v)); - } - - for (const auto& v : r.m_aggregates) { - m_aggregates.push_back(t_aggspec(v)); - } - - if (m_fmode == FMODE_SIMPLE_CLAUSES) { - for (const auto& v : r.m_fterms) { - m_fterms.push_back(t_fterm(v)); - } - } - - std::vector sort_pivot; - std::vector sort_pivot_by; - - for (const auto& v : r.m_sortby) { - sort_pivot.push_back(v.first); - sort_pivot_by.push_back(v.second); - } - - setup(m_detail_columns, sort_pivot, sort_pivot_by); -} - -t_config::t_config(const std::vector& row_pivots, - const std::vector& col_pivots, const std::vector& aggregates, - const std::vector& detail_columns, const t_totals totals, - const std::vector& sort_pivot, const std::vector& sort_pivot_by, - t_filter_op combiner, const std::vector& fterms, bool handle_nan_sort, - const std::string& parent_pkey_column, const std::string& child_pkey_column, - const std::string& grouping_label_column, t_fmode fmode, - const std::vector& filter_exprs, const std::string& grand_agg_str) - : m_row_pivots(row_pivots) - , m_col_pivots(col_pivots) - , m_aggregates(aggregates) - , m_detail_columns(detail_columns) - , m_totals(totals) - , m_combiner(combiner) - , m_fterms(fterms) - , m_handle_nan_sort(handle_nan_sort) - , m_parent_pkey_column(parent_pkey_column) - , m_child_pkey_column(child_pkey_column) - , m_grouping_label_column(grouping_label_column) - , m_fmode(fmode) - , m_filter_exprs(filter_exprs) - , m_grand_agg_str(grand_agg_str) { - setup(detail_columns, sort_pivot, sort_pivot_by); -} - -// view config +// Construct view config t_config::t_config(const std::vector& row_pivots, const std::vector& col_pivots, const std::vector& aggregates, const std::vector& sortspecs, const std::vector& col_sortspecs, @@ -104,59 +33,56 @@ t_config::t_config(const std::vector& row_pivots, } }; -t_config::t_config( - const std::vector& row_pivots, const std::vector& aggregates) - : m_row_pivots(row_pivots) - , m_aggregates(aggregates) - , m_fmode(FMODE_SIMPLE_CLAUSES) { - setup(m_detail_columns, std::vector{}, std::vector{}); -} +// t_ctx0 +t_config::t_config(const std::vector& detail_columns, t_filter_op combiner, + const std::vector& fterms) + : m_detail_columns(detail_columns) + , m_combiner(combiner) + , m_fterms(fterms) + , m_fmode(FMODE_SIMPLE_CLAUSES) {} -// grouped_pkeys +// t_ctx1 t_config::t_config(const std::vector& row_pivots, - const std::vector& detail_columns, t_filter_op combiner, - const std::vector& fterms, const std::string& parent_pkey_column, - const std::string& child_pkey_column, const std::string& grouping_label_column) - : m_detail_columns(detail_columns) + const std::vector& aggregates, t_filter_op combiner, + const std::vector& fterms) + : m_aggregates(aggregates) + , m_totals(TOTALS_BEFORE) , m_combiner(combiner) , m_fterms(fterms) , m_handle_nan_sort(true) - , m_parent_pkey_column(parent_pkey_column) - , m_child_pkey_column(child_pkey_column) - , m_grouping_label_column(grouping_label_column) - , m_fmode(FMODE_SIMPLE_CLAUSES) - -{ + , m_fmode(FMODE_SIMPLE_CLAUSES) { for (const auto& p : row_pivots) { m_row_pivots.push_back(t_pivot(p)); } - setup(m_detail_columns, std::vector{}, std::vector{}); } -// ctx2 +// t_ctx2 t_config::t_config(const std::vector& row_pivots, - const std::vector& col_pivots, const std::vector& aggregates) - : t_config(row_pivots, col_pivots, aggregates, TOTALS_HIDDEN, FILTER_OP_AND, {}) - -{} - -t_config::t_config(const std::vector& row_pivots, - const std::vector& col_pivots, const std::vector& aggregates, + const std::vector& col_pivots, const std::vector& aggregates, const t_totals totals, t_filter_op combiner, const std::vector& fterms, bool column_only) - : m_row_pivots(row_pivots) - , m_col_pivots(col_pivots) - , m_column_only(column_only) + : m_column_only(column_only) , m_aggregates(aggregates) , m_totals(totals) , m_combiner(combiner) , m_fterms(fterms) , m_handle_nan_sort(true) , m_fmode(FMODE_SIMPLE_CLAUSES) { + for (const auto& p : row_pivots) { + m_row_pivots.push_back(t_pivot(p)); + } + for (const auto& p : col_pivots) { + m_col_pivots.push_back(t_pivot(p)); + } setup(m_detail_columns, std::vector{}, std::vector{}); } +// Constructors used for C++ tests +t_config::t_config(const std::vector& row_pivots, + const std::vector& col_pivots, const std::vector& aggregates) + : t_config(row_pivots, col_pivots, aggregates, TOTALS_HIDDEN, FILTER_OP_AND, {}) {} + t_config::t_config(const std::vector& row_pivots, const std::vector& col_pivots, const std::vector& aggregates, const t_totals totals, t_filter_op combiner, const std::vector& fterms) @@ -177,7 +103,14 @@ t_config::t_config(const std::vector& row_pivots, setup(m_detail_columns, std::vector{}, std::vector{}); } -// t_ctx1 +t_config::t_config( + const std::vector& row_pivots, const std::vector& aggregates) + : m_row_pivots(row_pivots) + , m_aggregates(aggregates) + , m_fmode(FMODE_SIMPLE_CLAUSES) { + setup(m_detail_columns, std::vector{}, std::vector{}); +} + t_config::t_config( const std::vector& row_pivots, const std::vector& aggregates) : m_aggregates(aggregates) @@ -205,45 +138,10 @@ t_config::t_config(const std::vector& row_pivots, const t_aggspec& setup(m_detail_columns, std::vector{}, std::vector{}); } -t_config::t_config(const std::vector& row_pivots, - const std::vector& aggregates, t_filter_op combiner, - const std::vector& fterms) - : m_row_pivots(row_pivots) - , m_aggregates(aggregates) - , m_totals(TOTALS_BEFORE) - , m_combiner(combiner) - , m_fterms(fterms) - , m_handle_nan_sort(true) - , m_fmode(FMODE_SIMPLE_CLAUSES) { - setup(m_detail_columns, std::vector{}, std::vector{}); -} - -t_config::t_config(const std::vector& row_pivots, - const std::vector& aggregates, t_filter_op combiner, - const std::vector& fterms) - : m_aggregates(aggregates) - , m_totals(TOTALS_BEFORE) - , m_combiner(combiner) - , m_fterms(fterms) - , m_handle_nan_sort(true) - , m_fmode(FMODE_SIMPLE_CLAUSES) { - for (const auto& p : row_pivots) { - m_row_pivots.push_back(t_pivot(p)); - } - - setup(m_detail_columns, std::vector{}, std::vector{}); -} - -// t_ctx0 t_config::t_config(const std::vector& detail_columns) : t_config(detail_columns, FILTER_OP_AND, {}) {} -t_config::t_config(const std::vector& detail_columns, t_filter_op combiner, - const std::vector& fterms) - : m_detail_columns(detail_columns) - , m_combiner(combiner) - , m_fterms(fterms) - , m_fmode(FMODE_SIMPLE_CLAUSES) {} +t_config::t_config() {} void t_config::setup(const std::vector& detail_columns, @@ -496,38 +394,6 @@ t_config::get_grouping_label_column() const { return m_grouping_label_column; } -t_config_recipe -t_config::get_recipe() const { - t_config_recipe rv; - - for (const auto& p : m_row_pivots) { - rv.m_row_pivots.push_back(p.get_recipe()); - } - - for (const auto& p : m_col_pivots) { - rv.m_col_pivots.push_back(p.get_recipe()); - } - - rv.m_sortby = get_sortby_pairs(); - - for (const auto& a : m_aggregates) { - rv.m_aggregates.push_back(a.get_recipe()); - } - - rv.m_totals = m_totals; - rv.m_combiner = m_combiner; - - for (const auto& ft : m_fterms) { - rv.m_fterms.push_back(ft.get_recipe()); - } - - rv.m_handle_nan_sort = m_handle_nan_sort; - rv.m_parent_pkey_column = m_parent_pkey_column; - rv.m_child_pkey_column = m_child_pkey_column; - rv.m_grouping_label_column = m_grouping_label_column; - return rv; -} - std::string t_config::unity_get_column_name(t_uindex idx) const { if (m_aggregates.empty()) { diff --git a/cpp/perspective/src/cpp/emscripten.cpp b/cpp/perspective/src/cpp/emscripten.cpp index 77ab97f173..38854569f0 100644 --- a/cpp/perspective/src/cpp/emscripten.cpp +++ b/cpp/perspective/src/cpp/emscripten.cpp @@ -18,235 +18,17 @@ namespace binding { * * Utility */ - template <> - bool - has_value(t_val item) { - return (!item.isUndefined() && !item.isNull()); - } - - /****************************************************************************** - * - * Data Loading - */ - - t_index - _get_aggregate_index(const std::vector& agg_names, std::string name) { - auto it = std::find(agg_names.begin(), agg_names.end(), name); - if (it != agg_names.end()) { - return t_index(std::distance(agg_names.begin(), it)); - } - return t_index(); - } - - std::vector - _get_aggregate_names(const std::vector& aggs) { - std::vector names; - for (const t_aggspec& agg : aggs) { - names.push_back(agg.name()); - } - return names; - } - - template <> - std::vector - _get_aggspecs(const t_schema& schema, const std::vector& row_pivots, - const std::vector& column_pivots, bool column_only, - const std::vector& columns, const std::vector& sortbys, - t_val j_aggs) { - std::vector aggspecs; - t_val agg_columns = t_val::global("Object").call("keys", j_aggs); - std::vector aggs = vecFromArray(agg_columns); - - /** - * Provide aggregates for columns that are shown but NOT specified in - * the `j_aggs` object. - */ - for (const std::string& column : columns) { - if (std::find(aggs.begin(), aggs.end(), column) != aggs.end()) { - continue; - } - - t_dtype dtype = schema.get_dtype(column); - std::vector dependencies{t_dep(column, DEPTYPE_COLUMN)}; - t_aggtype agg_op - = t_aggtype::AGGTYPE_ANY; // use aggtype here since we are not parsing aggs - - if (!column_only) { - agg_op = _get_default_aggregate(dtype); - } - - aggspecs.push_back(t_aggspec(column, agg_op, dependencies)); - } - - // Construct aggregates from config object - for (const std::string& agg_column : aggs) { - if (std::find(columns.begin(), columns.end(), agg_column) == columns.end()) { - continue; - } - - std::string agg_op = j_aggs[agg_column].as(); - std::vector dependencies; - - if (column_only) { - agg_op = "any"; - } - - dependencies.push_back(t_dep(agg_column, DEPTYPE_COLUMN)); - - t_aggtype aggtype = str_to_aggtype(agg_op); - - if (aggtype == AGGTYPE_FIRST || aggtype == AGGTYPE_LAST) { - if (dependencies.size() == 1) { - dependencies.push_back(t_dep("psp_pkey", DEPTYPE_COLUMN)); - } - aggspecs.push_back(t_aggspec( - agg_column, agg_column, aggtype, dependencies, SORTTYPE_ASCENDING)); - } else { - aggspecs.push_back(t_aggspec(agg_column, aggtype, dependencies)); - } - } - - // construct aggspecs for hidden sorts - for (auto sortby : sortbys) { - std::string column = sortby[0].as(); - - bool is_hidden_column - = std::find(columns.begin(), columns.end(), column) == columns.end(); - bool not_aggregated = std::find(aggs.begin(), aggs.end(), column) == aggs.end(); - if (is_hidden_column) { - bool is_pivot = (std::find(row_pivots.begin(), row_pivots.end(), column) - != row_pivots.end()) - || (std::find(column_pivots.begin(), column_pivots.end(), column) - != column_pivots.end()); - - std::vector dependencies{t_dep(column, DEPTYPE_COLUMN)}; - t_aggtype agg_op; - - if (is_pivot || row_pivots.size() == 0 || column_only) { - agg_op = t_aggtype::AGGTYPE_ANY; - } else { - t_dtype dtype = schema.get_dtype(column); - agg_op = _get_default_aggregate(dtype); - } - - aggspecs.push_back(t_aggspec(column, agg_op, dependencies)); - } - } - - return aggspecs; - } - - template <> - std::vector - _get_sort(const std::vector& columns, bool is_column_sort, - const std::vector& sortbys) { - std::vector svec{}; - - auto _is_valid_sort = [is_column_sort](t_val sort_item) { - /** - * If column sort, make sure string matches. Otherwise make - * sure string is *not* a column sort. - */ - std::string op = sort_item[1].as(); - bool is_col_sortop = op.find("col") != std::string::npos; - return (is_column_sort && is_col_sortop) || (!is_col_sortop && !is_column_sort); - }; - - for (auto idx = 0; idx < sortbys.size(); ++idx) { - t_val sort_item = sortbys[idx]; - t_index agg_index; - std::string column; - t_sorttype sorttype; - - std::string sort_op_str; - if (!_is_valid_sort(sort_item)) { - continue; - } - - column = sort_item[0].as(); - sort_op_str = sort_item[1].as(); - sorttype = str_to_sorttype(sort_op_str); - - agg_index = _get_aggregate_index(columns, column); - - svec.push_back(t_sortspec(agg_index, sorttype)); - } - return svec; + template + std::vector + make_vector() { + return std::vector{}; } template <> - std::vector - _get_fterms(const t_schema& schema, t_val j_date_parser, t_val j_filters) { - std::vector fvec{}; - std::vector filters = vecFromArray(j_filters); - - auto _is_valid_filter = [j_date_parser](t_dtype type, std::vector filter) { - if (type == DTYPE_DATE || type == DTYPE_TIME) { - t_val parsed_date = j_date_parser.call("parse", filter[2]); - return has_value(parsed_date); - } else { - return has_value(filter[2]); - } - }; - - for (auto fidx = 0; fidx < filters.size(); ++fidx) { - std::vector filter = vecFromArray(filters[fidx]); - std::string col = filter[0].as(); - t_filter_op comp = str_to_filter_op(filter[1].as()); - - // check validity and if_date - t_dtype col_type = schema.get_dtype(col); - bool is_valid = _is_valid_filter(col_type, filter); - - if (!is_valid) { - continue; - } - - switch (comp) { - case FILTER_OP_NOT_IN: - case FILTER_OP_IN: { - std::vector terms{}; - std::vector j_terms - = vecFromArray(filter[2]); - for (auto jidx = 0; jidx < j_terms.size(); ++jidx) { - terms.push_back(mktscalar(get_interned_cstr(j_terms[jidx].c_str()))); - } - fvec.push_back(t_fterm(col, comp, mktscalar(0), terms)); - } break; - default: { - t_tscalar term; - switch (col_type) { - case DTYPE_INT32: { - term = mktscalar(filter[2].as()); - } break; - case DTYPE_INT64: - case DTYPE_FLOAT64: { - term = mktscalar(filter[2].as()); - } break; - case DTYPE_BOOL: { - term = mktscalar(filter[2].as()); - } break; - case DTYPE_DATE: { - t_val parsed_date = j_date_parser.call("parse", filter[2]); - term = mktscalar(jsdate_to_t_date(parsed_date)); - } break; - case DTYPE_TIME: { - t_val parsed_date = j_date_parser.call("parse", filter[2]); - term = mktscalar(t_time(static_cast( - parsed_date.call("getTime").as()))); - } break; - default: { - term = mktscalar( - get_interned_cstr(filter[2].as().c_str())); - } - } - - fvec.push_back(t_fterm(col, comp, term, std::vector())); - } - } - } - return fvec; + bool + has_value(t_val item) { + return (!item.isUndefined() && !item.isNull()); } /****************************************************************************** @@ -1515,151 +1297,137 @@ namespace binding { * * View API */ + template <> - t_config - make_view_config( - const t_schema& schema, std::string separator, t_val date_parser, t_val config) { - t_val j_row_pivots = config["row_pivots"]; - t_val j_column_pivots = config["column_pivots"]; - t_val j_aggregates = config["aggregates"]; - t_val j_columns = config["columns"]; - t_val j_filter = config["filter"]; - t_val j_sort = config["sort"]; - - std::vector row_pivots; - std::vector column_pivots; - std::vector aggregates; - std::vector aggregate_names; - std::vector columns; - std::vector filters; - std::vector sortbys; - std::vector sorts; - std::vector col_sorts; - - t_filter_op filter_op = t_filter_op::FILTER_OP_AND; - - if (has_value(j_row_pivots)) { - row_pivots = vecFromArray(j_row_pivots); + bool + is_valid_filter(t_dtype type, t_val date_parser, t_val filter_term) { + if (type == DTYPE_DATE || type == DTYPE_TIME) { + t_val parsed_date = date_parser.call("parse", filter_term); + return has_value(parsed_date); + } else { + return has_value(filter_term); } + }; - if (has_value(j_column_pivots)) { - column_pivots = vecFromArray(j_column_pivots); + template <> + std::tuple> + make_filter_term(t_dtype type, t_val date_parser, std::vector filter) { + std::string col = filter[0].as(); + std::string comp_str = filter[1].as(); + t_filter_op comp = str_to_filter_op(comp_str); + std::vector terms; + + switch (comp) { + case FILTER_OP_NOT_IN: + case FILTER_OP_IN: { + std::vector filter_terms + = vecFromArray(filter[2]); + for (auto term : filter_terms) { + terms.push_back(mktscalar(get_interned_cstr(term.c_str()))); + } + } break; + default: { + switch (type) { + case DTYPE_INT32: { + terms.push_back(mktscalar(filter[2].as())); + } break; + case DTYPE_INT64: + case DTYPE_FLOAT64: { + terms.push_back(mktscalar(filter[2].as())); + } break; + case DTYPE_BOOL: { + terms.push_back(mktscalar(filter[2].as())); + } break; + case DTYPE_DATE: { + t_val parsed_date = date_parser.call("parse", filter[2]); + terms.push_back(mktscalar(jsdate_to_t_date(parsed_date))); + } break; + case DTYPE_TIME: { + t_val parsed_date = date_parser.call("parse", filter[2]); + terms.push_back(mktscalar(t_time(static_cast( + parsed_date.call("getTime").as())))); + } break; + default: { + terms.push_back( + mktscalar(get_interned_cstr(filter[2].as().c_str()))); + } + } + } } + return std::make_tuple(col, comp_str, terms); + } + + template <> + t_view_config + make_view_config(const t_schema& schema, t_val date_parser, t_val config) { + // extract vectors from JS, where they were created + auto row_pivots = config.call>("get_row_pivots"); + auto column_pivots = config.call>("get_column_pivots"); + auto columns = config.call>("get_columns"); + auto sort = config.call>>("get_sort"); + auto filter_op = config["filter_op"].as(); + + // aggregates require manual parsing - std::maps read from JS are empty + t_val j_aggregate_keys + = t_val::global("Object").call("keys", config["aggregates"]); + auto aggregate_names = vecFromArray(j_aggregate_keys); + + tsl::ordered_map aggregates; + for (const auto& name : aggregate_names) { + aggregates[name] = config["aggregates"][name].as(); + }; + bool column_only = false; + // make sure that primary keys are created for column-only views if (row_pivots.size() == 0 && column_pivots.size() > 0) { row_pivots.push_back("psp_okey"); column_only = true; } - if (has_value(j_sort)) { - sortbys = vecFromArray(j_sort); - } + // construct filters with filter terms, and fill the vector of tuples + auto js_filter = config.call>>("get_filter"); + std::vector>> filter; - columns = vecFromArray(j_columns); - aggregates = _get_aggspecs( - schema, row_pivots, column_pivots, column_only, columns, sortbys, j_aggregates); - aggregate_names = _get_aggregate_names(aggregates); + for (auto f : js_filter) { + t_dtype type = schema.get_dtype(f[0].as()); - if (has_value(j_filter)) { - filters = _get_fterms(schema, date_parser, j_filter); - if (has_value(config["filter_op"])) { - filter_op = str_to_filter_op(config["filter_op"].as()); + // validate the filter before it goes into the core engine + if (is_valid_filter(type, date_parser, f[2])) { + filter.push_back(make_filter_term(type, date_parser, f)); } } - if (sortbys.size() > 0) { - sorts = _get_sort(aggregate_names, false, sortbys); - col_sorts = _get_sort(aggregate_names, true, sortbys); - } - - auto view_config = t_config(row_pivots, column_pivots, aggregates, sorts, col_sorts, - filter_op, filters, aggregate_names, column_only); - - return view_config; - } - - template <> - std::shared_ptr> - make_view_zero(std::shared_ptr table, std::string name, std::string separator, - t_val config, t_val date_parser) { - auto schema = table->get_schema(); - t_config view_config = make_view_config(schema, separator, date_parser, config); - - auto col_names = view_config.get_column_names(); - auto filter_op = view_config.get_combiner(); - auto filters = view_config.get_fterms(); - auto sorts = view_config.get_sortspecs(); - auto ctx = make_context_zero(table, schema, filter_op, col_names, filters, sorts, name); - - auto view_ptr - = std::make_shared>(table, ctx, name, separator, view_config); - - return view_ptr; - } - - template <> - std::shared_ptr> - make_view_one(std::shared_ptr
table, std::string name, std::string separator, - t_val config, t_val date_parser) { - auto schema = table->get_schema(); - t_config view_config = make_view_config(schema, separator, date_parser, config); + // create the `t_view_config` + t_view_config view_config(row_pivots, column_pivots, aggregates, columns, filter, sort, + filter_op, column_only); - auto aggregates = view_config.get_aggregates(); - auto row_pivots = view_config.get_row_pivots(); - auto filter_op = view_config.get_combiner(); - auto filters = view_config.get_fterms(); - auto sorts = view_config.get_sortspecs(); - - std::int32_t pivot_depth = -1; + // set pivot depths if provided if (has_value(config["row_pivot_depth"])) { - pivot_depth = config["row_pivot_depth"].as(); + view_config.set_row_pivot_depth(config["row_pivot_depth"].as()); } - auto ctx = make_context_one(table, schema, row_pivots, filter_op, filters, aggregates, - sorts, pivot_depth, name); + if (has_value(config["column_pivot_depth"])) { + view_config.set_column_pivot_depth(config["column_pivot_depth"].as()); + } - auto view_ptr - = std::make_shared>(table, ctx, name, separator, view_config); + // transform primitive values into abstractions that the engine can use + view_config.init(schema); - return view_ptr; + return view_config; } - template <> - std::shared_ptr> - make_view_two(std::shared_ptr
table, std::string name, std::string separator, - t_val config, t_val date_parser) { + template + std::shared_ptr> + make_view(std::shared_ptr
table, std::string name, std::string separator, + t_val view_config, t_val date_parser) { auto schema = table->get_schema(); - t_config view_config = make_view_config(schema, separator, date_parser, config); - - bool column_only = view_config.is_column_only(); - auto column_names = view_config.get_column_names(); - auto row_pivots = view_config.get_row_pivots(); - auto column_pivots = view_config.get_column_pivots(); - auto aggregates = view_config.get_aggregates(); - auto filter_op = view_config.get_combiner(); - auto filters = view_config.get_fterms(); - auto sorts = view_config.get_sortspecs(); - auto col_sorts = view_config.get_col_sortspecs(); - - std::int32_t rpivot_depth = -1; - std::int32_t cpivot_depth = -1; - - if (has_value(config["row_pivot_depth"])) { - rpivot_depth = config["row_pivot_depth"].as(); - } + t_view_config config = make_view_config(schema, date_parser, view_config); - if (has_value(config["column_pivot_depth"])) { - cpivot_depth = config["column_pivot_depth"].as(); - } + auto ctx = make_context(table, schema, config, name); - auto ctx - = make_context_two(table, schema, row_pivots, column_pivots, filter_op, filters, - aggregates, sorts, col_sorts, rpivot_depth, cpivot_depth, column_only, name); - - auto view_ptr - = std::make_shared>(table, ctx, name, separator, view_config); + auto view_ptr = std::make_shared>(table, ctx, name, separator, config); return view_ptr; } @@ -1669,14 +1437,19 @@ namespace binding { * Context API */ + template <> std::shared_ptr - make_context_zero(std::shared_ptr
table, t_schema schema, t_filter_op combiner, - std::vector columns, std::vector filters, - std::vector sorts, std::string name) { - auto cfg = t_config(columns, combiner, filters); + make_context(std::shared_ptr
table, const t_schema& schema, + const t_view_config& view_config, std::string name) { + auto columns = view_config.get_columns(); + auto filter_op = view_config.get_filter_op(); + auto fterm = view_config.get_fterm(); + auto sortspec = view_config.get_sortspec(); + + auto cfg = t_config(columns, filter_op, fterm); auto ctx0 = std::make_shared(schema, cfg); ctx0->init(); - ctx0->sort_by(sorts); + ctx0->sort_by(sortspec); auto pool = table->get_pool(); auto gnode = table->get_gnode(); @@ -1686,41 +1459,56 @@ namespace binding { return ctx0; } + template <> std::shared_ptr - make_context_one(std::shared_ptr
table, t_schema schema, std::vector pivots, - t_filter_op combiner, std::vector filters, std::vector aggregates, - std::vector sorts, std::int32_t pivot_depth, std::string name) { - auto cfg = t_config(pivots, aggregates, combiner, filters); + make_context(std::shared_ptr
table, const t_schema& schema, + const t_view_config& view_config, std::string name) { + auto row_pivots = view_config.get_row_pivots(); + auto aggspecs = view_config.get_aggspecs(); + auto filter_op = view_config.get_filter_op(); + auto fterm = view_config.get_fterm(); + auto sortspec = view_config.get_sortspec(); + auto row_pivot_depth = view_config.get_row_pivot_depth(); + + auto cfg = t_config(row_pivots, aggspecs, filter_op, fterm); auto ctx1 = std::make_shared(schema, cfg); ctx1->init(); - ctx1->sort_by(sorts); + ctx1->sort_by(sortspec); auto pool = table->get_pool(); auto gnode = table->get_gnode(); pool->register_context(gnode->get_id(), name, ONE_SIDED_CONTEXT, reinterpret_cast(ctx1.get())); - if (pivot_depth > -1) { - ctx1->set_depth(pivot_depth - 1); + if (row_pivot_depth > -1) { + ctx1->set_depth(row_pivot_depth - 1); } else { - ctx1->set_depth(pivots.size()); + ctx1->set_depth(row_pivots.size()); } return ctx1; } + template <> std::shared_ptr - make_context_two(std::shared_ptr
table, t_schema schema, - std::vector rpivots, std::vector cpivots, t_filter_op combiner, - std::vector filters, std::vector aggregates, - std::vector sorts, std::vector col_sorts, - std::int32_t rpivot_depth, std::int32_t cpivot_depth, bool column_only, - std::string name) { - t_totals total = sorts.size() > 0 ? TOTALS_BEFORE : TOTALS_HIDDEN; - - auto cfg - = t_config(rpivots, cpivots, aggregates, total, combiner, filters, column_only); + make_context(std::shared_ptr
table, const t_schema& schema, + const t_view_config& view_config, std::string name) { + bool column_only = view_config.is_column_only(); + auto row_pivots = view_config.get_row_pivots(); + auto column_pivots = view_config.get_column_pivots(); + auto aggspecs = view_config.get_aggspecs(); + auto filter_op = view_config.get_filter_op(); + auto fterm = view_config.get_fterm(); + auto sortspec = view_config.get_sortspec(); + auto col_sortspec = view_config.get_col_sortspec(); + auto row_pivot_depth = view_config.get_row_pivot_depth(); + auto column_pivot_depth = view_config.get_column_pivot_depth(); + + t_totals total = sortspec.size() > 0 ? TOTALS_BEFORE : TOTALS_HIDDEN; + + auto cfg = t_config( + row_pivots, column_pivots, aggspecs, total, filter_op, fterm, column_only); auto ctx2 = std::make_shared(schema, cfg); ctx2->init(); @@ -1730,24 +1518,24 @@ namespace binding { pool->register_context(gnode->get_id(), name, TWO_SIDED_CONTEXT, reinterpret_cast(ctx2.get())); - if (rpivot_depth > -1) { - ctx2->set_depth(t_header::HEADER_ROW, rpivot_depth - 1); + if (row_pivot_depth > -1) { + ctx2->set_depth(t_header::HEADER_ROW, row_pivot_depth - 1); } else { - ctx2->set_depth(t_header::HEADER_ROW, rpivots.size()); + ctx2->set_depth(t_header::HEADER_ROW, row_pivots.size()); } - if (cpivot_depth > -1) { - ctx2->set_depth(t_header::HEADER_COLUMN, cpivot_depth - 1); + if (column_pivot_depth > -1) { + ctx2->set_depth(t_header::HEADER_COLUMN, column_pivot_depth - 1); } else { - ctx2->set_depth(t_header::HEADER_COLUMN, cpivots.size()); + ctx2->set_depth(t_header::HEADER_COLUMN, column_pivots.size()); } - if (sorts.size() > 0) { - ctx2->sort_by(sorts); + if (sortspec.size() > 0) { + ctx2->sort_by(sortspec); } - if (col_sorts.size() > 0) { - ctx2->column_sort_by(col_sorts); + if (col_sortspec.size() > 0) { + ctx2->column_sort_by(col_sortspec); } return ctx2; @@ -1842,7 +1630,7 @@ EMSCRIPTEN_BINDINGS(perspective) { class_>("View_ctx0") .constructor, std::shared_ptr, std::string, std::string, - t_config>() + t_view_config>() .smart_ptr>>("shared_ptr") .function("sides", &View::sides) .function("num_rows", &View::num_rows) @@ -1865,7 +1653,7 @@ EMSCRIPTEN_BINDINGS(perspective) { class_>("View_ctx1") .constructor, std::shared_ptr, std::string, std::string, - t_config>() + t_view_config>() .smart_ptr>>("shared_ptr") .function("sides", &View::sides) .function("num_rows", &View::num_rows) @@ -1891,7 +1679,7 @@ EMSCRIPTEN_BINDINGS(perspective) { class_>("View_ctx2") .constructor, std::shared_ptr, std::string, std::string, - t_config>() + t_view_config>() .smart_ptr>>("shared_ptr") .function("sides", &View::sides) .function("num_rows", &View::num_rows) @@ -1916,6 +1704,17 @@ EMSCRIPTEN_BINDINGS(perspective) { .function("get_column_dtype", &View::get_column_dtype) .function("is_column_only", &View::is_column_only); + /****************************************************************************** + * + * t_view_config + */ + class_("t_view_config") + .constructor, std::vector, + tsl::ordered_map, std::vector, + std::vector>>, + std::vector>, std::string, bool>() + .function("add_filter_term", &t_view_config::add_filter_term); + /****************************************************************************** * * t_data_table @@ -2043,13 +1842,16 @@ EMSCRIPTEN_BINDINGS(perspective) { * vector */ register_vector("std::vector"); + register_vector("std::vector"); register_vector("std::vector"); register_vector("std::vector"); register_vector("std::vector"); - register_vector>("std::vector>"); - register_vector("std::vector"); register_vector("std::vector"); register_vector("std::vector"); + register_vector("std::vector"); + register_vector>("std::vector>"); + register_vector>("std::vector>"); + register_vector>("std::vector>"); /****************************************************************************** * @@ -2098,7 +1900,16 @@ EMSCRIPTEN_BINDINGS(perspective) { /****************************************************************************** * - * assorted functions + * Construct `std::vector`s + */ + function("make_string_vector", &make_vector); + function("make_val_vector", &make_vector); + function("make_2d_string_vector", &make_vector>); + function("make_2d_val_vector", &make_vector>); + + /****************************************************************************** + * + * Perspective functions */ function("make_table", &make_table); function("make_computed_table", &make_computed_table); @@ -2106,9 +1917,9 @@ EMSCRIPTEN_BINDINGS(perspective) { function("scalar_vec_to_string", &scalar_vec_to_string); function("table_add_computed_column", &table_add_computed_column); function("col_to_js_typed_array", &col_to_js_typed_array); - function("make_view_zero", &make_view_zero); - function("make_view_one", &make_view_one); - function("make_view_two", &make_view_two); + function("make_view_zero", &make_view); + function("make_view_one", &make_view); + function("make_view_two", &make_view); function("get_data_slice_zero", &get_data_slice, allow_raw_pointers()); function("get_from_data_slice_zero", &get_from_data_slice, allow_raw_pointers()); function("get_data_slice_one", &get_data_slice, allow_raw_pointers()); diff --git a/cpp/perspective/src/cpp/filter.cpp b/cpp/perspective/src/cpp/filter.cpp index 0b599fbe21..8e3c788bd4 100644 --- a/cpp/perspective/src/cpp/filter.cpp +++ b/cpp/perspective/src/cpp/filter.cpp @@ -14,15 +14,6 @@ namespace perspective { t_fterm::t_fterm() {} -t_fterm::t_fterm(const t_fterm_recipe& v) { - m_colname = v.m_colname; - m_op = v.m_op; - m_threshold = v.m_threshold; - m_bag = v.m_bag; - m_use_interned - = (m_op == FILTER_OP_EQ || m_op == FILTER_OP_NE) && m_threshold.m_type == DTYPE_STR; -} - t_fterm::t_fterm(const std::string& colname, t_filter_op op, t_tscalar threshold, const std::vector& bag, bool negated, bool is_primary) : m_colname(colname) @@ -55,16 +46,6 @@ t_fterm::coerce_numeric(t_dtype dtype) { } } -t_fterm_recipe -t_fterm::get_recipe() const { - t_fterm_recipe rv; - rv.m_colname = m_colname; - rv.m_op = m_op; - rv.m_threshold = m_threshold; - rv.m_bag = m_bag; - return rv; -} - std::string t_fterm::get_expr() const { std::stringstream ss; diff --git a/cpp/perspective/src/cpp/flat_traversal.cpp b/cpp/perspective/src/cpp/flat_traversal.cpp index 96a4e78f05..528a563213 100644 --- a/cpp/perspective/src/cpp/flat_traversal.cpp +++ b/cpp/perspective/src/cpp/flat_traversal.cpp @@ -102,10 +102,15 @@ t_ftrav::fill_sort_elem(std::shared_ptr state, const t_config& c out_elem.m_pkey = pkey; t_index sortby_size = m_sortby.size(); out_elem.m_row.reserve(sortby_size); - for (t_index idx = 0; idx < sortby_size; ++idx) { - t_index sortby_idx = m_sortby[idx].m_agg_index; - const std::string& colname = config.col_at(sortby_idx); - const std::string sortby_colname = config.get_sort_by(colname); + for (const t_sortspec& sort : m_sortby) { + // maintain backwards compatibility + std::string colname; + if (sort.m_colname != "") { + colname = config.get_sort_by(sort.m_colname); + } else { + colname = config.col_at(sort.m_agg_index); + } + const std::string& sortby_colname = config.get_sort_by(colname); out_elem.m_row.push_back( m_symtable.get_interned_tscalar(state->get(pkey, sortby_colname))); } @@ -117,10 +122,14 @@ t_ftrav::fill_sort_elem(std::shared_ptr state, const t_config& c out_elem.m_pkey = mknone(); t_index sortby_size = m_sortby.size(); out_elem.m_row.reserve(sortby_size); - for (t_index idx = 0; idx < sortby_size; ++idx) { - t_index sortby_idx = m_sortby[idx].m_agg_index; - const std::string& colname = config.col_at(sortby_idx); - const std::string sortby_colname = config.get_sort_by(colname); + for (const t_sortspec& sort : m_sortby) { + std::string colname; + if (sort.m_colname != "") { + colname = config.get_sort_by(sort.m_colname); + } else { + colname = config.col_at(sort.m_agg_index); + } + const std::string& sortby_colname = config.get_sort_by(colname); out_elem.m_row.push_back( get_interned_tscalar(row.at(config.get_colidx(sortby_colname)))); } diff --git a/cpp/perspective/src/cpp/sort_specification.cpp b/cpp/perspective/src/cpp/sort_specification.cpp index 8b7a71b54f..da93b9a3b3 100644 --- a/cpp/perspective/src/cpp/sort_specification.cpp +++ b/cpp/perspective/src/cpp/sort_specification.cpp @@ -13,6 +13,12 @@ namespace perspective { +t_sortspec::t_sortspec(const std::string& column_name, t_index agg_index, t_sorttype sort_type) + : m_colname(column_name) + , m_agg_index(agg_index) + , m_sort_type(sort_type) + , m_sortspec_type(SORTSPEC_TYPE_IDX) {} + t_sortspec::t_sortspec(t_index agg_index, t_sorttype sort_type) : m_agg_index(agg_index) , m_sort_type(sort_type) diff --git a/cpp/perspective/src/cpp/view.cpp b/cpp/perspective/src/cpp/view.cpp index 4afa1576fb..1d2089111b 100644 --- a/cpp/perspective/src/cpp/view.cpp +++ b/cpp/perspective/src/cpp/view.cpp @@ -14,27 +14,20 @@ namespace perspective { template View::View(std::shared_ptr
table, std::shared_ptr ctx, std::string name, - std::string separator, t_config config) + std::string separator, t_view_config view_config) : m_table(table) , m_ctx(ctx) , m_name(name) , m_separator(separator) , m_col_offset(0) - , m_config(config) { + , m_view_config(view_config) { - // We should deprecate t_pivot and just use string column names throughout - for (const t_pivot& rp : m_config.get_row_pivots()) { - m_row_pivots.push_back(rp.name()); - } - - for (const t_pivot& cp : m_config.get_column_pivots()) { - m_column_pivots.push_back(cp.name()); - } - - m_aggregates = m_config.get_aggregates(); - m_columns = m_config.get_column_names(); - m_filter = m_config.get_fterms(); - m_sort = m_config.get_sortspecs(); + m_row_pivots = m_view_config.get_row_pivots(); + m_column_pivots = m_view_config.get_column_pivots(); + m_aggregates = m_view_config.get_aggspecs(); + m_columns = m_view_config.get_columns(); + m_filter = m_view_config.get_fterm(); + m_sort = m_view_config.get_sortspec(); // configure data window for column-only rows is_column_only() ? m_row_offset = 1 : m_row_offset = 0; @@ -47,6 +40,12 @@ View::~View() { pool->unregister_context(gnode->get_id(), m_name); } +template +t_view_config +View::get_view_config() const { + return m_view_config; +} + template <> std::int32_t View::sides() const { @@ -465,7 +464,7 @@ View::get_column_dtype(t_uindex idx) const { template bool View::is_column_only() const { - return m_config.is_column_only(); + return m_view_config.is_column_only(); } /****************************************************************************** @@ -473,39 +472,26 @@ View::is_column_only() const { * Private */ -/* template -std::int32_t -View::_num_hidden_cols() { - std::int32_t hidden = 0; - for (const t_sortspec& sort : m_sort) { - } - return hidden; -} */ - template std::string View::_map_aggregate_types( const std::string& name, const std::string& typestring) const { - std::vector INTEGER_AGGS - = {"distinct_count", "distinct count", "distinctcount", "distinct", "count"}; - std::vector FLOAT_AGGS - = {"avg", "mean", "mean by count", "mean_by_count", "weighted mean", "weighted_mean", - "pct sum parent", "pct_sum_parent", "pct sum grand total", "pct_sum_grand_total"}; for (const t_aggspec& agg : m_aggregates) { if (agg.name() == name) { - std::string agg_str = agg.agg_str(); - bool int_agg = std::find(INTEGER_AGGS.begin(), INTEGER_AGGS.end(), agg_str) - != INTEGER_AGGS.end(); - bool float_agg - = std::find(FLOAT_AGGS.begin(), FLOAT_AGGS.end(), agg_str) != FLOAT_AGGS.end(); - - if (int_agg) { - return "integer"; - } else if (float_agg) { - return "float"; - } else { - return typestring; + switch (agg.agg()) { + case AGGTYPE_DISTINCT_COUNT: + case AGGTYPE_COUNT: { + return "integer"; + } break; + case AGGTYPE_MEAN: + case AGGTYPE_MEAN_BY_COUNT: + case AGGTYPE_WEIGHTED_MEAN: + case AGGTYPE_PCT_SUM_PARENT: + case AGGTYPE_PCT_SUM_GRAND_TOTAL: { + return "float"; + } break; + default: { return typestring; } break; } } } diff --git a/cpp/perspective/src/cpp/view_config.cpp b/cpp/perspective/src/cpp/view_config.cpp new file mode 100644 index 0000000000..92ca0b01b5 --- /dev/null +++ b/cpp/perspective/src/cpp/view_config.cpp @@ -0,0 +1,253 @@ +/****************************************************************************** + * + * Copyright (c) 2019, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ + +#include + +namespace perspective { + +t_view_config::t_view_config(std::vector row_pivots, + std::vector column_pivots, + tsl::ordered_map aggregates, std::vector columns, + std::vector>> filter, + std::vector> sort, std::string filter_op, bool column_only) + : m_init(false) + , m_row_pivots(row_pivots) + , m_column_pivots(column_pivots) + , m_aggregates(aggregates) + , m_columns(columns) + , m_filter(filter) + , m_sort(sort) + , m_row_pivot_depth(-1) + , m_column_pivot_depth(-1) + , m_filter_op(filter_op) + , m_column_only(column_only) {} + +void +t_view_config::init(const t_schema& schema) { + fill_aggspecs(schema); + fill_fterm(); + fill_sortspec(); + + m_init = true; +} + +void +t_view_config::add_filter_term( + std::tuple> term) { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + m_filter.push_back(term); +} + +void +t_view_config::set_row_pivot_depth(std::int32_t depth) { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + m_row_pivot_depth = depth; +} + +void +t_view_config::set_column_pivot_depth(std::int32_t depth) { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + m_column_pivot_depth = depth; +} + +std::vector +t_view_config::get_row_pivots() const { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + return m_row_pivots; +} + +std::vector +t_view_config::get_column_pivots() const { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + return m_column_pivots; +} + +std::vector +t_view_config::get_aggspecs() const { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + return m_aggspecs; +} + +std::vector +t_view_config::get_columns() const { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + return m_columns; +} + +std::vector +t_view_config::get_fterm() const { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + return m_fterm; +} + +std::vector +t_view_config::get_sortspec() const { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + return m_sortspec; +} + +std::vector +t_view_config::get_col_sortspec() const { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + return m_col_sortspec; +} + +t_filter_op +t_view_config::get_filter_op() const { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + return str_to_filter_op(m_filter_op); +} + +bool +t_view_config::is_column_only() const { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + return m_column_only; +} + +std::int32_t +t_view_config::get_row_pivot_depth() const { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + return m_row_pivot_depth; +} + +std::int32_t +t_view_config::get_column_pivot_depth() const { + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + return m_column_pivot_depth; +} + +// PRIVATE +void +t_view_config::fill_aggspecs(const t_schema& schema) { + /** + * Provide aggregates for columns that are shown but NOT specified in `m_aggregates`. + */ + for (const std::string& column : m_columns) { + if (m_aggregates.count(column) != 0) { + continue; + } + + t_dtype dtype = schema.get_dtype(column); + std::vector dependencies{t_dep(column, DEPTYPE_COLUMN)}; + t_aggtype agg_type + = t_aggtype::AGGTYPE_ANY; // use aggtype here since we are not parsing aggs + + if (!m_column_only) { + agg_type = _get_default_aggregate(dtype); + } + + // create aggregate specification, and memoize the column name + m_aggspecs.push_back(t_aggspec(column, agg_type, dependencies)); + m_aggregate_names.push_back(column); + } + + // Construct aggregates from config object + for (auto const& iter : m_aggregates) { + auto column = iter.first; + auto aggregate = iter.second; + if (std::find(m_columns.begin(), m_columns.end(), column) == m_columns.end()) { + continue; + } + + std::vector dependencies{t_dep(column, DEPTYPE_COLUMN)}; + t_aggtype agg_type; + + if (m_column_only) { + agg_type = t_aggtype::AGGTYPE_ANY; + } else { + agg_type = str_to_aggtype(aggregate); + } + + if (agg_type == AGGTYPE_FIRST || agg_type == AGGTYPE_LAST) { + dependencies.push_back(t_dep("psp_pkey", DEPTYPE_COLUMN)); + m_aggspecs.push_back( + t_aggspec(column, column, agg_type, dependencies, SORTTYPE_ASCENDING)); + } else { + m_aggspecs.push_back(t_aggspec(column, agg_type, dependencies)); + } + + m_aggregate_names.push_back(column); + } + + // construct aggspecs for hidden sorts + for (auto sort : m_sort) { + std::string column = sort[0]; + + bool is_hidden_column + = std::find(m_columns.begin(), m_columns.end(), column) == m_columns.end(); + + if (is_hidden_column) { + bool is_pivot = (std::find(m_row_pivots.begin(), m_row_pivots.end(), column) + != m_row_pivots.end()) + || (std::find(m_column_pivots.begin(), m_column_pivots.end(), column) + != m_column_pivots.end()); + + std::vector dependencies{t_dep(column, DEPTYPE_COLUMN)}; + t_aggtype agg_type; + + // use the `any` agg for columns used as pivots/column_only views + if (is_pivot || m_row_pivots.size() == 0 || m_column_only) { + agg_type = t_aggtype::AGGTYPE_ANY; + } else { + t_dtype dtype = schema.get_dtype(column); + agg_type = _get_default_aggregate(dtype); + } + + m_aggspecs.push_back(t_aggspec(column, agg_type, dependencies)); + m_aggregate_names.push_back(column); + } + } +} + +void +t_view_config::fill_fterm() { + for (auto filter : m_filter) { + t_filter_op op = str_to_filter_op(std::get<1>(filter)); + switch (op) { + case FILTER_OP_NOT_IN: + case FILTER_OP_IN: { + m_fterm.push_back( + t_fterm(std::get<0>(filter), op, mktscalar(0), std::get<2>(filter))); + } break; + default: { + t_tscalar filter_term = std::get<2>(filter)[0]; + m_fterm.push_back( + t_fterm(std::get<0>(filter), op, filter_term, std::vector())); + } + } + } +} + +void +t_view_config::fill_sortspec() { + for (auto s : m_sort) { + t_index agg_index = get_aggregate_index(s[0]); + t_sorttype sort_type = str_to_sorttype(s[1]); + + auto spec = t_sortspec(s[0], agg_index, sort_type); + + bool is_column_sort = s[1].find("col") != std::string::npos; + if (is_column_sort) { + m_col_sortspec.push_back(spec); + } else { + m_sortspec.push_back(spec); + } + } +} + +t_index +t_view_config::get_aggregate_index(const std::string& column) const { + auto it = std::find(m_aggregate_names.begin(), m_aggregate_names.end(), column); + if (it != m_aggregate_names.end()) { + return t_index(std::distance(m_aggregate_names.begin(), it)); + } + return t_index(); +} + +} // end namespace perspective \ No newline at end of file diff --git a/cpp/perspective/src/include/perspective/aggspec.h b/cpp/perspective/src/include/perspective/aggspec.h index 8dc5186006..ed3530c108 100644 --- a/cpp/perspective/src/include/perspective/aggspec.h +++ b/cpp/perspective/src/include/perspective/aggspec.h @@ -25,29 +25,12 @@ struct PERSPECTIVE_EXPORT t_col_name_type { t_dtype m_type; }; -struct PERSPECTIVE_EXPORT t_aggspec_recipe { - t_aggspec_recipe() {} - std::string m_name; - std::string m_disp_name; - t_aggtype m_agg; - std::vector m_dependencies; - std::vector m_odependencies; - t_sorttype m_sort_type; - t_uindex m_agg_one_idx; - t_uindex m_agg_two_idx; - double m_agg_one_weight; - double m_agg_two_weight; - t_invmode m_invmode; -}; - class PERSPECTIVE_EXPORT t_aggspec { public: t_aggspec(); ~t_aggspec(); - t_aggspec(const t_aggspec_recipe& v); - t_aggspec( const std::string& aggname, t_aggtype agg, const std::vector& dependencies); @@ -64,6 +47,7 @@ class PERSPECTIVE_EXPORT t_aggspec { t_aggspec(const std::string& aggname, const std::string& disp_aggname, t_aggtype agg, t_uindex agg_one_idx, t_uindex agg_two_idx, double agg_one_weight, double agg_two_weight); + std::string name() const; t_tscalar name_scalar() const; std::string disp_name() const; @@ -93,8 +77,6 @@ class PERSPECTIVE_EXPORT t_aggspec { std::string get_first_depname() const; - t_aggspec_recipe get_recipe() const; - private: std::string m_name; std::string m_disp_name; diff --git a/cpp/perspective/src/include/perspective/binding.h b/cpp/perspective/src/include/perspective/binding.h index f757c98559..fed8042f73 100644 --- a/cpp/perspective/src/include/perspective/binding.h +++ b/cpp/perspective/src/include/perspective/binding.h @@ -20,6 +20,7 @@ #include #include #include +#include #include #include #include @@ -58,56 +59,9 @@ namespace binding { /****************************************************************************** * - * Data Loading + * Manipulate scalar values */ - t_index _get_aggregate_index(const std::vector& agg_names, std::string name); - - std::vector _get_aggregate_names(const std::vector& aggs); - - /** - * @brief Calculate aggregates specified in `j_aggs` and use default aggregates for - * columns marked in `columns`. - * - * @tparam - * @param schema - * @param row_pivots - * @param column_pivots - * @param sortbys - * @param columns - * @param j_aggs - */ - template - std::vector _get_aggspecs(const t_schema& schema, - const std::vector& row_pivots, - const std::vector& column_pivots, bool column_only, - const std::vector& columns, const std::vector& sortbys, T j_aggs); - - /** - * @brief Retrieve and validate how we sort the dataset in the view. - * - * @tparam T - * @param columns - * @param is_column_sort - * @param sortbys - * @return std::vector - */ - template - std::vector _get_sort(const std::vector& columns, - bool is_column_sort, const std::vector& sortbys); - - /** - * @brief From the binding language, retrieve what we need to filter the dataset by. - * - * @tparam T - * @param schema - * @param j_date_parser - * @param j_filters - * @return std::vector - */ - template - std::vector _get_fterms(const t_schema& schema, T j_date_parser, T j_filters); - /** * @brief Converts a `t_scalar` to a value in the binding language. * @@ -271,57 +225,58 @@ namespace binding { template std::shared_ptr
make_computed_table(std::shared_ptr
table, T computed); + /****************************************************************************** + * + * View API + */ + /** - * @brief Extracts and validates the config from the binding language, - * creating a t_config for the new View. + * @brief For date/datetime values, is the filter term a valid date? + * + * Otherwise, make sure the filter term is not null/undefined. * * @tparam T - * @param schema - * @param separator + * @param type * @param date_parser - * @param config - * @return t_config + * @param filter + * @return true + * @return false */ template - t_config make_view_config( - const t_schema& schema, std::string separator, T date_parser, T config); + bool is_valid_filter(t_dtype type, T date_parser, T filter_term); /** - * @brief Create a new zero-sided view. - * - * Zero-sided views have no aggregates applied. + * @brief Create a filter by parsing the filter term from the binding language. * * @tparam T - * @param table - * @param name - * @param separator - * @param config + * @param schema * @param date_parser - * @return std::shared_ptr> + * @param filter + * @return std::tuple> */ template - std::shared_ptr> make_view_zero(std::shared_ptr
table, std::string name, - std::string separator, T config, T date_parser); - + std::tuple> make_filter_term( + t_dtype type, T date_parser, std::vector filter); /** - * @brief Create a new one-sided view. - * - * One-sided views have one or more `row-pivots` applied, + * @brief Create a `t_view_config` object from the binding language's `view_config` object. * * @tparam T - * @param table - * @param name - * @param separator - * @param config + * @param schema * @param date_parser - * @return std::shared_ptr> + * @param config + * @return t_config */ template - std::shared_ptr> make_view_one(std::shared_ptr
table, std::string name, - std::string separator, T config, T date_parser); + t_view_config make_view_config(const t_schema& schema, T date_parser, T config); /** - * @brief Create a new two-sided view. + * @brief Create a new view. + * + * Zero-sided views have no pivots or aggregates applied. + * + * Views are backed by an underlying `t_ctx_*` object, represented by the `CTX_T` template. + * + * One-sided views have one or more `row-pivots` applied. * * Two sided views have one or more `row-pivots` and `column-pivots` applied, or they have * one or more `column-pivots` applied without any row pivots, hence the term `column_only`. @@ -332,45 +287,26 @@ namespace binding { * @param separator * @param config * @param date_parser - * @return std::shared_ptr> + * @return std::shared_ptr> */ - template - std::shared_ptr> make_view_two(std::shared_ptr
table, std::string name, - std::string separator, T config, T date_parser); + template + std::shared_ptr> make_view(std::shared_ptr
table, std::string name, + std::string separator, T view_config, T date_parser); /** - * @brief Create a new zero-sided context. + * @brief Create a new context of type `CTX_T`, which will be one of 3 types: * - * Contexts contain the underlying aggregates, sort specifications, filter terms, and other - * metadata allowing for data manipulation and view creation. + * `t_ctx0`, `t_ctx1`, `t_ctx2`. * - * @return std::shared_ptr - */ - std::shared_ptr make_context_zero(std::shared_ptr
table, t_schema schema, - t_filter_op combiner, std::vector columns, std::vector filters, - std::vector sorts, std::string name); - - /** - * @brief Create a new one-sided context. * - * @return std::shared_ptr - */ - std::shared_ptr make_context_one(std::shared_ptr
table, t_schema schema, - std::vector pivots, t_filter_op combiner, std::vector filters, - std::vector aggregates, std::vector sorts, - std::int32_t pivot_depth, std::string name); - - /** - * @brief Create a new two-sided context. + * Contexts contain the underlying aggregates, sort specifications, filter terms, and other + * metadata allowing for data manipulation and view creation. * - * @return std::shared_ptr + * @return std::shared_ptr */ - std::shared_ptr make_context_two(std::shared_ptr
table, t_schema schema, - std::vector rpivots, std::vector cpivots, t_filter_op combiner, - std::vector filters, std::vector aggregates, - std::vector sorts, std::vector col_sorts, - std::int32_t rpivot_depth, std::int32_t cpivot_depth, bool column_only, - std::string name); + template + std::shared_ptr make_context(std::shared_ptr
table, const t_schema& schema, + const t_view_config& view_config, std::string name); /** * @brief Get a slice of data for a single column, serialized to t_val. diff --git a/cpp/perspective/src/include/perspective/config.h b/cpp/perspective/src/include/perspective/config.h index 2d2d6f6d6d..cce544dc2f 100644 --- a/cpp/perspective/src/include/perspective/config.h +++ b/cpp/perspective/src/include/perspective/config.h @@ -20,83 +20,91 @@ namespace perspective { -struct PERSPECTIVE_EXPORT t_config_recipe { - t_config_recipe(); - - std::vector m_row_pivots; - std::vector m_col_pivots; - std::vector> m_sortby; - std::vector> m_col_sortby; - std::vector m_aggregates; - std::vector m_detail_columns; - t_totals m_totals; - t_filter_op m_combiner; - std::vector m_fterms; - bool m_handle_nan_sort; - std::string m_parent_pkey_column; - std::string m_child_pkey_column; - std::string m_grouping_label_column; - t_fmode m_fmode; - std::vector m_filter_exprs; -}; - +/** + * @brief `t_config` contains metadata for the `View` and `t_ctx*` structures, containing + * specifications for how pivots, columns, filters, and sorts should be constructed. + * + */ class PERSPECTIVE_EXPORT t_config { public: - t_config(); - t_config(const t_config_recipe& r); - t_config(const std::vector& row_pivots, const std::vector& aggregates); - t_config(const std::vector& row_pivots, const std::vector& col_pivots, - const std::vector& aggregates, - const std::vector& detail_columns, const t_totals totals, - const std::vector& sort_pivot, - const std::vector& sort_pivot_by, t_filter_op combiner, - const std::vector& fterms, bool handle_nan_sort, - const std::string& parent_pkey_column, const std::string& child_pkey_column, - const std::string& grouping_label_column, t_fmode fmode, - const std::vector& filter_exprs, const std::string& grand_agg_str); - - // view config + /** + * @brief Construct a config for a `View` object. Pivots are passed in as vectors of + * strings, which are converted to `t_pivot` objects. + * + * @param row_pivots + * @param column_pivots + * @param aggregates + * @param sortspecs + * @param col_sortspecs + * @param combiner + * @param fterms + * @param col_names + * @param column_only + */ t_config(const std::vector& row_pivots, const std::vector& column_pivots, const std::vector& aggregates, const std::vector& sortspecs, const std::vector& col_sortspecs, t_filter_op combiner, const std::vector& fterms, const std::vector& col_names, bool column_only); - // grouped_pkeys + /** + * @brief Construct a new config for a `t_ctx0` object. + * + * @param detail_columns the columns to be displayed in the context + * @param combiner + * @param fterms specifications for filtering down the context + */ + t_config(const std::vector& detail_columns, t_filter_op combiner, + const std::vector& fterms); + + /** + * @brief Construct a new config for a `t_ctx1` object, which has 1 or more `row_pivot`s + * applied. + * + * @param row_pivots + * @param aggregates + * @param combiner + * @param fterms + */ t_config(const std::vector& row_pivots, - const std::vector& detail_columns, t_filter_op combiner, - const std::vector& fterms, const std::string& parent_pkey_column, - const std::string& child_pkey_column, const std::string& grouping_label_column); + const std::vector& aggregates, t_filter_op combiner, + const std::vector& fterms); - // ctx2 + /** + * @brief Construct a new config for a `t_ctx2` object, which has 1 or more `row_pivot`s and + * 1 or more `col_pivot`s applied. + * + * @param row_pivots + * @param col_pivots + * @param aggregates + * @param totals + * @param combiner + * @param fterms + * @param column_only + */ t_config(const std::vector& row_pivots, - const std::vector& col_pivots, const std::vector& aggregates); + const std::vector& col_pivots, const std::vector& aggregates, + const t_totals totals, t_filter_op combiner, const std::vector& fterms, + bool column_only); - t_config(const std::vector& row_pivots, const std::vector& col_pivots, - const std::vector& aggregates, const t_totals totals, t_filter_op combiner, - const std::vector& fterms, bool column_only); + // Constructors used for C++ tests, not exposed to other parts of the engine + t_config(const std::vector& row_pivots, + const std::vector& col_pivots, const std::vector& aggregates); t_config(const std::vector& row_pivots, const std::vector& col_pivots, const std::vector& aggregates, const t_totals totals, t_filter_op combiner, const std::vector& fterms); - // t_ctx1 + t_config(const std::vector& row_pivots, const std::vector& aggregates); + t_config( const std::vector& row_pivots, const std::vector& aggregates); t_config(const std::vector& row_pivots, const t_aggspec& agg); - t_config(const std::vector& row_pivots, const std::vector& aggregates, - t_filter_op combiner, const std::vector& fterms); - - t_config(const std::vector& row_pivots, - const std::vector& aggregates, t_filter_op combiner, - const std::vector& fterms); - - // t_ctx0 t_config(const std::vector& detail_columns); - t_config(const std::vector& detail_columns, t_filter_op combiner, - const std::vector& fterms); + + t_config(); void setup(const std::vector& detail_columns, const std::vector& sort_pivot, @@ -150,8 +158,6 @@ class PERSPECTIVE_EXPORT t_config { const std::string& get_grouping_label_column() const; - t_config_recipe get_recipe() const; - std::string unity_get_column_name(t_uindex idx) const; std::string unity_get_column_display_name(t_uindex idx) const; t_fmode get_fmode() const; diff --git a/cpp/perspective/src/include/perspective/emscripten.h b/cpp/perspective/src/include/perspective/emscripten.h index de8e5aa256..692ba468ce 100644 --- a/cpp/perspective/src/include/perspective/emscripten.h +++ b/cpp/perspective/src/include/perspective/emscripten.h @@ -22,6 +22,14 @@ typedef emscripten::val t_val; namespace perspective { namespace binding { + /** + * @brief Helper function for creating `std::vector`s for use in Javascript. + * + * @tparam T + * @return std::vector + */ + template + std::vector make_vector(); /** * @brief namespace `js_typed_array` contains utility bindings that initialize typed arrays diff --git a/cpp/perspective/src/include/perspective/filter.h b/cpp/perspective/src/include/perspective/filter.h index d7a281180f..fccc437c86 100644 --- a/cpp/perspective/src/include/perspective/filter.h +++ b/cpp/perspective/src/include/perspective/filter.h @@ -125,20 +125,9 @@ struct t_operator_contains { } }; -struct PERSPECTIVE_EXPORT t_fterm_recipe { - t_fterm_recipe() {} - - std::string m_colname; - t_filter_op m_op; - t_tscalar m_threshold; - std::vector m_bag; -}; - struct PERSPECTIVE_EXPORT t_fterm { t_fterm(); - t_fterm(const t_fterm_recipe& v); - t_fterm(const std::string& colname, t_filter_op op, t_tscalar threshold, const std::vector& bag); @@ -164,7 +153,6 @@ struct PERSPECTIVE_EXPORT t_fterm { std::string get_expr() const; void coerce_numeric(t_dtype dtype); - t_fterm_recipe get_recipe() const; std::string m_colname; t_filter_op m_op; diff --git a/cpp/perspective/src/include/perspective/slice.h b/cpp/perspective/src/include/perspective/slice.h index 76dbe4cdc2..28eeba3558 100644 --- a/cpp/perspective/src/include/perspective/slice.h +++ b/cpp/perspective/src/include/perspective/slice.h @@ -30,7 +30,6 @@ class PERSPECTIVE_EXPORT t_slice { const std::vector& column_data() const; const std::vector& row_depth() const; const std::vector& column_depth() const; - const t_config_recipe& config_recipe() const; const std::vector& is_row_expanded() const; const std::vector& is_column_expanded() const; @@ -57,7 +56,6 @@ class PERSPECTIVE_EXPORT t_slice { std::vector m_is_root; std::vector m_is_row_expanded; std::vector m_is_column_expanded; - t_config_recipe m_config_recipe; std::vector m_row_depth; std::vector m_column_depth; }; diff --git a/cpp/perspective/src/include/perspective/sort_specification.h b/cpp/perspective/src/include/perspective/sort_specification.h index 2d8eb59b02..b026d68e4c 100644 --- a/cpp/perspective/src/include/perspective/sort_specification.h +++ b/cpp/perspective/src/include/perspective/sort_specification.h @@ -20,15 +20,16 @@ enum t_sortspec_type { SORTSPEC_TYPE_IDX, SORTSPEC_TYPE_COLNAME, SORTSPEC_TYPE_P struct PERSPECTIVE_EXPORT t_sortspec { t_sortspec(); + t_sortspec(const std::string& column_name, t_index agg_index, t_sorttype sort_type); t_sortspec(t_index agg_index, t_sorttype sort_type); t_sortspec(const std::vector& path, t_index agg_index, t_sorttype sort_type); bool operator==(const t_sortspec& s2) const; bool operator!=(const t_sortspec& s2) const; + std::string m_colname; t_index m_agg_index; t_sorttype m_sort_type; - bool m_colname; t_sortspec_type m_sortspec_type; std::vector m_path; }; diff --git a/cpp/perspective/src/include/perspective/view.h b/cpp/perspective/src/include/perspective/view.h index eb41ad9b9d..7f839cb4f2 100644 --- a/cpp/perspective/src/include/perspective/view.h +++ b/cpp/perspective/src/include/perspective/view.h @@ -12,12 +12,12 @@ #include #include #include -#include #include #include #include #include #include +#include #include #include #include @@ -28,10 +28,16 @@ template class PERSPECTIVE_EXPORT View { public: View(std::shared_ptr
table, std::shared_ptr ctx, std::string name, - std::string separator, t_config config); - + std::string separator, t_view_config view_config); ~View(); + /** + * @brief The `t_view_config` object that created this `View`. + * + * @return t_view_config + */ + t_view_config get_view_config() const; + /** * @brief The number of pivoted sides of this View. * @@ -179,6 +185,6 @@ class PERSPECTIVE_EXPORT View { t_uindex m_row_offset; t_uindex m_col_offset; - t_config m_config; + t_view_config m_view_config; }; } // end namespace perspective \ No newline at end of file diff --git a/cpp/perspective/src/include/perspective/view_config.h b/cpp/perspective/src/include/perspective/view_config.h new file mode 100644 index 0000000000..e50ec0f003 --- /dev/null +++ b/cpp/perspective/src/include/perspective/view_config.h @@ -0,0 +1,188 @@ +/****************************************************************************** + * + * Copyright (c) 2019, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace perspective { + +/** + * @brief The `t_view_config` API provides a unified view configuration object, which specifies + * how the `View` transforms the underlying `Table`. By storing configuration values in native + * C++ types, we allow easy integration with binding languages. + * + */ +class t_view_config { +public: + /** + * @brief Construct a `t_view_config` object. + * + * @param row_pivots + * @param column_pivots + * @param aggregates + * @param columns + * @param filter + * @param sort + */ + t_view_config(std::vector row_pivots, std::vector column_pivots, + tsl::ordered_map aggregates, std::vector columns, + std::vector>> filter, + std::vector> sort, std::string filter_op, bool column_only); + + /** + * @brief Given a `t_schema` specifying the underlying `Table`'s columns, construct the + * abstractions necessary for the core engine. + * + * @param schema + */ + void init(const t_schema& schema); + + /** + * @brief Add filter terms manually, as the filter term must be calculated from the value + * passed through the binding. + * + * @param term + */ + void add_filter_term(std::tuple> term); + + /** + * @brief Set the number of pivot levels the engine should generate. + * + * @param depth + */ + void set_row_pivot_depth(std::int32_t depth); + void set_column_pivot_depth(std::int32_t depth); + + std::vector get_row_pivots() const; + + std::vector get_column_pivots() const; + + std::vector get_aggspecs() const; + + std::vector get_columns() const; + + std::vector get_fterm() const; + + std::vector get_sortspec() const; + + std::vector get_col_sortspec() const; + + t_filter_op get_filter_op() const; + + bool is_column_only() const; + + std::int32_t get_row_pivot_depth() const; + std::int32_t get_column_pivot_depth() const; + +private: + bool m_init; + + /** + * @brief Fill the `m_aggspecs` vector with `t_aggspec` objects which define the view's + * aggregate settings. + * + * The method calculates aggregates using the columns marked for display (in `m_columns`), + * user-provided aggregates (in `m_aggregates`), and if sorts are applied using a column + * that is not displayed. + * + * At the same time, the method fills the `m_aggregate_names` vector with each column that + * has an aggregate applied. + * + * See documentation for the `m_aggregate_names` vector below for details regarding its + * logic. + * + * @param schema + * @return void + */ + void fill_aggspecs(const t_schema& schema); + + /** + * @brief Fill the `m_fterm` vector with `t_fterm` objects which define the view's filters. + * + * @return void + */ + void fill_fterm(); + + /** + * @brief Fill the `m_sortspec` vectors with `t_sortspec` objects which define the view's + * sorting. + * + * The method fills both the `m_sortspec` and `m_col_sortspec` vectors. The former is used + * in all views, and the latter refers to sorting based on a column when column pivots are + * applied; it is only used in 2-sided views. + * + * @return void + */ + void fill_sortspec(); + + /** + * @brief Given a column name, find its position in `m_aggregate_names`. Used for + * determining sort specifications. + * + * @param column + * @return t_index + */ + t_index get_aggregate_index(const std::string& column) const; + + // containers for primitive data that does not need transformation into abstractions + std::vector m_row_pivots; + std::vector m_column_pivots; + tsl::ordered_map m_aggregates; + std::vector m_columns; + std::vector>> m_filter; + std::vector> m_sort; + + /** + * @brief The ordered list of aggregate columns: + * + * 1. all columns marked as "shown" by the user in `m_columns` + * 2. all specified aggregates from `m_aggregates` + * 3. all "hidden sorts", i.e. columns to sort by that do not appear in `m_columns` + * + */ + std::vector m_aggregate_names; + + // store containers for abstractions which are used by the engine + std::vector m_aggspecs; + + std::vector m_fterm; + + std::vector m_sortspec; + + std::vector m_col_sortspec; + + /** + * @brief If specified, the number of pivot levels the engine should generate. + * + * Used in `expand` and `collapse` tree operations, and in the grid. + */ + std::int32_t m_row_pivot_depth; + std::int32_t m_column_pivot_depth; + + /** + * @brief the `t_filter_op` used to combine values inside the engine, stored as a string. + * + * Defaults to "and" unless specified. + */ + std::string m_filter_op; + + /** + * @brief whether the view is `column_only`, i.e. having > 1 `column_pivots` without any + * `row_pivots`. + * + */ + bool m_column_only; +}; +} // end namespace perspective \ No newline at end of file diff --git a/cpp/perspective/test/cpp/simple.cpp b/cpp/perspective/test/cpp/simple.cpp index 2b16fc424b..76f30268a7 100644 --- a/cpp/perspective/test/cpp/simple.cpp +++ b/cpp/perspective/test/cpp/simple.cpp @@ -2250,7 +2250,7 @@ TEST(GNODE_TEST, get_registered_contexts) std::vector expected_ctx0_pkeys{1_ts}; EXPECT_EQ(ctx0_pkeys, expected_ctx0_pkeys); - ctx0->sort_by(std::vector{{0, SORTTYPE_DESCENDING}}); + ctx0->sort_by(std::vector{{"i", 0, SORTTYPE_DESCENDING}}); EXPECT_EQ(ctx0->get_cell_data({{0, 0}}), std::vector{"b"_ts}); auto trees = gn->get_trees(); diff --git a/packages/perspective/src/js/emscripten.js b/packages/perspective/src/js/emscripten.js index 30e3a8dc5c..3d78b924e5 100644 --- a/packages/perspective/src/js/emscripten.js +++ b/packages/perspective/src/js/emscripten.js @@ -35,3 +35,19 @@ export const extract_map = function(map) { keys.delete(); return extracted; }; + +/** + * Given a C++ vector constructed in Emscripten, fill it with data. Assume that data types are already validated, + * thus Emscripten will throw an error if the vector is filled with the wrong type of data. + * + * @param {*} vector the `std::vector` to be filled + * @param {Array} arr the `Array` from which to draw data + * + * @private + */ +export const fill_vector = function(vector, arr) { + for (const elem of arr) { + vector.push_back(elem); + } + return vector; +}; diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index 004378b635..24f11c9a70 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -10,7 +10,7 @@ import * as defaults from "./config/constants.js"; import {DataAccessor} from "./data_accessor"; import {DateParser} from "./data_accessor/date_parser.js"; -import {extract_map} from "./emscripten.js"; +import {extract_map, fill_vector} from "./emscripten.js"; import {bindall, get_column_type} from "./utils.js"; import {Server} from "./api/server.js"; @@ -110,7 +110,7 @@ export default function(Module) { const pool = _Table.get_pool(); const table_id = _Table.get_id(); - console.log(table_id); + if (op == __MODULE__.t_op.OP_UPDATE || op == __MODULE__.t_op.OP_DELETE) { _set_process(pool, table_id); } else { @@ -253,17 +253,18 @@ export default function(Module) { * @class * @hideconstructor */ - function view(table, sides, config, name, callbacks) { + function view(table, sides, config, view_config, name, callbacks) { this._View = undefined; this.date_parser = new DateParser(); this.config = config || {}; + this.view_config = view_config || new view_config(); if (sides === 0) { - this._View = __MODULE__.make_view_zero(table._Table, name, defaults.COLUMN_SEPARATOR_STRING, this.config, this.date_parser); + this._View = __MODULE__.make_view_zero(table._Table, name, defaults.COLUMN_SEPARATOR_STRING, this.view_config, this.date_parser); } else if (sides === 1) { - this._View = __MODULE__.make_view_one(table._Table, name, defaults.COLUMN_SEPARATOR_STRING, this.config, this.date_parser); + this._View = __MODULE__.make_view_one(table._Table, name, defaults.COLUMN_SEPARATOR_STRING, this.view_config, this.date_parser); } else if (sides === 2) { - this._View = __MODULE__.make_view_two(table._Table, name, defaults.COLUMN_SEPARATOR_STRING, this.config, this.date_parser); + this._View = __MODULE__.make_view_two(table._Table, name, defaults.COLUMN_SEPARATOR_STRING, this.view_config, this.date_parser); } this.table = table; @@ -844,6 +845,74 @@ export default function(Module) { this._delete_callback = undefined; }; + /** + * A view config is a set of options that configures the underlying {@link module:perspective~view}, specifying + * its pivots, columns to show, aggregates, filters, and sorts. + * + * The view config receives an `Object` containing configuration options, and the `view_config` transforms it into a + * canonical format for interfacing with the core engine. + * + * Note This constructor is not public - view config objects should be created using standard + * Javascript `Object`s in the {@link module:perspective~table#view} method, which has an `options` parameter. + * + * @param {Object} config the configuration `Object` passed by the user to the {@link module:perspective~table#view} method. + * + * @class + * @hideconstructor + */ + function view_config(config) { + this.row_pivots = config.row_pivots || []; + this.column_pivots = config.column_pivots || []; + this.aggregates = config.aggregates || {}; + this.columns = config.columns; + this.filter = config.filter || []; + this.sort = config.sort || []; + this.filter_op = config.filter_op || "and"; + this.row_pivot_depth = config.row_pivot_depth; + this.column_pivot_depth = config.column_pivot_depth; + } + + /** + * Transform configuration items into `std::vector` objects for interface with C++. + * + * `this.aggregates` is not transformed into a C++ map, as the use of `ordered_map` in the engine + * makes binding more difficult. + */ + view_config.prototype.get_row_pivots = function() { + let vector = __MODULE__.make_string_vector(); + return fill_vector(vector, this.row_pivots); + }; + + view_config.prototype.get_column_pivots = function() { + let vector = __MODULE__.make_string_vector(); + return fill_vector(vector, this.column_pivots); + }; + + view_config.prototype.get_columns = function() { + let vector = __MODULE__.make_string_vector(); + return fill_vector(vector, this.columns); + }; + + view_config.prototype.get_filter = function() { + let vector = __MODULE__.make_2d_val_vector(); + for (let filter of this.filter) { + let filter_vector = __MODULE__.make_val_vector(); + let filled = fill_vector(filter_vector, filter); + vector.push_back(filled); + } + return vector; + }; + + view_config.prototype.get_sort = function() { + let vector = __MODULE__.make_2d_string_vector(); + for (let sort of this.sort) { + let sort_vector = __MODULE__.make_string_vector(); + let filled = fill_vector(sort_vector, sort); + vector.push_back(filled); + } + return vector; + }; + /****************************************************************************** * * Table @@ -1120,7 +1189,8 @@ export default function(Module) { sides = 0; } - let v = new view(this, sides, config, name, this.callbacks); + let vc = new view_config(config); + let v = new view(this, sides, config, vc, name, this.callbacks); this.views.push(v); return v; }; diff --git a/packages/perspective/test/js/filters.js b/packages/perspective/test/js/filters.js index cb00771300..0a710a5e33 100644 --- a/packages/perspective/test/js/filters.js +++ b/packages/perspective/test/js/filters.js @@ -341,7 +341,7 @@ module.exports = perspective => { table.delete(); }); - it("x > null", async function() { + it("x > null should be an invalid filter", async function() { var table = perspective.table({x: "float", y: "integer"}); const dataSet = [{x: 3.5, y: 1}, {x: 2.5, y: 1}, {x: null, y: 1}, {x: null, y: 1}, {x: 4.5, y: 2}, {x: null, y: 2}]; table.update(dataSet);