From acdc9eb68e65b8bf73fb9c055b53a3f672c9bad0 Mon Sep 17 00:00:00 2001 From: Jun Tan Date: Wed, 8 Jul 2020 10:29:55 -0400 Subject: [PATCH 01/11] Fix day_bucket to use local time, add edge case tests --- cpp/perspective/src/cpp/computed_function.cpp | 16 ++--- .../perspective/test/js/computed/datetime.js | 31 ++++++++++ .../perspective/table/_data_formatter.py | 1 + .../tests/table/test_table_datetime.py | 59 ++++++++++++++++++- 4 files changed, 99 insertions(+), 8 deletions(-) diff --git a/cpp/perspective/src/cpp/computed_function.cpp b/cpp/perspective/src/cpp/computed_function.cpp index fc11d9b058..16d70174da 100644 --- a/cpp/perspective/src/cpp/computed_function.cpp +++ b/cpp/perspective/src/cpp/computed_function.cpp @@ -617,18 +617,20 @@ t_tscalar day_bucket(t_tscalar x) { // Convert the timestamp to a `sys_time` (alias for `time_point`) date::sys_time ts(ms_timestamp); - // Create a copy of the timestamp with day precision - auto days = date::floor(ts); + // Use localtime so that the day of week is consistent with all output + // datetimes, which are in local time + std::time_t temp = std::chrono::system_clock::to_time_t(ts); - // Cast the `time_point` to contain year/month/day - auto ymd = date::year_month_day(days); + // Convert to a std::tm + std::tm* t = std::localtime(&temp); // Get the year and create a new `t_date` - std::int32_t year = static_cast(ymd.year()); + std::int32_t year = static_cast(t->tm_year + 1900); // Month in `t_date` is [0-11] - std::int32_t month = static_cast(ymd.month()) - 1; - std::uint32_t day = static_cast(ymd.day()); + std::int32_t month = static_cast(t->tm_mon); + std::uint32_t day = static_cast(t->tm_mday); + rval.set(t_date(year, month, day)); return rval; } diff --git a/packages/perspective/test/js/computed/datetime.js b/packages/perspective/test/js/computed/datetime.js index c20045a24f..d8e27eff12 100644 --- a/packages/perspective/test/js/computed/datetime.js +++ b/packages/perspective/test/js/computed/datetime.js @@ -1108,6 +1108,37 @@ module.exports = perspective => { table.delete(); }); + it("Bucket (D), datetime at UTC edge", async function() { + const table = perspective.table({ + a: "datetime" + }); + + const view = table.view({ + computed_columns: [ + { + column: "computed", + computed_function_name: "Bucket (D)", + inputs: ["a"] + } + ] + }); + + const schema = await view.schema(); + expect(schema).toEqual({ + a: "datetime", + computed: "date" + }); + + table.update({ + a: [new Date(2020, 0, 15, 23, 30, 15), null, undefined, new Date(2020, 3, 29, 23, 30, 0), new Date(2020, 4, 30, 23, 30, 15)] + }); + + let result = await view.to_columns(); + expect(result.computed.map(x => (x ? new Date(x) : null))).toEqual(result.a.map(x => (x ? common.day_bucket(x) : null))); + view.delete(); + table.delete(); + }); + it("Bucket (W), datetime", async function() { const table = perspective.table({ a: "datetime" diff --git a/python/perspective/perspective/table/_data_formatter.py b/python/perspective/perspective/table/_data_formatter.py index 4404cbd5df..0c1a61ebf9 100644 --- a/python/perspective/perspective/table/_data_formatter.py +++ b/python/perspective/perspective/table/_data_formatter.py @@ -104,6 +104,7 @@ def to_format(options, view, output_format): if output_format == 'numpy': for k, v in data.items(): # TODO push into C++ + print(type(v), v) data[k] = np.array(v) return data diff --git a/python/perspective/perspective/tests/table/test_table_datetime.py b/python/perspective/perspective/tests/table/test_table_datetime.py index 6ed01100da..7c29529693 100644 --- a/python/perspective/perspective/tests/table/test_table_datetime.py +++ b/python/perspective/perspective/tests/table/test_table_datetime.py @@ -1221,7 +1221,7 @@ def test_table_hour_of_day_in_PST(object): def test_table_day_of_week_edge_in_EST(object): """Make sure edge cases are fixed for day of week - if a local - time converted to UTC is in the next day, theday of week + time converted to UTC is in the next day, the day of week computation needs to be in local time.""" computed_columns = [{ "inputs": ["a"], @@ -1276,6 +1276,63 @@ def test_table_day_of_week_edge_in_PST(object): result = view.to_dict() assert result["computed"] == ["6 Friday"] + def test_table_day_bucket_edge_in_EST(object): + """Make sure edge cases are fixed for day_bucket - if a local + time converted to UTC is in the next day, the day_bucket + computation needs to be in local time.""" + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "day_bucket" + }] + + data = { + "a": [datetime(2020, 1, 31, 23, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == [datetime(2020, 1, 31)] + + def test_table_day_bucket_edge_in_CST(object): + os.environ["TZ"] = "US/Central" + time.tzset() + + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "day_bucket" + }] + + data = { + "a": [datetime(2020, 1, 31, 23, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == [datetime(2020, 1, 31)] + + def test_table_day_bucket_edge_in_PST(object): + os.environ["TZ"] = "US/Pacific" + time.tzset() + + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "day_bucket" + }] + + data = { + "a": [datetime(2020, 1, 31, 23, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == [datetime(2020, 1, 31)] + def test_table_month_of_year_edge_in_EST(object): """Make sure edge cases are fixed for month of year - if a local time converted to UTC is in the next month, the month of year From b4a60852370f48563ee894f7ede937afc2270112 Mon Sep 17 00:00:00 2001 From: Jun Tan Date: Wed, 8 Jul 2020 12:58:31 -0400 Subject: [PATCH 02/11] Fix month_bucket to use local time --- cpp/perspective/src/cpp/computed_function.cpp | 18 +- .../tests/table/test_table_datetime.py | 200 ++++++++++++++++-- 2 files changed, 194 insertions(+), 24 deletions(-) diff --git a/cpp/perspective/src/cpp/computed_function.cpp b/cpp/perspective/src/cpp/computed_function.cpp index 16d70174da..948547c4d1 100644 --- a/cpp/perspective/src/cpp/computed_function.cpp +++ b/cpp/perspective/src/cpp/computed_function.cpp @@ -715,20 +715,18 @@ t_tscalar month_bucket(t_tscalar x) { if (x.is_none() || !x.is_valid()) return rval; // Convert the int64 to a milliseconds duration timestamp - std::chrono::milliseconds timestamp(x.to_int64()); + std::chrono::milliseconds ms_timestamp(x.to_int64()); // Convert the timestamp to a `sys_time` (alias for `time_point`) - date::sys_time ts(timestamp); - - // Create a copy of the timestamp with day precision - auto days = date::floor(ts); + date::sys_time ts(ms_timestamp); - // Cast the `time_point` to contain year/month/day - auto ymd = date::year_month_day(days); + // Convert the timestamp to local time + std::time_t temp = std::chrono::system_clock::to_time_t(ts); + std::tm* t = std::localtime(&temp); - // Get the year and create a new `t_date` - std::int32_t year = static_cast(ymd.year()); - std::int32_t month = static_cast(ymd.month()) - 1; + // Use the `tm` to create the `t_date` + std::int32_t year = static_cast(t->tm_year + 1900); + std::int32_t month = static_cast(t->tm_mon); rval.set(t_date(year, month, 1)); return rval; } diff --git a/python/perspective/perspective/tests/table/test_table_datetime.py b/python/perspective/perspective/tests/table/test_table_datetime.py index 7c29529693..ec5e04c0a0 100644 --- a/python/perspective/perspective/tests/table/test_table_datetime.py +++ b/python/perspective/perspective/tests/table/test_table_datetime.py @@ -1276,6 +1276,63 @@ def test_table_day_of_week_edge_in_PST(object): result = view.to_dict() assert result["computed"] == ["6 Friday"] + def test_table_month_of_year_edge_in_EST(object): + """Make sure edge cases are fixed for month of year - if a local + time converted to UTC is in the next month, the month of year + computation needs to be in local time.""" + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "month_of_year" + }] + + data = { + "a": [datetime(2020, 1, 31, 23, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == ["01 January"] + + def test_table_month_of_year_edge_in_CST(object): + os.environ["TZ"] = "US/Central" + time.tzset() + + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "month_of_year" + }] + + data = { + "a": [datetime(2020, 1, 31, 23, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == ["01 January"] + + def test_table_month_of_year_edge_in_PST(object): + os.environ["TZ"] = "US/Pacific" + time.tzset() + + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "month_of_year" + }] + + data = { + "a": [datetime(2020, 1, 31, 23, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == ["01 January"] + def test_table_day_bucket_edge_in_EST(object): """Make sure edge cases are fixed for day_bucket - if a local time converted to UTC is in the next day, the day_bucket @@ -1333,62 +1390,177 @@ def test_table_day_bucket_edge_in_PST(object): result = view.to_dict() assert result["computed"] == [datetime(2020, 1, 31)] - def test_table_month_of_year_edge_in_EST(object): - """Make sure edge cases are fixed for month of year - if a local - time converted to UTC is in the next month, the month of year + @mark.skip + def test_table_week_bucket_edge_in_EST(object): + """Make sure edge cases are fixed for week_bucket - if a local + time converted to UTC is in the next day, the week_bucket computation needs to be in local time.""" computed_columns = [{ "inputs": ["a"], "column": "computed", - "computed_function_name": "month_of_year" + "computed_function_name": "week_bucket" }] data = { - "a": [datetime(2020, 1, 31, 23, 59)] + "a": [datetime(2020, 2, 2, 23, 59)] } table = Table(data) view = table.view(computed_columns=computed_columns) result = view.to_dict() - assert result["computed"] == ["01 January"] + assert result["computed"] == [datetime(2020, 1, 27)] - def test_table_month_of_year_edge_in_CST(object): + @mark.skip + def test_table_week_bucket_edge_in_CST(object): os.environ["TZ"] = "US/Central" time.tzset() computed_columns = [{ "inputs": ["a"], "column": "computed", - "computed_function_name": "month_of_year" + "computed_function_name": "week_bucket" }] data = { - "a": [datetime(2020, 1, 31, 23, 59)] + "a": [datetime(2020, 2, 2, 23, 59)] } table = Table(data) view = table.view(computed_columns=computed_columns) result = view.to_dict() - assert result["computed"] == ["01 January"] + assert result["computed"] == [datetime(2020, 1, 27)] - def test_table_month_of_year_edge_in_PST(object): + @mark.skip + def test_table_week_bucket_edge_in_PST(object): os.environ["TZ"] = "US/Pacific" time.tzset() computed_columns = [{ "inputs": ["a"], "column": "computed", - "computed_function_name": "month_of_year" + "computed_function_name": "week_bucket" }] data = { - "a": [datetime(2020, 1, 31, 23, 59)] + "a": [datetime(2020, 2, 2, 23, 59)] } table = Table(data) view = table.view(computed_columns=computed_columns) result = view.to_dict() - assert result["computed"] == ["01 January"] + assert result["computed"] == [datetime(2020, 1, 27)] + + def test_table_week_bucket_edge_flip_in_EST(object): + """Week bucket should flip backwards to last month.""" + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "week_bucket" + }] + + data = { + "a": [datetime(2020, 3, 1, 12, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == [datetime(2020, 2, 24)] + + def test_table_week_bucket_edge_flip_in_CST(object): + os.environ["TZ"] = "US/Central" + time.tzset() + + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "week_bucket" + }] + + data = { + "a": [datetime(2020, 3, 1, 12, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == [datetime(2020, 2, 24)] + + def test_table_week_bucket_edge_flip_in_PST(object): + os.environ["TZ"] = "US/Pacific" + time.tzset() + + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "week_bucket" + }] + + data = { + "a": [datetime(2020, 3, 1, 12, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == [datetime(2020, 2, 24)] + + def test_table_month_bucket_edge_in_EST(object): + """Make sure edge cases are fixed for month_bucket - if a local + time converted to UTC is in the next day, the month_bucket + computation needs to be in local time.""" + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "month_bucket" + }] + + data = { + "a": [datetime(2020, 6, 30, 23, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == [datetime(2020, 6, 1)] + + def test_table_month_bucket_edge_in_CST(object): + os.environ["TZ"] = "US/Central" + time.tzset() + + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "month_bucket" + }] + + data = { + "a": [datetime(2020, 6, 30, 23, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == [datetime(2020, 6, 1)] + + def test_table_month_bucket_edge_in_PST(object): + os.environ["TZ"] = "US/Pacific" + time.tzset() + + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "month_bucket" + }] + + data = { + "a": [datetime(2020, 6, 30, 23, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == [datetime(2020, 6, 1)] class TestTableDateTimePivots(object): From 96f450b8aa1a15fc743d542910cf3a7486046cf6 Mon Sep 17 00:00:00 2001 From: Jun Tan Date: Wed, 8 Jul 2020 13:06:06 -0400 Subject: [PATCH 03/11] year_bucket uses local time --- cpp/perspective/src/cpp/computed_function.cpp | 16 +++--- .../tests/table/test_table_datetime.py | 57 +++++++++++++++++++ 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/cpp/perspective/src/cpp/computed_function.cpp b/cpp/perspective/src/cpp/computed_function.cpp index 948547c4d1..9f5afb0985 100644 --- a/cpp/perspective/src/cpp/computed_function.cpp +++ b/cpp/perspective/src/cpp/computed_function.cpp @@ -747,19 +747,17 @@ t_tscalar year_bucket(t_tscalar x) { if (x.is_none() || !x.is_valid()) return rval; // Convert the int64 to a milliseconds duration timestamp - std::chrono::milliseconds timestamp(x.to_int64()); + std::chrono::milliseconds ms_timestamp(x.to_int64()); // Convert the timestamp to a `sys_time` (alias for `time_point`) - date::sys_time ts(timestamp); - - // Create a copy of the timestamp with day precision - auto days = date::floor(ts); + date::sys_time ts(ms_timestamp); - // Cast the `time_point` to contain year/month/day - auto ymd = date::year_month_day(days); + // Convert the timestamp to local time + std::time_t temp = std::chrono::system_clock::to_time_t(ts); + std::tm* t = std::localtime(&temp); - // Get the year and create a new `t_date` - std::int32_t year = static_cast(ymd.year()); + // Use the `tm` to create the `t_date` + std::int32_t year = static_cast(t->tm_year + 1900); rval.set(t_date(year, 0, 1)); return rval; } diff --git a/python/perspective/perspective/tests/table/test_table_datetime.py b/python/perspective/perspective/tests/table/test_table_datetime.py index ec5e04c0a0..0c36e889ca 100644 --- a/python/perspective/perspective/tests/table/test_table_datetime.py +++ b/python/perspective/perspective/tests/table/test_table_datetime.py @@ -1562,6 +1562,63 @@ def test_table_month_bucket_edge_in_PST(object): result = view.to_dict() assert result["computed"] == [datetime(2020, 6, 1)] + def test_table_year_bucket_edge_in_EST(object): + """Make sure edge cases are fixed for year_bucket - if a local + time converted to UTC is in the next day, the year_bucket + computation needs to be in local time.""" + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "year_bucket" + }] + + data = { + "a": [datetime(2019, 12, 31, 23, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == [datetime(2019, 1, 1)] + + def test_table_year_bucket_edge_in_CST(object): + os.environ["TZ"] = "US/Central" + time.tzset() + + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "year_bucket" + }] + + data = { + "a": [datetime(2019, 12, 31, 23, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == [datetime(2019, 1, 1)] + + def test_table_year_bucket_edge_in_PST(object): + os.environ["TZ"] = "US/Pacific" + time.tzset() + + computed_columns = [{ + "inputs": ["a"], + "column": "computed", + "computed_function_name": "year_bucket" + }] + + data = { + "a": [datetime(2019, 12, 31, 23, 59)] + } + + table = Table(data) + view = table.view(computed_columns=computed_columns) + result = view.to_dict() + assert result["computed"] == [datetime(2019, 1, 1)] + class TestTableDateTimePivots(object): From b1eb8b871b33e24b55408758938626451acbefc3 Mon Sep 17 00:00:00 2001 From: Jun Tan Date: Wed, 8 Jul 2020 14:27:21 -0400 Subject: [PATCH 04/11] Fix week_bucket to use local time --- cpp/perspective/src/cpp/computed_function.cpp | 27 ++++++++++++++----- .../tests/table/test_table_datetime.py | 5 +--- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/cpp/perspective/src/cpp/computed_function.cpp b/cpp/perspective/src/cpp/computed_function.cpp index 9f5afb0985..af6ea63809 100644 --- a/cpp/perspective/src/cpp/computed_function.cpp +++ b/cpp/perspective/src/cpp/computed_function.cpp @@ -681,21 +681,34 @@ t_tscalar week_bucket(t_tscalar x) { // Convert the timestamp to a `sys_time` (alias for `time_point`) date::sys_time ts(timestamp); - // Create a copy of the timestamp with day precision - auto days = date::floor(ts); + // Convert the timestamp to local time + std::time_t temp = std::chrono::system_clock::to_time_t(ts); + std::tm* t = std::localtime(&temp); + + // Take the ymd from the `tm`, now in local time, and create a + // date::year_month_day. + date::year year {1900 + t->tm_year}; + + // date::month is [1-12], whereas `std::tm::tm_mon` is [0-11] + date::month month {static_cast(t->tm_mon) + 1}; + date::day day {static_cast(t->tm_mday)}; + date::year_month_day ymd(year, month, day); + + // Convert to a `sys_days` representing no. of days since epoch + date::sys_days days_since_epoch = ymd; // Subtract Sunday from the ymd to get the beginning of the last day - date::year_month_day ymd = days - (date::weekday{days} - date::Monday); + ymd = days_since_epoch - (date::weekday{days_since_epoch} - date::Monday); // Get the day of month and day of the week - std::int32_t year = static_cast(ymd.year()); + std::int32_t year_int = static_cast(ymd.year()); // date::month is [1-12], whereas `t_date.month()` is [0-11] - std::uint32_t month = static_cast(ymd.month()) - 1; - std::uint32_t day = static_cast(ymd.day()); + std::uint32_t month_int = static_cast(ymd.month()) - 1; + std::uint32_t day_int = static_cast(ymd.day()); // Return the new `t_date` - t_date new_date = t_date(year, month, day); + t_date new_date = t_date(year_int, month_int, day_int); rval.set(new_date); return rval; } diff --git a/python/perspective/perspective/tests/table/test_table_datetime.py b/python/perspective/perspective/tests/table/test_table_datetime.py index 0c36e889ca..1fdda7c036 100644 --- a/python/perspective/perspective/tests/table/test_table_datetime.py +++ b/python/perspective/perspective/tests/table/test_table_datetime.py @@ -1390,7 +1390,6 @@ def test_table_day_bucket_edge_in_PST(object): result = view.to_dict() assert result["computed"] == [datetime(2020, 1, 31)] - @mark.skip def test_table_week_bucket_edge_in_EST(object): """Make sure edge cases are fixed for week_bucket - if a local time converted to UTC is in the next day, the week_bucket @@ -1410,7 +1409,6 @@ def test_table_week_bucket_edge_in_EST(object): result = view.to_dict() assert result["computed"] == [datetime(2020, 1, 27)] - @mark.skip def test_table_week_bucket_edge_in_CST(object): os.environ["TZ"] = "US/Central" time.tzset() @@ -1430,7 +1428,6 @@ def test_table_week_bucket_edge_in_CST(object): result = view.to_dict() assert result["computed"] == [datetime(2020, 1, 27)] - @mark.skip def test_table_week_bucket_edge_in_PST(object): os.environ["TZ"] = "US/Pacific" time.tzset() @@ -1571,7 +1568,7 @@ def test_table_year_bucket_edge_in_EST(object): "column": "computed", "computed_function_name": "year_bucket" }] - + data = { "a": [datetime(2019, 12, 31, 23, 59)] } From dc61b8890eeae3db53fe50e7eb377a5611b9313f Mon Sep 17 00:00:00 2001 From: Jun Tan Date: Wed, 8 Jul 2020 14:53:45 -0400 Subject: [PATCH 05/11] Add yarn --network-timeout on catalina azure job --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f69f7a4ba4..7a7412acbd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -369,7 +369,7 @@ jobs: - script: npm install -g yarn displayName: "Install Yarn" - - script: yarn + - script: yarn --network-timeout 600000 displayName: 'Install Deps' - script: yarn build_python --ci $(python_flag) From ffd467602aed8e85658cbd53322aa40a76fe6452 Mon Sep 17 00:00:00 2001 From: Jun Tan Date: Thu, 9 Jul 2020 16:21:12 -0400 Subject: [PATCH 06/11] Fix computed columns not displaying axis correctly in highcharts --- examples/simple/superstore.html | 6 ++-- .../src/js/highcharts/draw.js | 29 ++++++++++++++----- .../test/js/axis_tests.js | 13 +++++++++ 3 files changed, 37 insertions(+), 11 deletions(-) diff --git a/examples/simple/superstore.html b/examples/simple/superstore.html index c2d56b2e90..688a329940 100644 --- a/examples/simple/superstore.html +++ b/examples/simple/superstore.html @@ -17,6 +17,7 @@ + @@ -29,9 +30,8 @@ - + diff --git a/packages/perspective-viewer-highcharts/src/js/highcharts/draw.js b/packages/perspective-viewer-highcharts/src/js/highcharts/draw.js index 2eaa4e2edc..40ed4bc3bc 100644 --- a/packages/perspective-viewer-highcharts/src/js/highcharts/draw.js +++ b/packages/perspective-viewer-highcharts/src/js/highcharts/draw.js @@ -53,22 +53,35 @@ export const draw = (mode, set_config, restyle) => const columns = config.columns; const real_columns = JSON.parse(this.getAttribute("columns")); - const [schema, tschema] = await Promise.all([view.schema(false), this._table.schema(false)]); + const [view_schema, computed_schema, tschema] = await Promise.all([view.schema(false), view.computed_schema(false), this._table.schema(false)]); let element; if (task.cancelled) { return; } + /** + * Retrieve a tree axis column from the table and computed schemas, + * returning a String type or `undefined`. + * @param {*} column a column name + */ + const get_tree_type = function(column) { + let type = tschema[column]; + if (!type) { + type = computed_schema[column]; + } + return type; + }; + let configs = [], xaxis_name = columns.length > 0 ? columns[0] : undefined, - xaxis_type = schema[xaxis_name], + xaxis_type = view_schema[xaxis_name], yaxis_name = columns.length > 1 ? columns[1] : undefined, - yaxis_type = schema[yaxis_name], + yaxis_type = view_schema[yaxis_name], xtree_name = row_pivots.length > 0 ? row_pivots[row_pivots.length - 1] : undefined, - xtree_type = tschema[xtree_name], + xtree_type = get_tree_type(xtree_name), ytree_name = col_pivots.length > 0 ? col_pivots[col_pivots.length - 1] : undefined, - ytree_type = tschema[ytree_name], + ytree_type = get_tree_type(ytree_name), num_aggregates = columns.length; try { @@ -80,7 +93,7 @@ export const draw = (mode, set_config, restyle) => cols = await view.to_columns(); } const config = (configs[0] = default_config.call(this, real_columns, mode)); - const [series, xtop, colorRange, ytop] = make_xy_column_data(cols, schema, real_columns, row_pivots, col_pivots); + const [series, xtop, colorRange, ytop] = make_xy_column_data(cols, view_schema, real_columns, row_pivots, col_pivots); config.legend.floating = series.length <= 20; config.legend.enabled = col_pivots.length > 0; @@ -158,7 +171,7 @@ export const draw = (mode, set_config, restyle) => } else { cols = await view.to_columns(); } - s = await make_xy_column_data(cols, schema, columns, row_pivots, col_pivots); + s = await make_xy_column_data(cols, view_schema, columns, row_pivots, col_pivots); } else { let js; if (end_col || end_row) { @@ -166,7 +179,7 @@ export const draw = (mode, set_config, restyle) => } else { js = await view.to_json(); } - s = await make_xy_data(js, schema, columns, row_pivots, col_pivots); + s = await make_xy_data(js, view_schema, columns, row_pivots, col_pivots); } const series = s[0]; diff --git a/packages/perspective-viewer-highcharts/test/js/axis_tests.js b/packages/perspective-viewer-highcharts/test/js/axis_tests.js index e456f58d4e..83bd565195 100644 --- a/packages/perspective-viewer-highcharts/test/js/axis_tests.js +++ b/packages/perspective-viewer-highcharts/test/js/axis_tests.js @@ -26,5 +26,18 @@ exports.default = function() { await page.evaluate(element => element.setAttribute("columns", '["State","Sales"]'), viewer); await page.waitForSelector("perspective-viewer:not([updating])"); }); + + test.capture("Sets a category axis when pivoted by a computed datetime", async page => { + const viewer = await page.$("perspective-viewer"); + await page.shadow_click("perspective-viewer", "#config_button"); + await page.evaluate(element => element.setAttribute("computed-columns", JSON.stringify(["hour_bucket('Ship Date')"])), viewer); + await page.waitForSelector("perspective-viewer:not([updating])"); + await page.evaluate(element => element.setAttribute("row-pivots", '["hour_bucket(Ship Date)"]'), viewer); + await page.waitForSelector("perspective-viewer:not([updating])"); + await page.evaluate(element => element.setAttribute("columns", '["State","Sales"]'), viewer); + await page.waitForSelector("perspective-viewer:not([updating])"); + await page.evaluate(element => element.setAttribute("aggregates", '{"State":"dominant","Sales":"sum"}'), viewer); + await page.waitForSelector("perspective-viewer:not([updating])"); + }); }); }; From 448e0e5a5a221f86a11e0519d3b060fcfa8ee95f Mon Sep 17 00:00:00 2001 From: Jun Tan Date: Thu, 9 Jul 2020 17:32:30 -0400 Subject: [PATCH 07/11] Fix computed columns not displaying axis in d3fc --- examples/simple/superstore.html | 6 +++--- .../src/js/charts/treemap.js | 2 ++ .../src/js/plugin/plugin.js | 21 +++++++++++++++---- .../test/js/integration/line.spec.js | 13 ++++++++++++ .../src/js/highcharts/draw.js | 2 +- 5 files changed, 36 insertions(+), 8 deletions(-) diff --git a/examples/simple/superstore.html b/examples/simple/superstore.html index 688a329940..c2d56b2e90 100644 --- a/examples/simple/superstore.html +++ b/examples/simple/superstore.html @@ -17,7 +17,6 @@ - @@ -30,8 +29,9 @@ - + diff --git a/packages/perspective-viewer-d3fc/src/js/charts/treemap.js b/packages/perspective-viewer-d3fc/src/js/charts/treemap.js index 97e4a376a5..a17eca9686 100644 --- a/packages/perspective-viewer-d3fc/src/js/charts/treemap.js +++ b/packages/perspective-viewer-d3fc/src/js/charts/treemap.js @@ -48,6 +48,8 @@ function treemap(container, settings) { .each(function({split, data}) { const treemapSvg = d3.select(this); + console.log(split, data); + if (!settings.treemaps[split]) settings.treemaps[split] = {}; treemapSeries() diff --git a/packages/perspective-viewer-d3fc/src/js/plugin/plugin.js b/packages/perspective-viewer-d3fc/src/js/plugin/plugin.js index 0369ed0520..c7f8aeab85 100644 --- a/packages/perspective-viewer-d3fc/src/js/plugin/plugin.js +++ b/packages/perspective-viewer-d3fc/src/js/plugin/plugin.js @@ -54,12 +54,25 @@ function drawChart(chart) { jsonp = view.to_json({leaves_only: true}); } - let [tschema, schema, json, config] = await Promise.all([this._table.schema(false), view.schema(false), jsonp, view.get_config()]); + let [table_schema, computed_schema, view_schema, json, config] = await Promise.all([this._table.schema(false), view.computed_schema(false), view.schema(false), jsonp, view.get_config()]); if (task.cancelled) { return; } + /** + * Retrieve a tree axis column from the table and computed schemas, + * returning a String type or `undefined`. + * @param {String} column a column name + */ + const get_pivot_column_type = function(column) { + let type = table_schema[column]; + if (!type) { + type = computed_schema[column]; + } + return type; + }; + const {columns, row_pivots, column_pivots, filter} = config; const filtered = row_pivots.length > 0 ? json.filter(col => col.__ROW_PATH__ && col.__ROW_PATH__.length == row_pivots.length) : json; const dataMap = (col, i) => (!row_pivots.length ? {...col, __ROW_PATH__: [i]} : col); @@ -67,9 +80,9 @@ function drawChart(chart) { let settings = { realValues, - crossValues: row_pivots.map(r => ({name: r, type: tschema[r]})), - mainValues: columns.map(a => ({name: a, type: schema[a]})), - splitValues: column_pivots.map(r => ({name: r, type: tschema[r]})), + crossValues: row_pivots.map(r => ({name: r, type: get_pivot_column_type(r)})), + mainValues: columns.map(a => ({name: a, type: view_schema[a]})), + splitValues: column_pivots.map(r => ({name: r, type: get_pivot_column_type(r)})), filter, data: mapped }; diff --git a/packages/perspective-viewer-d3fc/test/js/integration/line.spec.js b/packages/perspective-viewer-d3fc/test/js/integration/line.spec.js index e462e019f0..bfeaa88402 100644 --- a/packages/perspective-viewer-d3fc/test/js/integration/line.spec.js +++ b/packages/perspective-viewer-d3fc/test/js/integration/line.spec.js @@ -20,6 +20,19 @@ utils.with_server({}, () => { "line.html", () => { simple_tests.default(); + + test.capture("Sets a category axis when pivoted by a computed datetime", async page => { + const viewer = await page.$("perspective-viewer"); + await page.shadow_click("perspective-viewer", "#config_button"); + await page.evaluate(element => element.setAttribute("computed-columns", JSON.stringify(["hour_bucket('Ship Date')"])), viewer); + await page.waitForSelector("perspective-viewer:not([updating])"); + await page.evaluate(element => element.setAttribute("row-pivots", '["hour_bucket(Ship Date)"]'), viewer); + await page.waitForSelector("perspective-viewer:not([updating])"); + await page.evaluate(element => element.setAttribute("columns", '["State","Sales"]'), viewer); + await page.waitForSelector("perspective-viewer:not([updating])"); + await page.evaluate(element => element.setAttribute("aggregates", '{"State":"dominant","Sales":"sum"}'), viewer); + await page.waitForSelector("perspective-viewer:not([updating])"); + }); }, {reload_page: false, root: path.join(__dirname, "..", "..", "..")} ); diff --git a/packages/perspective-viewer-highcharts/src/js/highcharts/draw.js b/packages/perspective-viewer-highcharts/src/js/highcharts/draw.js index 40ed4bc3bc..07c7b43220 100644 --- a/packages/perspective-viewer-highcharts/src/js/highcharts/draw.js +++ b/packages/perspective-viewer-highcharts/src/js/highcharts/draw.js @@ -63,7 +63,7 @@ export const draw = (mode, set_config, restyle) => /** * Retrieve a tree axis column from the table and computed schemas, * returning a String type or `undefined`. - * @param {*} column a column name + * @param {String} column a column name */ const get_tree_type = function(column) { let type = tschema[column]; From 3489c6ff9be69ae3ce887706b5837d5bfc0439ef Mon Sep 17 00:00:00 2001 From: Jun Tan Date: Thu, 9 Jul 2020 18:04:12 -0400 Subject: [PATCH 08/11] differentiate properly between table.computed_schema and view.computed_schema in wire api --- .../perspective/manager/manager_internal.py | 63 +++++---- .../perspective/table/_data_formatter.py | 1 - .../perspective/tests/manager/test_manager.py | 120 ++++++++++++++++++ 3 files changed, 160 insertions(+), 24 deletions(-) diff --git a/python/perspective/perspective/manager/manager_internal.py b/python/perspective/perspective/manager/manager_internal.py index 3c74635413..3ca8c845f9 100644 --- a/python/perspective/perspective/manager/manager_internal.py +++ b/python/perspective/perspective/manager/manager_internal.py @@ -119,39 +119,56 @@ def _process_method_call(self, msg, post_callback, client_id): self._process_subscribe( msg, table_or_view, post_callback, client_id) else: - args = {} + # Decide how to dispatch the method + arguments = {} + if msg["method"] in ("schema", "computed_schema", "get_computation_input_types"): - # make sure schema returns string types - args["as_string"] = True + # make sure schema returns string types through the + # wire API. `as_string` is respected by both the table + # and view's `schema` and `computed_schema` methods. + arguments["as_string"] = True elif msg["method"].startswith("to_"): # parse options in `to_format` calls for d in msg.get("args", []): - args.update(d) + arguments.update(d) else: - args = msg.get("args", []) - - if msg["method"] == "delete" and msg["cmd"] == "view_method": - # views can be removed, but tables cannot - self._views[msg["name"]].delete() - self._views.pop(msg["name"], None) - return - + # Otherwise, arguments are always passed as arrays of + # individual arguments. + arguments = msg.get("args", []) + + if msg["method"] == "delete": + if msg["cmd"] == "view_method": + # views can be removed, but tables cannot - intercept + # calls to `delete` on the view and return. + self._views[msg["name"]].delete() + self._views.pop(msg["name"], None) + return + else: + # Return an error when `table.delete()` is called + # over the wire API. + raise PerspectiveError("table.delete() cannot be called on a remote table, as the remote has full ownership.") + + # Dispatch the method using the expected argument form if msg["method"].startswith("to_"): # to_format takes dictionary of options - result = getattr(table_or_view, msg["method"])(**args) + result = getattr(table_or_view, msg["method"])(**arguments) elif msg["method"] in ("update", "remove"): - # Apply first arg as position, then options dict as kwargs - data = args[0] + # Apply first arg as positional, then options dict as kwargs + data = arguments[0] options = {} - if (len(args) > 1 and isinstance(args[1], dict)): - options = args[1] + if (len(arguments) > 1 and isinstance(arguments[1], dict)): + options = arguments[1] result = getattr(table_or_view, msg["method"])(data, **options) - elif msg["method"] in ("computed_schema", "get_computation_input_types"): - # these methods take args and kwargs - result = getattr(table_or_view, msg["method"])(*msg.get("args", []), **args) - elif msg["method"] != "delete": - # otherwise parse args as list - result = getattr(table_or_view, msg["method"])(*args) + elif msg["cmd"] == "table_method" and msg["method"] in ("computed_schema", "get_computation_input_types"): + # computed_schema on the table takes kwargs; computed + # schema on the view takes args. + result = getattr(table_or_view, msg["method"])(*msg.get("args", []), **arguments) + else: + # otherwise parse arguments as list + result = getattr(table_or_view, msg["method"])(*arguments) + + # result has been returned from Perspective, now deliver + # it back to the user. if isinstance(result, bytes) and msg["method"] != "to_csv": # return a binary to the client without JSON serialization, # i.e. when we return an Arrow. If a method is added that diff --git a/python/perspective/perspective/table/_data_formatter.py b/python/perspective/perspective/table/_data_formatter.py index 0c1a61ebf9..4404cbd5df 100644 --- a/python/perspective/perspective/table/_data_formatter.py +++ b/python/perspective/perspective/table/_data_formatter.py @@ -104,7 +104,6 @@ def to_format(options, view, output_format): if output_format == 'numpy': for k, v in data.items(): # TODO push into C++ - print(type(v), v) data[k] = np.array(v) return data diff --git a/python/perspective/perspective/tests/manager/test_manager.py b/python/perspective/perspective/tests/manager/test_manager.py index c2773fbbe2..7a1bef205c 100644 --- a/python/perspective/perspective/tests/manager/test_manager.py +++ b/python/perspective/perspective/tests/manager/test_manager.py @@ -315,6 +315,114 @@ def test_arbitary_lock_unlock_manager(self): "error": "`table_method.update` failed - access denied" })) + # schema + + def test_manager_table_schema(self): + post_callback = partial(self.validate_post, expected={ + "id": 1, + "data": { + "a": "integer", + "b": "string" + } + }) + + message = {"id": 1, "name": "table1", "cmd": "table_method", "method": "schema", "args": [False]} + manager = PerspectiveManager() + table = Table(data) + view = table.view() + manager.host_table("table1", table) + manager.host_view("view1", view) + manager._process(message, post_callback) + + def test_manager_view_schema(self): + post_callback = partial(self.validate_post, expected={ + "id": 1, + "data": { + "a": "integer", + "b": "integer" + } + }) + + message = {"id": 1, "name": "view1", "cmd": "view_method", "method": "schema", "args": [False]} + manager = PerspectiveManager() + table = Table(data) + view = table.view(row_pivots=["a"]) + manager.host_table("table1", table) + manager.host_view("view1", view) + manager._process(message, post_callback) + + def test_manager_table_computed_schema(self): + post_callback = partial(self.validate_post, expected={ + "id": 1, + "data": { + "abc": "float" + } + }) + + message = { + "id": 1, + "name": "table1", + "cmd": "table_method", + "method": "computed_schema", + "args": [ + [ + { + "column": "abc", + "computed_function_name": "+", + "inputs": ["a", "a"] + } + ] + ] + } + manager = PerspectiveManager() + table = Table(data) + view = table.view() + manager.host_table("table1", table) + manager.host_view("view1", view) + manager._process(message, post_callback) + + def test_manager_table_get_computation_input_types(self): + post_callback = partial(self.validate_post, expected={ + "id": 1, + "data": ["string"] + }) + + message = { + "id": 1, + "name": "table1", + "cmd": "table_method", + "method": "get_computation_input_types", + "args": ["concat_comma"] + } + manager = PerspectiveManager() + table = Table(data) + view = table.view() + manager.host_table("table1", table) + manager.host_view("view1", view) + manager._process(message, post_callback) + + def test_manager_view_computed_schema(self): + post_callback = partial(self.validate_post, expected={ + "id": 1, + "data": { + "abc": "float" + } + }) + + message = {"id": 1, "name": "view1", "cmd": "view_method", "method": "computed_schema", "args": [False]} + manager = PerspectiveManager() + table = Table(data) + view = table.view(computed_columns=[ + { + "column": "abc", + "computed_function_name": "+", + "inputs": ["a", "a"] + } + ]) + manager.host_table("table1", table) + manager.host_view("view1", view) + manager._process(message, post_callback) + # serialization def test_manager_to_dict(self, sentinel): @@ -813,6 +921,18 @@ def post_update(msg): manager._process(update2, self.post) assert s.get() == 100 + def test_manager_delete_table_should_fail(self): + post_callback = partial(self.validate_post, expected={ + "id": 2, + "error": "table.delete() cannot be called on a remote table, as the remote has full ownership." + }) + make_table = {"id": 1, "name": "table1", "cmd": "table", "args": [data]} + manager = PerspectiveManager() + manager._process(make_table, self.post) + delete_table = {"id": 2, "name": "table1", "cmd": "table_method", "method": "delete", "args": []} + manager._process(delete_table, post_callback) + assert len(manager._tables) == 1 + def test_manager_delete_view(self): make_table = {"id": 1, "name": "table1", "cmd": "table", "args": [data]} manager = PerspectiveManager() From 66c751c99ac69898969049e164e63bb368e877dd Mon Sep 17 00:00:00 2001 From: Jun Tan Date: Tue, 14 Jul 2020 13:05:14 -0400 Subject: [PATCH 09/11] Fix tests for highcharts --- .../perspective-viewer-highcharts/test/js/axis_tests.js | 6 ++---- .../test/results/linux.docker.json | 6 ++++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/perspective-viewer-highcharts/test/js/axis_tests.js b/packages/perspective-viewer-highcharts/test/js/axis_tests.js index 83bd565195..c10e52fc28 100644 --- a/packages/perspective-viewer-highcharts/test/js/axis_tests.js +++ b/packages/perspective-viewer-highcharts/test/js/axis_tests.js @@ -27,14 +27,12 @@ exports.default = function() { await page.waitForSelector("perspective-viewer:not([updating])"); }); - test.capture("Sets a category axis when pivoted by a computed datetime", async page => { + test.capture("Sets a category axis when category is a computed datetime", async page => { const viewer = await page.$("perspective-viewer"); await page.shadow_click("perspective-viewer", "#config_button"); await page.evaluate(element => element.setAttribute("computed-columns", JSON.stringify(["hour_bucket('Ship Date')"])), viewer); await page.waitForSelector("perspective-viewer:not([updating])"); - await page.evaluate(element => element.setAttribute("row-pivots", '["hour_bucket(Ship Date)"]'), viewer); - await page.waitForSelector("perspective-viewer:not([updating])"); - await page.evaluate(element => element.setAttribute("columns", '["State","Sales"]'), viewer); + await page.evaluate(element => element.setAttribute("columns", '["hour_bucket(Ship Date)","Sales"]'), viewer); await page.waitForSelector("perspective-viewer:not([updating])"); await page.evaluate(element => element.setAttribute("aggregates", '{"State":"dominant","Sales":"sum"}'), viewer); await page.waitForSelector("perspective-viewer:not([updating])"); diff --git a/packages/perspective-viewer-highcharts/test/results/linux.docker.json b/packages/perspective-viewer-highcharts/test/results/linux.docker.json index 2125b79222..6a1b9442ab 100644 --- a/packages/perspective-viewer-highcharts/test/results/linux.docker.json +++ b/packages/perspective-viewer-highcharts/test/results/linux.docker.json @@ -25,7 +25,7 @@ "render_warning_render_warning_should_show_above_size_limit_": "54dacb0e820609b0630ebcbe0e7d08dc", "render_warning_dismissing_render_warning_should_trigger_render_": "a19460e94971e107754140056d0844fd", "render_warning_underlying_data_updates_should_not_trigger_rerender_if_warning_is_visible_": "de22dce000eb9bfafc85359c620a8365", - "__GIT_COMMIT__": "410648f3426fe2358cdf2a9a89424ed6cbe6122a", + "__GIT_COMMIT__": "3489c6ff9be69ae3ce887706b5837d5bfc0439ef", "line_shows_a_grid_without_any_settings_applied_": "daa48e26ee315f01377eaaf054088934", "line_pivots_by_a_row_": "47381fac3329abe680ee05457635ace4", "line_pivots_by_two_rows_": "f4938c42922f08a3af0741afc997d5e2", @@ -90,5 +90,7 @@ "heatmap_tooltip_tests_tooltip_shows_on_hover_": "d459cde6c4044d6d30cbddf8f6b069e7", "line_replaces_all_rows_": "fe3a3c6244530496bb997919399c2517", "line_tooltip_columns_work_with_extra_tooltip_columns": "db7238f748b0e2328971fe6c0cf16489", - "scatter_tooltip_tests_tooltip_columns_works_when_color_column_is_null": "4e89313f4b839af69b9ed78b5ccadedb" + "scatter_tooltip_tests_tooltip_columns_works_when_color_column_is_null": "4e89313f4b839af69b9ed78b5ccadedb", + "scatter_axis_tests_Sets_a_category_axis_when_category_is_a_computed_datetime": "915171a1db77a324aff824d01543686e", + "line_axis_tests_Sets_a_category_axis_when_category_is_a_computed_datetime": "b92993af73cc8106ffa57ebb6f7874d5" } \ No newline at end of file From ad4984ee0d913472a52701ba8e10c8c81a841e4c Mon Sep 17 00:00:00 2001 From: Jun Tan Date: Tue, 14 Jul 2020 13:15:30 -0400 Subject: [PATCH 10/11] Fix tests for d3fc --- .../perspective-viewer-d3fc/test/results/linux.docker.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/perspective-viewer-d3fc/test/results/linux.docker.json b/packages/perspective-viewer-d3fc/test/results/linux.docker.json index 4c92a2530e..1375808fcc 100644 --- a/packages/perspective-viewer-d3fc/test/results/linux.docker.json +++ b/packages/perspective-viewer-d3fc/test/results/linux.docker.json @@ -1,7 +1,7 @@ { "candlestick_filter_by_a_single_instrument_": "f988ca6494d7a36bada09928cd1a544e", "candlestick_filter_to_date_range_": "8ca4da0a6229d4f9db4a845d5d415c20", - "__GIT_COMMIT__": "410648f3426fe2358cdf2a9a89424ed6cbe6122a", + "__GIT_COMMIT__": "66c751c99ac69898969049e164e63bb368e877dd", "ohlc_filter_by_a_single_instrument_": "0110fac1f2befac1b97a9d33f0022acf", "ohlc_filter_to_date_range_": "3ec466996be47e2c8df135a4303bf383", "scatter_shows_a_grid_without_any_settings_applied_": "8677946ab48f16a376c421500d59e6c0", @@ -125,5 +125,6 @@ "treemap_sorts_by_an_alpha_column_": "046d040908811f04c15f7646d9d70733", "treemap_displays_visible_columns_": "62996aa87b1237b0be568a339a700bdf", "treemap_with_column_position_1_set_to_null_": "a2f594ff864fb673cde708e4533d8edc", - "treemap_tooltip_columns_works": "9b3a44cc1b4b1d11cb8b635c850a0612" + "treemap_tooltip_columns_works": "9b3a44cc1b4b1d11cb8b635c850a0612", + "line_Sets_a_category_axis_when_pivoted_by_a_computed_datetime": "eb1c86dc44988ad9a65fdd5a335850b8" } \ No newline at end of file From 0693630277c4f4cce00912789a9fee5b65d1de40 Mon Sep 17 00:00:00 2001 From: Jun Tan Date: Thu, 16 Jul 2020 21:32:42 -0400 Subject: [PATCH 11/11] Fix last_by_index aggregate when live updates are made --- cpp/perspective/src/cpp/aggspec.cpp | 7 +- cpp/perspective/src/cpp/base.cpp | 2 +- cpp/perspective/src/cpp/config.cpp | 2 +- cpp/perspective/src/cpp/extract_aggregate.cpp | 2 +- cpp/perspective/src/cpp/sparse_tree.cpp | 2 +- cpp/perspective/src/cpp/view_config.cpp | 4 +- .../src/include/perspective/base.h | 2 +- packages/perspective/test/js/pivots.js | 92 +++++++++++++++++++ 8 files changed, 102 insertions(+), 11 deletions(-) diff --git a/cpp/perspective/src/cpp/aggspec.cpp b/cpp/perspective/src/cpp/aggspec.cpp index ee4f406f57..631144024d 100644 --- a/cpp/perspective/src/cpp/aggspec.cpp +++ b/cpp/perspective/src/cpp/aggspec.cpp @@ -140,8 +140,8 @@ t_aggspec::agg_str() const { case AGGTYPE_FIRST: { return "first"; } break; - case AGGTYPE_LAST: { - return "last"; + case AGGTYPE_LAST_BY_INDEX: { + return "last_by_index"; } break; case AGGTYPE_PY_AGG: { return "py_agg"; @@ -297,7 +297,7 @@ t_aggspec::get_output_specs(const t_schema& schema) const { case AGGTYPE_DOMINANT: case AGGTYPE_MEDIAN: case AGGTYPE_FIRST: - case AGGTYPE_LAST: + case AGGTYPE_LAST_BY_INDEX: case AGGTYPE_OR: case AGGTYPE_LAST_VALUE: case AGGTYPE_HIGH_WATER_MARK: @@ -318,7 +318,6 @@ t_aggspec::get_output_specs(const t_schema& schema) const { return mk_col_name_type_vec(name(), DTYPE_F64PAIR); } case AGGTYPE_WEIGHTED_MEAN: { - return mk_col_name_type_vec(name(), DTYPE_F64PAIR); } case AGGTYPE_JOIN: { diff --git a/cpp/perspective/src/cpp/base.cpp b/cpp/perspective/src/cpp/base.cpp index 91040c5d04..7f6852280e 100644 --- a/cpp/perspective/src/cpp/base.cpp +++ b/cpp/perspective/src/cpp/base.cpp @@ -577,7 +577,7 @@ str_to_aggtype(const std::string& str) { } else if (str == "first by index" || str == "first") { return t_aggtype::AGGTYPE_FIRST; } else if (str == "last by index") { - return t_aggtype::AGGTYPE_LAST; + return t_aggtype::AGGTYPE_LAST_BY_INDEX; } else if (str == "py_agg") { return t_aggtype::AGGTYPE_PY_AGG; } else if (str == "and") { diff --git a/cpp/perspective/src/cpp/config.cpp b/cpp/perspective/src/cpp/config.cpp index 14f97c5a65..12bc8455c2 100644 --- a/cpp/perspective/src/cpp/config.cpp +++ b/cpp/perspective/src/cpp/config.cpp @@ -183,7 +183,7 @@ t_config::setup(const std::vector& detail_columns, case AGGTYPE_OR: case AGGTYPE_ANY: case AGGTYPE_FIRST: - case AGGTYPE_LAST: + case AGGTYPE_LAST_BY_INDEX: case AGGTYPE_MEAN: case AGGTYPE_WEIGHTED_MEAN: case AGGTYPE_UNIQUE: diff --git a/cpp/perspective/src/cpp/extract_aggregate.cpp b/cpp/perspective/src/cpp/extract_aggregate.cpp index 73b36a4cde..4ed05398f5 100644 --- a/cpp/perspective/src/cpp/extract_aggregate.cpp +++ b/cpp/perspective/src/cpp/extract_aggregate.cpp @@ -54,9 +54,9 @@ extract_aggregate( case AGGTYPE_DOMINANT: case AGGTYPE_MEDIAN: case AGGTYPE_FIRST: - case AGGTYPE_LAST: case AGGTYPE_AND: case AGGTYPE_OR: + case AGGTYPE_LAST_BY_INDEX: case AGGTYPE_LAST_VALUE: case AGGTYPE_HIGH_WATER_MARK: case AGGTYPE_LOW_WATER_MARK: diff --git a/cpp/perspective/src/cpp/sparse_tree.cpp b/cpp/perspective/src/cpp/sparse_tree.cpp index a8b4da38fb..929777616e 100644 --- a/cpp/perspective/src/cpp/sparse_tree.cpp +++ b/cpp/perspective/src/cpp/sparse_tree.cpp @@ -1083,7 +1083,7 @@ t_stree::update_agg_table(t_uindex nidx, t_agg_update_info& info, t_uindex src_r dst->set_scalar(dst_ridx, new_value); } break; case AGGTYPE_FIRST: - case AGGTYPE_LAST: { + case AGGTYPE_LAST_BY_INDEX: { old_value.set(dst->get_scalar(dst_ridx)); new_value.set(first_last_helper(nidx, spec, gstate)); dst->set_scalar(dst_ridx, new_value); diff --git a/cpp/perspective/src/cpp/view_config.cpp b/cpp/perspective/src/cpp/view_config.cpp index aa9320cec0..f9750146f1 100644 --- a/cpp/perspective/src/cpp/view_config.cpp +++ b/cpp/perspective/src/cpp/view_config.cpp @@ -185,8 +185,8 @@ t_view_config::fill_aggspecs(std::shared_ptr schema) { } } - if (agg_type == AGGTYPE_FIRST || agg_type == AGGTYPE_LAST) { - dependencies.push_back(t_dep("psp_pkey", DEPTYPE_COLUMN)); + if (agg_type == AGGTYPE_FIRST || agg_type == AGGTYPE_LAST_BY_INDEX) { + dependencies.push_back(t_dep("psp_okey", DEPTYPE_COLUMN)); m_aggspecs.push_back( t_aggspec(column, column, agg_type, dependencies, SORTTYPE_ASCENDING)); } else { diff --git a/cpp/perspective/src/include/perspective/base.h b/cpp/perspective/src/include/perspective/base.h index 87ec7e17df..4e8ffc20ca 100644 --- a/cpp/perspective/src/include/perspective/base.h +++ b/cpp/perspective/src/include/perspective/base.h @@ -296,7 +296,7 @@ enum t_aggtype { AGGTYPE_SCALED_MUL, AGGTYPE_DOMINANT, AGGTYPE_FIRST, - AGGTYPE_LAST, + AGGTYPE_LAST_BY_INDEX, AGGTYPE_PY_AGG, AGGTYPE_AND, AGGTYPE_OR, diff --git a/packages/perspective/test/js/pivots.js b/packages/perspective/test/js/pivots.js index 73f692747a..50689d6ad3 100644 --- a/packages/perspective/test/js/pivots.js +++ b/packages/perspective/test/js/pivots.js @@ -131,6 +131,52 @@ module.exports = perspective => { table.delete(); }); + it("['z'], first by index with appends", async function() { + var table = perspective.table(data, {index: "y"}); + var view = table.view({ + row_pivots: ["z"], + columns: ["x"], + aggregates: {x: "first by index"} + }); + const answer = [ + {__ROW_PATH__: [], x: 1}, + {__ROW_PATH__: [false], x: 2}, + {__ROW_PATH__: [true], x: 1} + ]; + table.update({ + x: [5], + y: ["e"], + z: [true] + }); + const result = await view.to_json(); + expect(result).toEqual(answer); + view.delete(); + table.delete(); + }); + + it("['z'], first by index with partial updates", async function() { + var table = perspective.table(data, {index: "y"}); + var view = table.view({ + row_pivots: ["z"], + columns: ["x"], + aggregates: {x: "first by index"} + }); + const answer = [ + {__ROW_PATH__: [], x: 5}, + {__ROW_PATH__: [false], x: 2}, + {__ROW_PATH__: [true], x: 5} + ]; + table.update({ + x: [5], + y: ["a"], + z: [true] + }); + const result = await view.to_json(); + expect(result).toEqual(answer); + view.delete(); + table.delete(); + }); + it("['z'], last by index", async function() { var table = perspective.table(data); var view = table.view({ @@ -149,6 +195,52 @@ module.exports = perspective => { table.delete(); }); + it("['z'], last by index with appends", async function() { + const table = perspective.table(data); + const view = table.view({ + row_pivots: ["z"], + columns: ["x"], + aggregates: {x: "last by index"} + }); + const answer = [ + {__ROW_PATH__: [], x: 5}, + {__ROW_PATH__: [false], x: 4}, + {__ROW_PATH__: [true], x: 5} + ]; + table.update({ + x: [5], + y: ["e"], + z: [true] + }); + const result = await view.to_json(); + expect(result).toEqual(answer); + view.delete(); + table.delete(); + }); + + it("['z'], last by index with partial updates", async function() { + const table = perspective.table(data, {index: "y"}); + const view = table.view({ + row_pivots: ["z"], + columns: ["x"], + aggregates: {x: "last by index"} + }); + const answer = [ + {__ROW_PATH__: [], x: 4}, + {__ROW_PATH__: [false], x: 4}, + {__ROW_PATH__: [true], x: 5} + ]; + table.update({ + x: [5], + y: ["c"], + z: [true] + }); + const result = await view.to_json(); + expect(result).toEqual(answer); + view.delete(); + table.delete(); + }); + it("['z'], last", async function() { var table = perspective.table(data); var view = table.view({