From 55b9d1f7d08a16cc1422eef13f20b55c3624d019 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Mon, 1 Jan 2018 15:50:40 -0500 Subject: [PATCH 01/29] Fixed highcharts bug which caused empty strings in column pivots to be misnamed. --- packages/perspective-viewer-highcharts/src/js/highcharts.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/perspective-viewer-highcharts/src/js/highcharts.js b/packages/perspective-viewer-highcharts/src/js/highcharts.js index a1307038b4..766c9d7cda 100644 --- a/packages/perspective-viewer-highcharts/src/js/highcharts.js +++ b/packages/perspective-viewer-highcharts/src/js/highcharts.js @@ -133,7 +133,7 @@ function _make_series(js, pivots, col_pivots, mode, hidden) { for (let prop of columns) { var sname = prop.split(','); var gname = sname[sname.length - 1]; - sname = sname.slice(0, sname.length - 1).join(", ") || gname; + sname = sname.slice(0, sname.length - 1).join(", ") || " "; var s; if (prev === undefined) prev = sname; for (var sidx = 0; sidx < series.length; sidx++) { @@ -170,7 +170,7 @@ function _make_series(js, pivots, col_pivots, mode, hidden) { if (is_stacked) { sname = sname.join(", ") || gname; } else { - sname = sname.slice(0, sname.length - 1).join(", ") || gname; + sname = sname.slice(0, sname.length - 1).join(", ") || " "; } var s; for (var sidx=0; sidx Date: Mon, 1 Jan 2018 16:02:41 -0500 Subject: [PATCH 02/29] Refactored json output to make fewer embind calls. --- packages/perspective/src/cpp/main.cpp | 27 ++++++++++++++++++++++ packages/perspective/src/js/perspective.js | 21 ++++++++++++----- 2 files changed, 42 insertions(+), 6 deletions(-) diff --git a/packages/perspective/src/cpp/main.cpp b/packages/perspective/src/cpp/main.cpp index 304b543b62..5175e500cf 100644 --- a/packages/perspective/src/cpp/main.cpp +++ b/packages/perspective/src/cpp/main.cpp @@ -490,6 +490,30 @@ scalar_to_val(const t_tscalvec& scalars, t_uint32 idx) } } +/** + * + * + * Params + * ------ + * + * + * Returns + * ------- + * + */ +template +val +get_data(T ctx, t_uint32 start_row, t_uint32 end_row, t_uint32 start_col, t_uint32 end_col) +{ + auto slice = ctx->get_data(start_row, end_row, start_col, end_col); + val arr = val::array(); + for (auto idx = 0; idx < slice.size(); ++idx) + { + arr.set(idx, scalar_to_val(slice, idx)); + } + return arr; +} + /** * Main */ @@ -761,4 +785,7 @@ EMSCRIPTEN_BINDINGS(perspective) function("make_context_one", &make_context_one); function("make_context_two", &make_context_two); function("scalar_to_val", &scalar_to_val); + function("get_data_zero", &get_data); + function("get_data_one", &get_data); + function("get_data_two", &get_data); } diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index dfe8ebc10a..300d49db01 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -67,6 +67,7 @@ const DATE_PARSE_CANDIDATES = [ 'MM-DD-YYYY', 'MM/DD/YYYY', 'M/D/YYYY', + 'M/D/YY', 'DD MMM YYYY' ]; @@ -439,7 +440,15 @@ view.prototype.to_json = async function(options) { let end_row = options.end_row || (viewport.height ? start_row + viewport.height : this.ctx.get_row_count()); let start_col = options.start_col || (viewport.left ? viewport.left : 0); let end_col = options.end_col || (viewport.width ? start_row + viewport.width : this.ctx.unity_get_column_count() + (this.sides() === 0 ? 0 : 1)); - let slice = this.ctx.get_data(start_row, end_row, start_col, end_col); + let slice; + if (this.sides() === 0) { + slice = __MODULE__.get_data_zero(this.ctx, start_row, end_row, start_col, end_col); + } else if (this.sides() === 1) { + slice = __MODULE__.get_data_one(this.ctx, start_row, end_row, start_col, end_col); + } else { + slice = __MODULE__.get_data_two(this.ctx, start_row, end_row, start_col, end_col); + } + let data; if (options.format && options.format === "table") { @@ -452,7 +461,7 @@ view.prototype.to_json = async function(options) { let row, prev_row; let depth = []; let ridx = -1; - for (let idx = 0; idx < slice.size(); idx++) { + for (let idx = 0; idx < slice.length; idx++) { let cidx = idx % (end_col - start_col); if (cidx === 0) { if (row) { @@ -463,12 +472,12 @@ view.prototype.to_json = async function(options) { } if (this.sides() === 0) { let col_name = col_names[start_col + cidx + 1]; - row[col_name] = __MODULE__.scalar_to_val(slice, idx); + row[col_name] = slice[idx]; } else { if (cidx === 0) { let col_name = "__ROW_PATH__"; let new_depth = this.ctx.unity_get_row_depth(ridx); - let row_name = __MODULE__.scalar_to_val(slice, idx); + let row_name = slice[idx]; if (new_depth === 0) { row[col_name] = []; } else if (new_depth > depth.length + 1) { @@ -486,11 +495,11 @@ view.prototype.to_json = async function(options) { prev_row = row_name; } else { let col_name = col_names[start_col + cidx]; - row[col_name] = __MODULE__.scalar_to_val(slice, idx); + row[col_name] = slice[idx]; } } } - slice.delete(); + if (row) data.push(row); return data; } From 7c392ceabf1d7570d9391ed72152f47114508341 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Mon, 1 Jan 2018 16:03:37 -0500 Subject: [PATCH 03/29] Fused loop iterations in hypergrid plugin. --- .../src/js/hypergrid.js | 44 +++++++++---------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js index f8181b2237..63743286c5 100644 --- a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js +++ b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js @@ -738,47 +738,47 @@ function psp2hypergrid(data, schema) { } } var is_tree = data[0].hasOwnProperty('__ROW_PATH__'); - var row_paths = data.map(function(row) { - if (is_tree) { - return ["ROOT"].concat(row.__ROW_PATH__) || ["ROOT"]; - } else { - return []; - } - }); - var columnPaths = Object.keys(data[0]).filter(function(row) { - return row !== "__ROW_PATH__"; - }).map(function(row) { - return row.split(','); - }); + + var columnPaths = Object.keys(data[0]) + .filter(row => row !== "__ROW_PATH__") + .map(row => row.split(',')); + let flat_columns = columnPaths.map(col => col.join(",")); - let rows = data.map(function(row, idx) { + + let row_paths = []; + let rows = []; + let row_leaves = []; + for (const idx in data) { + const row = data[idx]; + let new_row = []; if (is_tree) { + row_paths.push(["ROOT"].concat(row.__ROW_PATH__) || ["ROOT"]); let name = row['__ROW_PATH__'][row['__ROW_PATH__'].length - 1]; if (name === undefined && idx === 0) name = "TOTAL" - var new_row = [name]; + new_row = [name]; + row_leaves.push(row.__ROW_PATH__.length >= (data[idx + 1] ? data[idx + 1].__ROW_PATH__.length : 0)); } else { - new_row = []; + row_paths.push([]); } - for (var col of flat_columns) { new_row.push(row[col]); } - return new_row; - }) + rows.push(new_row); + } + var hg_data = { rowPaths: row_paths, data: rows, isTree: is_tree, configuration: {}, columnPaths: (is_tree ? [[" "]] : []).concat(columnPaths), - columnTypes: (is_tree ? ["str"] : []).concat(columnPaths.map(function(col) { return conv[schema[col[col.length - 1]]] })) + columnTypes: (is_tree ? ["str"] : []).concat(columnPaths.map(col => conv[schema[col[col.length - 1]]])) }; if (is_tree) { - hg_data['rowLeaf'] = data.map(function(row, idx) { - return row.__ROW_PATH__.length >= (data[idx + 1] ? data[idx + 1].__ROW_PATH__.length : 0); - }) + hg_data['rowLeaf'] = row_leaves; } + return hg_data } From 9f69ff58c50664f7ddf60355b98eb49a8a633b95 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Mon, 1 Jan 2018 16:04:14 -0500 Subject: [PATCH 04/29] Removed cache from travis build. --- .travis.yml | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 6d16245ae0..46621fa67e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,9 +9,6 @@ node_js: env: global: - EM_USE_GLOBAL_CACHE=1 - - CACHE_DIR=$HOME/.cache/docker - - CACHE_FILE_EMSCRIPTEN=$CACHE_DIR/emscripten.tar.gz - - CACHE_FILE_PUPPETEER=$CACHE_DIR/puppeteer.tar.gz addons: apt: @@ -23,15 +20,9 @@ sudo: required services: - docker -cache: - directories: - - $CACHE_DIR - before_install: - mkdir boost_includes - cp -r /usr/include/boost boost_includes/ - - if [ -f ${CACHE_FILE_EMSCRIPTEN} ]; then gunzip -c ${CACHE_FILE_EMSCRIPTEN} | docker load; fi - - if [ -f ${CACHE_FILE_PUPPETEER} ]; then gunzip -c ${CACHE_FILE_PUPPETEER} | docker load; fi - docker run -dit --name emscripten -v $(pwd):/src trzeci/emscripten:sdk-incoming-64bit bash install: @@ -40,7 +31,3 @@ install: script: - docker exec -it emscripten npm run start - npm run travis_test - -before_cache: - - if [ ! -f ${CACHE_FILE_EMSCRIPTEN} ]; then docker save trzeci/emscripten:sdk-incoming-64bit | gzip > ${CACHE_FILE_EMSCRIPTEN}; fi - - if [ ! -f ${CACHE_FILE_PUPPETEER} ]; then docker save zenato/puppeteer | gzip > ${CACHE_FILE_PUPPETEER}; fi From 47c507dd951b70def99f13b41131a7131e34a295 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Mon, 1 Jan 2018 21:52:56 -0500 Subject: [PATCH 05/29] Cancel pending transformations when user initiates a new state change before previous has completed; added bluebird promise polyfill. --- packages/perspective-common/common.config.js | 2 +- packages/perspective-common/index.js | 2 ++ packages/perspective-common/package.json | 3 +++ packages/perspective-viewer-highcharts/src/js/highcharts.js | 2 ++ packages/perspective-viewer/package.json | 3 +++ packages/perspective-viewer/src/js/view.js | 5 ++++- packages/perspective/package.json | 3 +++ 7 files changed, 18 insertions(+), 2 deletions(-) diff --git a/packages/perspective-common/common.config.js b/packages/perspective-common/common.config.js index bc79b492ba..d925414b22 100644 --- a/packages/perspective-common/common.config.js +++ b/packages/perspective-common/common.config.js @@ -53,7 +53,7 @@ module.exports = function() { loader: "babel-loader", options: { presets: ['env'], - plugins: ['transform-runtime', ["transform-es2015-for-of", { + plugins: ['transform-promise-to-bluebird', 'transform-async-to-bluebird', 'transform-runtime', ["transform-es2015-for-of", { "loose": true }]] } diff --git a/packages/perspective-common/index.js b/packages/perspective-common/index.js index ecd66de0a6..13b8fa6146 100644 --- a/packages/perspective-common/index.js +++ b/packages/perspective-common/index.js @@ -7,6 +7,8 @@ * */ +require('bluebird').config({cancellation: true}); + /** * Detect Internet Explorer. * diff --git a/packages/perspective-common/package.json b/packages/perspective-common/package.json index e1ab7bb32b..7e7a6955bd 100644 --- a/packages/perspective-common/package.json +++ b/packages/perspective-common/package.json @@ -16,12 +16,15 @@ "author": "", "license": "Apache", "dependencies": { + "bluebird": "^3.5.1", "babel-runtime": "^6.26.0" }, "devDependencies": { "babel-core": "^6.26.0", "babel-loader": "^7.1.2", + "babel-plugin-transform-async-to-bluebird": "^1.1.1", "babel-plugin-transform-es2015-for-of": "^6.23.0", + "babel-plugin-transform-promise-to-bluebird": "^1.1.1", "babel-plugin-transform-runtime": "^6.23.0", "babel-polyfill": "^6.26.0", "babel-preset-env": "^1.6.0", diff --git a/packages/perspective-viewer-highcharts/src/js/highcharts.js b/packages/perspective-viewer-highcharts/src/js/highcharts.js index 766c9d7cda..501acb781c 100644 --- a/packages/perspective-viewer-highcharts/src/js/highcharts.js +++ b/packages/perspective-viewer-highcharts/src/js/highcharts.js @@ -7,6 +7,8 @@ * */ +import '@jpmorganchase/perspective-common'; + import highcharts from 'highcharts'; const Highcharts = highcharts; diff --git a/packages/perspective-viewer/package.json b/packages/perspective-viewer/package.json index 0eda623e69..0ba5f94cca 100644 --- a/packages/perspective-viewer/package.json +++ b/packages/perspective-viewer/package.json @@ -34,6 +34,7 @@ "@jpmorganchase/perspective-common": "^0.0.1", "babel-polyfill": "^6.26.0", "babel-runtime": "^6.26.0", + "bluebird": "^3.5.1", "d3-array": "^1.2.1", "mobile-drag-drop": "^2.2.0", "underscore": "^1.8.3" @@ -42,7 +43,9 @@ "@jpmorganchase/perspective-common": "^0.0.1", "babel-core": "^6.26.0", "babel-loader": "^7.1.2", + "babel-plugin-transform-async-to-bluebird": "^1.1.1", "babel-plugin-transform-es2015-for-of": "^6.23.0", + "babel-plugin-transform-promise-to-bluebird": "^1.1.1", "babel-plugin-transform-runtime": "^6.23.0", "babel-preset-env": "^1.6.0", "css-loader": "^0.28.7", diff --git a/packages/perspective-viewer/src/js/view.js b/packages/perspective-viewer/src/js/view.js index 53447e85c6..dd50ab4227 100644 --- a/packages/perspective-viewer/src/js/view.js +++ b/packages/perspective-viewer/src/js/view.js @@ -277,7 +277,10 @@ function update() { const t = performance.now(); this._render_count = (this._render_count || 0) + 1; - this._plugin.create.call(this, this._datavis, this._view, hidden, true).then(() => { + if (this._task) { + this._task.cancel(); + } + this._task = this._plugin.create.call(this, this._datavis, this._view, hidden, true).then(() => { if (!this.hasAttribute('render_time')) { this.dispatchEvent(new Event('loaded', {bubbles: true})); } diff --git a/packages/perspective/package.json b/packages/perspective/package.json index 668d2f6b8f..01e857038f 100644 --- a/packages/perspective/package.json +++ b/packages/perspective/package.json @@ -40,6 +40,7 @@ "dependencies": { "@jpmorganchase/perspective-common": "^0.0.1", "babel-runtime": "^6.26.0", + "bluebird": "^3.5.1", "d3-array": "^1.2.1", "moment": "^2.19.1", "papaparse": "^4.3.6", @@ -49,7 +50,9 @@ "@jpmorganchase/perspective-common": "^0.0.1", "babel-core": "^6.26.0", "babel-loader": "^7.1.2", + "babel-plugin-transform-async-to-bluebird": "^1.1.1", "babel-plugin-transform-es2015-for-of": "^6.23.0", + "babel-plugin-transform-promise-to-bluebird": "^1.1.1", "babel-plugin-transform-runtime": "^6.23.0", "babel-polyfill": "^6.26.0", "babel-preset-env": "^1.6.0", From be6dde50ba8d37ca9baaca4bec4928d38c9d4871 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Mon, 1 Jan 2018 22:19:56 -0500 Subject: [PATCH 06/29] Fixed regression in pivoted views in hypergrid plugin. --- packages/perspective-viewer-hypergrid/src/js/hypergrid.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js index 63743286c5..465f974bf2 100644 --- a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js +++ b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js @@ -748,7 +748,7 @@ function psp2hypergrid(data, schema) { let row_paths = []; let rows = []; let row_leaves = []; - for (const idx in data) { + for (let idx = 0; idx < data.length; idx++) { const row = data[idx]; let new_row = []; if (is_tree) { From a3070ee923dd4d1ca95e847d89624d38fa0f9c9c Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Tue, 2 Jan 2018 03:09:01 -0500 Subject: [PATCH 07/29] Fixed incorrect row_path in windowed views. --- packages/perspective/src/js/perspective.js | 21 +++++---------------- packages/perspective/test/js/pivots.js | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index 300d49db01..764f6cb415 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -476,23 +476,12 @@ view.prototype.to_json = async function(options) { } else { if (cidx === 0) { let col_name = "__ROW_PATH__"; - let new_depth = this.ctx.unity_get_row_depth(ridx); - let row_name = slice[idx]; - if (new_depth === 0) { - row[col_name] = []; - } else if (new_depth > depth.length + 1) { - depth.push(prev_row); - row[col_name] = depth.concat([row_name]); - } else if (new_depth <= depth.length) { - let poptimes = (depth.length - new_depth); - for (let i = 0; i <= poptimes; i++) { - depth.pop(); - } - row[col_name] = depth.concat([row_name]); - } else { - row[col_name] = depth.concat([row_name]); + let row_path = this.ctx.unity_get_row_path(start_row + ridx); + row[col_name] = []; + for (let i = 0; i < row_path.size(); i++) { + row[col_name].unshift(__MODULE__.scalar_to_val(row_path, i)); } - prev_row = row_name; + row_path.delete(); } else { let col_name = col_names[start_col + cidx]; row[col_name] = slice[idx]; diff --git a/packages/perspective/test/js/pivots.js b/packages/perspective/test/js/pivots.js index 6c582e0b96..d3c36abcc9 100644 --- a/packages/perspective/test/js/pivots.js +++ b/packages/perspective/test/js/pivots.js @@ -136,6 +136,27 @@ module.exports = (perspective) => { expect(answer).toEqual(result2); }); + + it("['x', 'z'] windowed", async function () { + var table = perspective.table(data); + var view = table.view({ + row_pivot: ['x', 'z'] + }); + + var answer = [ + { __ROW_PATH__: [ 1, true ], x: 1, y: 1, z: 1}, + { __ROW_PATH__: [ 2 ], x: 1, y: 1, z: 1}, + { __ROW_PATH__: [ 2, false ], x: 1, y: 1, z: 1}, + { __ROW_PATH__: [ 3 ], x: 1, y: 1, z: 1}, + { __ROW_PATH__: [ 3, true ], x: 1, y: 1, z: 1}, + { __ROW_PATH__: [ 4 ], x: 1, y: 1, z: 1 }, + { __ROW_PATH__: [ 4, false ], x: 1, y: 1, z: 1} + ]; + + let result2 = await view.to_json({start_row: 2}); + expect(answer).toEqual(result2); + }); + it("['x', 'z'], pivot_depth = 1", async function () { var table = perspective.table(data); var view = table.view({ From 4e39b40bd3f80818f01fca0ee861f9e470262fe1 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Tue, 2 Jan 2018 03:09:19 -0500 Subject: [PATCH 08/29] Incremental loading for hypergrid plugin. --- packages/perspective-common/index.js | 5 +- .../src/js/hypergrid.js | 58 ++++++++++++++----- packages/perspective-viewer/src/js/view.js | 8 ++- 3 files changed, 55 insertions(+), 16 deletions(-) diff --git a/packages/perspective-common/index.js b/packages/perspective-common/index.js index 13b8fa6146..36db446112 100644 --- a/packages/perspective-common/index.js +++ b/packages/perspective-common/index.js @@ -7,7 +7,10 @@ * */ -require('bluebird').config({cancellation: true}); +require('bluebird').config({ + cancellation: true, + longStackTraces: false +}); /** * Detect Internet Explorer. diff --git a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js index 465f974bf2..8427845812 100644 --- a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js +++ b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js @@ -568,7 +568,7 @@ function PerspectiveDataModel(grid) { grid.mixIn.call(grid.behavior.dataModel, { // Override setData - setData: function (dataPayload, schema) { + setData: function (dataPayload, schema, cache_update) { this.viewData = dataPayload; this.source.setData(dataPayload, schema); }, @@ -749,10 +749,13 @@ function psp2hypergrid(data, schema) { let rows = []; let row_leaves = []; for (let idx = 0; idx < data.length; idx++) { - const row = data[idx]; + const row = data[idx] || {}; let new_row = []; if (is_tree) { - row_paths.push(["ROOT"].concat(row.__ROW_PATH__) || ["ROOT"]); + if (row.__ROW_PATH__ === undefined) { + row.__ROW_PATH__ = []; + } + row_paths.push(["ROOT"].concat(row.__ROW_PATH__)); let name = row['__ROW_PATH__'][row['__ROW_PATH__'].length - 1]; if (name === undefined && idx === 0) name = "TOTAL" new_row = [name]; @@ -865,11 +868,12 @@ registerElement(TEMPLATE, { }); +const PAGE_SIZE = 1000; -async function grid(div, view, hidden) { - let [json, schema] = await Promise.all([view.to_json(), view.schema()]); +async function fill_page(view, json, hidden, start_row, end_row) { + let next_page = await view.to_json({start_row: start_row, end_row: end_row}); if (hidden.length > 0) { - let first = json[0]; + let first = next_page[0]; let to_delete = []; for (let key in first) { let split_key = key.split(','); @@ -877,28 +881,56 @@ async function grid(div, view, hidden) { to_delete.push(key); } } - for (let row of json) { + for (let row of next_page) { for (let h of to_delete) { delete row[h]; } } } + for (let idx = 0; idx < next_page.length; idx++) { + json[start_row + idx] = next_page[idx]; + } + return json; +} +async function load_incrementally(view, schema, hidden, json, total, page) { + json = await fill_page(view, json, hidden, page * PAGE_SIZE, (page + 1) * PAGE_SIZE); + this.grid.set_data(json, schema); + this.grid.grid.canvas.resize(); + this.grid.grid.canvas.resize(); + if ((page + 1) * PAGE_SIZE < total) { + await load_incrementally.call(this, view, schema, hidden, json, total, page + 1); + } +} + +async function grid(div, view, hidden) { + let [nrows, json, schema] = await Promise.all([ + view.num_rows(), + view.to_json({end_row: 1}), + view.schema() + ]); + let visible_rows = []; if (!this.grid) { this.grid = document.createElement('perspective-hypergrid'); + visible_rows = [0, 0, 100]; } else if (this.grid.grid) { this.grid.grid.canvas.stopResizing(); + visible_rows = this.grid.grid.getVisibleRows(); + } + json.length = nrows; + if (visible_rows.length > 0) { + json = await fill_page(view, json, hidden, visible_rows[1], visible_rows[visible_rows.length - 1] + 1); } if (!(document.contains ? document.contains(this.grid) : false)) { div.innerHTML = ""; div.appendChild(this.grid); } - this.grid.set_data(json, schema); - - // TODO this resolves a bug in the TreeRenderer, the calculated tree column - // width is 0 initially. - this.grid.grid.canvas.resize(); - this.grid.grid.canvas.resize(); + if (visible_rows.length > 0) { + this.grid.set_data(json, schema); + this.grid.grid.canvas.resize(); + this.grid.grid.canvas.resize(); + } + await load_incrementally.call(this, view, schema, hidden, json, nrows, 0); } global.registerPlugin("hypergrid", { diff --git a/packages/perspective-viewer/src/js/view.js b/packages/perspective-viewer/src/js/view.js index dd50ab4227..027194e709 100644 --- a/packages/perspective-viewer/src/js/view.js +++ b/packages/perspective-viewer/src/js/view.js @@ -175,9 +175,9 @@ async function loadTable(table) { } this._column_names.innerHTML = ""; + this._table = table; let [cols, schema] = await Promise.all([table.columns(), table.schema()]); - this._table = table; if (!this.hasAttribute('columns')) { this.setAttribute('columns', JSON.stringify(cols)); @@ -238,6 +238,7 @@ function update() { let column_pivots = this._view_columns('#column_pivots perspective-row:not(.off)'); let filters = JSON.parse(this.getAttribute('filters')); let aggregates = this._get_view_aggregates(); + if (aggregates.length === 0) return; if (row_pivots.length === 0 && column_pivots.length > 0) { row_pivots = column_pivots; column_pivots = []; @@ -269,7 +270,10 @@ function update() { timeout = Math.min(10000, Math.max(0, timeout)); this._debounced = setTimeout(() => { this._debounced = undefined; - this._plugin.create.call(this, this._datavis, this._view, hidden, false); + const t = performance.now(); + this._plugin.create.call(this, this._datavis, this._view, hidden, false).then(() => { + this.setAttribute('render_time', performance.now() - t); + }); }, timeout || 0); } }); From 80b66b21b79df6acd3cb29087ed7d6902e1e7f14 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Wed, 3 Jan 2018 13:51:06 -0500 Subject: [PATCH 09/29] Fixed mixed es5/es6 module syntax. --- packages/perspective/src/js/perspective.parallel.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/perspective/src/js/perspective.parallel.js b/packages/perspective/src/js/perspective.parallel.js index 112c22a306..d6ab5de91c 100644 --- a/packages/perspective/src/js/perspective.parallel.js +++ b/packages/perspective/src/js/perspective.parallel.js @@ -229,13 +229,15 @@ worker.prototype._start_same_origin = function() { this._worker.postMessage({cmd: 'init', path: __SCRIPT_PATH__.path()}); }; +let _initialized = false; + worker.prototype._handle = function(e) { if (!this._worker.initialized.value) { - if (!module.exports._initialized) { + if (_initialized) { var event = document.createEvent("Event"); event.initEvent("perspective-ready", false, true); window.dispatchEvent(event); - module.exports._initialized = true; + _initialized = true; } for (var m in this._worker.messages) { if (this._worker.messages.hasOwnProperty(m)) { @@ -265,7 +267,7 @@ worker.prototype.terminate = function() { this._worker = undefined; }; -module.exports = { +export default { worker: function() { if (window.location.href.indexOf(__SCRIPT_PATH__.host()) === -1 && detectIE()) { return perspective; From ea753e908b60163cc1a55459946d6e688124d00d Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Tue, 2 Jan 2018 14:51:00 -0500 Subject: [PATCH 10/29] Lazy hypergrid rendering. --- .../src/js/hypergrid.js | 38 ++++++++++++++++--- 1 file changed, 33 insertions(+), 5 deletions(-) diff --git a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js index 8427845812..723296b1be 100644 --- a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js +++ b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js @@ -118,8 +118,8 @@ function generateGridProperties(overrides) { function setPSP(payload) { if (payload.data.length === 0) { - this.grid.setData({data: []}) - return + this.grid.setData({data: []}); + return; }; if (payload.isTree) { this.grid.renderer.properties.fixedColumnCount = 1; @@ -229,8 +229,6 @@ function setPSP(payload) { function GridUIFixPlugin(grid) { - - grid.canvas.resize = function() { var box = this.size = this.div.getBoundingClientRect(); @@ -271,6 +269,24 @@ function GridUIFixPlugin(grid) { this.paintNow(); } + grid.canvas._tickPaint = grid.canvas.tickPaint; + grid.canvas.tickPaint = async function (t) { + let range = this.component.grid.getVisibleRows(); + let s = range[1]; + let e = range[range.length - 1]; + if (range.length > 1 && (this.dirty || this.__cached_start !== s || this.__cached_end !== e)) { + if (this._updating_cache) { + this._updating_cache.cancel(); + } + this._updating_cache = this.component.grid._cache_update(s, e); + await this._updating_cache; + this._updateing_cache = undefined; + this.__cached_start = s; + this.__cached_end = e; + } + this.component.grid.canvas._tickPaint(t); + } + grid._getGridCellFromMousePoint = grid.getGridCellFromMousePoint; grid.getGridCellFromMousePoint = function(mouse) { if (this.getRowCount() === 0) { @@ -618,6 +634,14 @@ function PerspectiveDataModel(grid) { // Returns the number of rows for this dataset getRowCount: function () { + // let range = this.grid.getVisibleRows(); + // let s = range[1]; + // let e = range[range.length - 1]; + // if (range.length > 1 && (this.__cached_start !== s || this.__cached_end !== e)) { + // this.__cached_start = s; + // this.__cached_end = e; + // this.grid._cache_update(s, e); + // } return this.dataSource.data.length; }, @@ -925,12 +949,16 @@ async function grid(div, view, hidden) { div.innerHTML = ""; div.appendChild(this.grid); } + this.grid.grid._cache_update = async (s, e) => { + json = await fill_page(view, json, hidden, s, e + 1); + this.grid.set_data(json, schema); + } if (visible_rows.length > 0) { this.grid.set_data(json, schema); this.grid.grid.canvas.resize(); this.grid.grid.canvas.resize(); } - await load_incrementally.call(this, view, schema, hidden, json, nrows, 0); + // await load_incrementally.call(this, view, schema, hidden, json, nrows, 0); } global.registerPlugin("hypergrid", { From 5ef16cb05a867d1da7c719055aa23199260daefb Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Tue, 2 Jan 2018 14:51:29 -0500 Subject: [PATCH 11/29] Don't allocate a string vector for filling psp string columns. --- packages/perspective/src/cpp/main.cpp | 15 ++++++++++----- packages/perspective/src/js/perspective.js | 1 - packages/perspective/test/js/benchmark.js | 8 +++++++- 3 files changed, 17 insertions(+), 7 deletions(-) diff --git a/packages/perspective/src/cpp/main.cpp b/packages/perspective/src/cpp/main.cpp index 5175e500cf..80f162f7fa 100644 --- a/packages/perspective/src/cpp/main.cpp +++ b/packages/perspective/src/cpp/main.cpp @@ -206,12 +206,17 @@ _fill_data(t_table_sptr tbl, break; default: { - std::vector dcol2 = - vecFromJSArray(data_cols[cidx]); - col->fill_vector(dcol2); - if (name == index) + auto dcol = data_cols[cidx]; + t_uint32 size = dcol["length"].as(); + auto fill_index === name == index; + for (auto i = 0; i < size; ++i) { - key_col->fill_vector(dcol2); + auto elem = dcol[i].as(); + col->set_nth(i, elem); + if (fill_index) + { + key_col->set_nth(i, elem); + } } } } diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index 764f6cb415..42509b3c91 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -1027,7 +1027,6 @@ const perspective = { data = "_" + data; } let js = papaparse.parse(data, {dynamicTyping: true, header: true}).data; - //let js = csv2json(data); return perspective.table(js, options); } let pdata = parse_data(data); diff --git a/packages/perspective/test/js/benchmark.js b/packages/perspective/test/js/benchmark.js index 3fce1d6f9d..873b537201 100644 --- a/packages/perspective/test/js/benchmark.js +++ b/packages/perspective/test/js/benchmark.js @@ -190,7 +190,13 @@ function _run_tests() { window.addEventListener("perspective-ready", function() { get_csv('flight_small.csv', function(csv) { - var table = perspective.table(csv); + + let table = perspective.table(csv); + + test(50, function(resolve) { + perspective.table(csv); + resolve(); + }); test(500, function(resolve) { var view = table.view({ From 78960f054d36b906f2c8c06c6e6e59d23fcbe404 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Tue, 2 Jan 2018 18:55:09 -0500 Subject: [PATCH 12/29] hypergrid.js cleanup. --- .../src/js/fixes.js | 320 ++++++++++++++ .../src/js/hypergrid.js | 389 ++---------------- 2 files changed, 344 insertions(+), 365 deletions(-) create mode 100644 packages/perspective-viewer-hypergrid/src/js/fixes.js diff --git a/packages/perspective-viewer-hypergrid/src/js/fixes.js b/packages/perspective-viewer-hypergrid/src/js/fixes.js new file mode 100644 index 0000000000..28dd21e08e --- /dev/null +++ b/packages/perspective-viewer-hypergrid/src/js/fixes.js @@ -0,0 +1,320 @@ +/****************************************************************************** + * + * Copyright (c) 2017, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ + +import rectangular from 'rectangular'; + +export function GridUIFixPlugin(grid) { + + grid.canvas.resize = function() { + var box = this.size = this.div.getBoundingClientRect(); + + this.width = Math.floor(this.div.clientWidth); + this.height = Math.floor(this.div.clientHeight); + + //fix ala sir spinka, see + //http://www.html5rocks.com/en/tutorials/canvas/hidpi/ + //just add 'hdpi' as an attribute to the fin-canvas tag + var ratio = 1; + var isHIDPI = window.devicePixelRatio && this.component.properties.useHiDPI; + if (isHIDPI) { + var devicePixelRatio = window.devicePixelRatio || 1; + var backingStoreRatio = this.gc.webkitBackingStorePixelRatio || + this.gc.mozBackingStorePixelRatio || + this.gc.msBackingStorePixelRatio || + this.gc.oBackingStorePixelRatio || + this.gc.backingStorePixelRatio || 1; + + ratio = devicePixelRatio / backingStoreRatio; + //this.canvasCTX.scale(ratio, ratio); + } + + this.buffer.width = this.canvas.width = this.width * ratio; + this.buffer.height = this.canvas.height = this.height * ratio; + + this.canvas.style.width = this.buffer.style.width = this.width + 'px'; + this.canvas.style.height = this.buffer.style.height = this.height + 'px'; + + this.bc.scale(ratio, ratio); + if (isHIDPI && !this.component.properties.useBitBlit) { + this.gc.scale(ratio, ratio); + } + + this.bounds = new rectangular.Rectangle(0, 0, this.width, this.height); + this.component.setBounds(this.bounds); + this.resizeNotification(); + this.paintNow(); + } + + grid.canvas._tickPaint = grid.canvas.tickPaint; + grid.canvas.tickPaint = async function (t) { + let range = this.component.grid.getVisibleRows(); + let s = range[1]; + let e = range[range.length - 1]; + if (range.length > 1 && this.dirty && (this.__cached_start !== s || this.__cached_end !== e)) { + if (this._updating_cache) { + this._updating_cache.cancel(); + } + this._updating_cache = this.component.grid._cache_update(s, e); + await this._updating_cache; + this._updateing_cache = undefined; + this.__cached_start = s; + this.__cached_end = e; + } + this.component.grid.canvas._tickPaint(t); + } + + grid._getGridCellFromMousePoint = grid.getGridCellFromMousePoint; + grid.getGridCellFromMousePoint = function(mouse) { + if (this.getRowCount() === 0) { + return {'fake': 1}; + } else { + try { + return this._getGridCellFromMousePoint(mouse); + } catch (e) { + return {'fake': 1}; + + } + } + }; + + + grid.renderer._paintGridlines = grid.renderer.paintGridlines; + grid.renderer.paintGridlines = function(gc) { + var visibleColumns = this.visibleColumns, C = visibleColumns.length, + visibleRows = this.visibleRows, R = visibleRows.length; + + if (C && R) { + var gridProps = this.properties, + lineWidth = gridProps.lineWidth, + lineColor = gridProps.lineColor, + viewHeight = visibleRows[R - 1].bottom, + viewWidth = visibleColumns[C - 1].right; + + gc.cache.fillStyle = lineColor; + + if (gridProps.fixedColumnCount > 0 && gridProps.fixedColumnCount < C) { + var lw = gridProps.gridLinesV ? lineWidth + 1: lineWidth; + var fixedX = visibleColumns[gridProps.fixedColumnCount].left - 1; + gc.fillRect(fixedX, 0, lw, viewHeight); + } + + if (gridProps.fixedRowCount > 0 && gridProps.fixedRowCount < R) { + var lw = gridProps.gridLinesH ? lineWidth + 1: lineWidth; + var fixedY = visibleRows[gridProps.fixedRowCount].bottom; + gc.fillRect(0, fixedY, viewWidth, lw); + } + + gc.fillRect(0, visibleRows[0].bottom, viewWidth, 1); + + this._paintGridlines(gc); + } + }; + + grid.renderer.computeCellsBounds = function() { + + var scrollTop = this.getScrollTop(), + scrollLeft = this.getScrollLeft(), + + fixedColumnCount = this.grid.getFixedColumnCount(), + fixedRowCount = this.grid.getFixedRowCount(), + + bounds = this.getBounds(), + grid = this.grid, + behavior = grid.behavior, + editorCellEvent = grid.cellEditor && grid.cellEditor.event, + + vcEd, xEd, + vrEd, yEd, + sgEd, isSubgridEd, + + insertionBoundsCursor = 0, + previousInsertionBoundsCursorValue = 0, + + lineWidthX = grid.properties.gridLinesV ? grid.properties.lineWidth : 1, + lineWidthY = grid.properties.gridLinesH ? grid.properties.lineWidth : 0, + + start = 0, + numOfInternalCols = 0, + x, X, // horizontal pixel loop index and limit + y, Y, // vertical pixel loop index and limit + c, C, // column loop index and limit + g, G, // subgrid loop index and limit + r, R, // row loop index and limitrows in current subgrid + subrows, // rows in subgrid g + base, // sum of rows for all subgrids so far + subgrids = behavior.subgrids, + subgrid, + rowIndex, + scrollableSubgrid, + footerHeight, + vx, vy, + vr, vc, + width, height, + firstVX, lastVX, + firstVY, lastVY, + topR, + xSpaced, widthSpaced, heightSpaced; // adjusted for cell spacing + + if (editorCellEvent) { + xEd = editorCellEvent.gridCell.x; + yEd = editorCellEvent.dataCell.y; + sgEd = editorCellEvent.subgrid; + } + if (grid.properties.showRowNumbers) { + start = behavior.rowColumnIndex; + numOfInternalCols = Math.abs(start); + } + + this.scrollHeight = 0; + + this.visibleColumns.length = 0; + this.visibleRows.length = 0; + + this.visibleColumnsByIndex = []; // array because number of columns will always be reasonable + this.visibleRowsByDataRowIndex = {}; // hash because keyed by (fixed and) scrolled row indexes + + this.insertionBounds = []; + + for ( + x = 0, c = start, C = grid.getColumnCount(), X = bounds.width || grid.canvas.width; + c < C && x <= X; + c++ + ) { + if (c === behavior.treeColumnIndex && !behavior.hasTreeColumn()) { + numOfInternalCols = (numOfInternalCols > 0) ? numOfInternalCols - 1 : 0; + this.visibleColumns[c] = undefined; + continue; + } + + vx = c; + if (c >= fixedColumnCount) { + lastVX = vx += scrollLeft; + if (firstVX === undefined) { + firstVX = lastVX; + } + } + if (vx >= C) { + break; // scrolled beyond last column + } + + width = Math.ceil(behavior.getColumnWidth(vx)); + + xSpaced = x ? x + lineWidthX : x; + widthSpaced = x ? width - lineWidthX : width; + this.visibleColumns[c] = this.visibleColumnsByIndex[vx] = vc = { + index: c, + columnIndex: vx, + column: behavior.getActiveColumn(vx), + left: xSpaced, + width: widthSpaced, + right: xSpaced + widthSpaced + }; + if (xEd === vx) { + vcEd = vc; + } + + x += width; + + insertionBoundsCursor += Math.round(width / 2) + previousInsertionBoundsCursorValue; + this.insertionBounds.push(insertionBoundsCursor); + previousInsertionBoundsCursorValue = Math.round(width / 2); + } + + // get height of total number of rows in all subgrids following the data subgrid + footerHeight = grid.properties.defaultRowHeight * + subgrids.reduce(function(rows, subgrid) { + if (scrollableSubgrid) { + rows += subgrid.getRowCount(); + } else { + scrollableSubgrid = subgrid.isData; + } + return rows; + }, 0); + + for ( + base = r = g = y = 0, G = subgrids.length, Y = bounds.height - footerHeight; + g < G; + g++, base += subrows + ) { + subgrid = subgrids[g]; + subrows = subgrid.getRowCount(); + scrollableSubgrid = subgrid.isData; + isSubgridEd = (sgEd === subgrid); + topR = r; + + // For each row of each subgrid... + for (R = r + subrows; r < R && y < Y; r++) { + vy = r; + if (scrollableSubgrid && r >= fixedRowCount) { + vy += scrollTop; + lastVY = vy - base; + if (firstVY === undefined) { + firstVY = lastVY; + } + if (vy >= R) { + break; // scrolled beyond last row + } + } + + rowIndex = vy - base; + height = behavior.getRowHeight(rowIndex, subgrid); + + heightSpaced = height - lineWidthY; + this.visibleRows[r] = vr = { + index: r, + subgrid: subgrid, + rowIndex: rowIndex, + top: y, + height: heightSpaced, + bottom: y + heightSpaced + }; + + if (scrollableSubgrid) { + this.visibleRowsByDataRowIndex[vy - base] = vr; + } + + if (isSubgridEd && yEd === rowIndex) { + vrEd = vr; + } + + y += height; + } + + if (scrollableSubgrid) { + subrows = r - topR; + Y += footerHeight; + } + } + + if (editorCellEvent) { + editorCellEvent.visibleColumn = vcEd; + editorCellEvent.visibleRow = vrEd; + editorCellEvent.gridCell.y = vrEd && vrEd.index; + editorCellEvent._bounds = null; + } + + this.viewHeight = Y; + + this.dataWindow = this.grid.newRectangle(firstVX, firstVY, lastVX - firstVX, lastVY - firstVY); + + // Resize CellEvent pool + var pool = this.cellEventPool, + previousLength = pool.length, + P = (this.visibleColumns.length + numOfInternalCols) * this.visibleRows.length; + + if (P > previousLength) { + pool.length = P; // grow pool to accommodate more cells + } + for (var p = previousLength; p < P; p++) { + pool[p] = new behavior.CellEvent; // instantiate new members + } + + this.resetAllGridRenderers(); + }; +} \ No newline at end of file diff --git a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js index 723296b1be..7e8d190a59 100644 --- a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js +++ b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js @@ -7,6 +7,8 @@ * */ +import {GridUIFixPlugin} from "./fixes.js"; + const Hypergrid = require("fin-hypergrid"); const Behaviors = require("fin-hypergrid/src/behaviors"); const Base = require("fin-hypergrid/src/Base.js"); @@ -16,8 +18,6 @@ const GroupedHeader = require("./grouped-header.js"); const _ = require("underscore"); -const rectangular = require('rectangular') - import {registerElement, detectChrome} from "@jpmorganchase/perspective-common"; var TEMPLATE = require('../html/hypergrid.html'); @@ -117,7 +117,7 @@ function generateGridProperties(overrides) { } function setPSP(payload) { - if (payload.data.length === 0) { + if (payload.rows.length === 0) { this.grid.setData({data: []}); return; }; @@ -149,11 +149,9 @@ function setPSP(payload) { var old_schema = this.grid.behavior.subgrids.lookup.data.schema; this.schema_loaded = this.schema_loaded && _.isEqual(processed_schema, old_schema); this.schema = processed_schema; - var mergedData = mergePathsAndRows(payload.rowPaths, payload.data, payload.rowLeaf); - this.grid._rowPathsIndex = mergedData.rowPathsIndex; if (this.schema_loaded) { this.grid.setData({ - data: mergedData.data, + data: payload.rows, }); } else { @@ -172,7 +170,7 @@ function setPSP(payload) { } console.log('Setting up initial schema and data load into HyperGrid'); this.grid.behavior.setData({ - data: mergedData.data, + data: payload.rows, schema: this.schema }); this.schema_loaded = true; @@ -227,315 +225,6 @@ function setPSP(payload) { } -function GridUIFixPlugin(grid) { - - grid.canvas.resize = function() { - var box = this.size = this.div.getBoundingClientRect(); - - this.width = Math.floor(this.div.clientWidth); - this.height = Math.floor(this.div.clientHeight); - - //fix ala sir spinka, see - //http://www.html5rocks.com/en/tutorials/canvas/hidpi/ - //just add 'hdpi' as an attribute to the fin-canvas tag - var ratio = 1; - var isHIDPI = window.devicePixelRatio && this.component.properties.useHiDPI; - if (isHIDPI) { - var devicePixelRatio = window.devicePixelRatio || 1; - var backingStoreRatio = this.gc.webkitBackingStorePixelRatio || - this.gc.mozBackingStorePixelRatio || - this.gc.msBackingStorePixelRatio || - this.gc.oBackingStorePixelRatio || - this.gc.backingStorePixelRatio || 1; - - ratio = devicePixelRatio / backingStoreRatio; - //this.canvasCTX.scale(ratio, ratio); - } - - this.buffer.width = this.canvas.width = this.width * ratio; - this.buffer.height = this.canvas.height = this.height * ratio; - - this.canvas.style.width = this.buffer.style.width = this.width + 'px'; - this.canvas.style.height = this.buffer.style.height = this.height + 'px'; - - this.bc.scale(ratio, ratio); - if (isHIDPI && !this.component.properties.useBitBlit) { - this.gc.scale(ratio, ratio); - } - - this.bounds = new rectangular.Rectangle(0, 0, this.width, this.height); - this.component.setBounds(this.bounds); - this.resizeNotification(); - this.paintNow(); - } - - grid.canvas._tickPaint = grid.canvas.tickPaint; - grid.canvas.tickPaint = async function (t) { - let range = this.component.grid.getVisibleRows(); - let s = range[1]; - let e = range[range.length - 1]; - if (range.length > 1 && (this.dirty || this.__cached_start !== s || this.__cached_end !== e)) { - if (this._updating_cache) { - this._updating_cache.cancel(); - } - this._updating_cache = this.component.grid._cache_update(s, e); - await this._updating_cache; - this._updateing_cache = undefined; - this.__cached_start = s; - this.__cached_end = e; - } - this.component.grid.canvas._tickPaint(t); - } - - grid._getGridCellFromMousePoint = grid.getGridCellFromMousePoint; - grid.getGridCellFromMousePoint = function(mouse) { - if (this.getRowCount() === 0) { - return {'fake': 1}; - } else { - try { - return this._getGridCellFromMousePoint(mouse); - } catch (e) { - return {'fake': 1}; - - } - } - }; - - - grid.renderer._paintGridlines = grid.renderer.paintGridlines; - grid.renderer.paintGridlines = function(gc) { - var visibleColumns = this.visibleColumns, C = visibleColumns.length, - visibleRows = this.visibleRows, R = visibleRows.length; - - if (C && R) { - var gridProps = this.properties, - lineWidth = gridProps.lineWidth, - lineColor = gridProps.lineColor, - viewHeight = visibleRows[R - 1].bottom, - viewWidth = visibleColumns[C - 1].right; - - gc.cache.fillStyle = lineColor; - - if (gridProps.fixedColumnCount > 0 && gridProps.fixedColumnCount < C) { - var lw = gridProps.gridLinesV ? lineWidth + 1: lineWidth; - var fixedX = visibleColumns[gridProps.fixedColumnCount].left - 1; - gc.fillRect(fixedX, 0, lw, viewHeight); - } - - if (gridProps.fixedRowCount > 0 && gridProps.fixedRowCount < R) { - var lw = gridProps.gridLinesH ? lineWidth + 1: lineWidth; - var fixedY = visibleRows[gridProps.fixedRowCount].bottom; - gc.fillRect(0, fixedY, viewWidth, lw); - } - - gc.fillRect(0, visibleRows[0].bottom, viewWidth, 1); - - this._paintGridlines(gc); - } - }; - - grid.renderer.computeCellsBounds = function() { - - var scrollTop = this.getScrollTop(), - scrollLeft = this.getScrollLeft(), - - fixedColumnCount = this.grid.getFixedColumnCount(), - fixedRowCount = this.grid.getFixedRowCount(), - - bounds = this.getBounds(), - grid = this.grid, - behavior = grid.behavior, - editorCellEvent = grid.cellEditor && grid.cellEditor.event, - - vcEd, xEd, - vrEd, yEd, - sgEd, isSubgridEd, - - insertionBoundsCursor = 0, - previousInsertionBoundsCursorValue = 0, - - lineWidthX = grid.properties.gridLinesV ? grid.properties.lineWidth : 1, - lineWidthY = grid.properties.gridLinesH ? grid.properties.lineWidth : 0, - - start = 0, - numOfInternalCols = 0, - x, X, // horizontal pixel loop index and limit - y, Y, // vertical pixel loop index and limit - c, C, // column loop index and limit - g, G, // subgrid loop index and limit - r, R, // row loop index and limitrows in current subgrid - subrows, // rows in subgrid g - base, // sum of rows for all subgrids so far - subgrids = behavior.subgrids, - subgrid, - rowIndex, - scrollableSubgrid, - footerHeight, - vx, vy, - vr, vc, - width, height, - firstVX, lastVX, - firstVY, lastVY, - topR, - xSpaced, widthSpaced, heightSpaced; // adjusted for cell spacing - - if (editorCellEvent) { - xEd = editorCellEvent.gridCell.x; - yEd = editorCellEvent.dataCell.y; - sgEd = editorCellEvent.subgrid; - } - if (grid.properties.showRowNumbers) { - start = behavior.rowColumnIndex; - numOfInternalCols = Math.abs(start); - } - - this.scrollHeight = 0; - - this.visibleColumns.length = 0; - this.visibleRows.length = 0; - - this.visibleColumnsByIndex = []; // array because number of columns will always be reasonable - this.visibleRowsByDataRowIndex = {}; // hash because keyed by (fixed and) scrolled row indexes - - this.insertionBounds = []; - - for ( - x = 0, c = start, C = grid.getColumnCount(), X = bounds.width || grid.canvas.width; - c < C && x <= X; - c++ - ) { - if (c === behavior.treeColumnIndex && !behavior.hasTreeColumn()) { - numOfInternalCols = (numOfInternalCols > 0) ? numOfInternalCols - 1 : 0; - this.visibleColumns[c] = undefined; - continue; - } - - vx = c; - if (c >= fixedColumnCount) { - lastVX = vx += scrollLeft; - if (firstVX === undefined) { - firstVX = lastVX; - } - } - if (vx >= C) { - break; // scrolled beyond last column - } - - width = Math.ceil(behavior.getColumnWidth(vx)); - - xSpaced = x ? x + lineWidthX : x; - widthSpaced = x ? width - lineWidthX : width; - this.visibleColumns[c] = this.visibleColumnsByIndex[vx] = vc = { - index: c, - columnIndex: vx, - column: behavior.getActiveColumn(vx), - left: xSpaced, - width: widthSpaced, - right: xSpaced + widthSpaced - }; - if (xEd === vx) { - vcEd = vc; - } - - x += width; - - insertionBoundsCursor += Math.round(width / 2) + previousInsertionBoundsCursorValue; - this.insertionBounds.push(insertionBoundsCursor); - previousInsertionBoundsCursorValue = Math.round(width / 2); - } - - // get height of total number of rows in all subgrids following the data subgrid - footerHeight = grid.properties.defaultRowHeight * - subgrids.reduce(function(rows, subgrid) { - if (scrollableSubgrid) { - rows += subgrid.getRowCount(); - } else { - scrollableSubgrid = subgrid.isData; - } - return rows; - }, 0); - - for ( - base = r = g = y = 0, G = subgrids.length, Y = bounds.height - footerHeight; - g < G; - g++, base += subrows - ) { - subgrid = subgrids[g]; - subrows = subgrid.getRowCount(); - scrollableSubgrid = subgrid.isData; - isSubgridEd = (sgEd === subgrid); - topR = r; - - // For each row of each subgrid... - for (R = r + subrows; r < R && y < Y; r++) { - vy = r; - if (scrollableSubgrid && r >= fixedRowCount) { - vy += scrollTop; - lastVY = vy - base; - if (firstVY === undefined) { - firstVY = lastVY; - } - if (vy >= R) { - break; // scrolled beyond last row - } - } - - rowIndex = vy - base; - height = behavior.getRowHeight(rowIndex, subgrid); - - heightSpaced = height - lineWidthY; - this.visibleRows[r] = vr = { - index: r, - subgrid: subgrid, - rowIndex: rowIndex, - top: y, - height: heightSpaced, - bottom: y + heightSpaced - }; - - if (scrollableSubgrid) { - this.visibleRowsByDataRowIndex[vy - base] = vr; - } - - if (isSubgridEd && yEd === rowIndex) { - vrEd = vr; - } - - y += height; - } - - if (scrollableSubgrid) { - subrows = r - topR; - Y += footerHeight; - } - } - - if (editorCellEvent) { - editorCellEvent.visibleColumn = vcEd; - editorCellEvent.visibleRow = vrEd; - editorCellEvent.gridCell.y = vrEd && vrEd.index; - editorCellEvent._bounds = null; - } - - this.viewHeight = Y; - - this.dataWindow = this.grid.newRectangle(firstVX, firstVY, lastVX - firstVX, lastVY - firstVY); - - // Resize CellEvent pool - var pool = this.cellEventPool, - previousLength = pool.length, - P = (this.visibleColumns.length + numOfInternalCols) * this.visibleRows.length; - - if (P > previousLength) { - pool.length = P; // grow pool to accommodate more cells - } - for (var p = previousLength; p < P; p++) { - pool[p] = new behavior.CellEvent; // instantiate new members - } - - this.resetAllGridRenderers(); - }; -} function CheckboxTrackingPlugin(grid) { @@ -565,7 +254,11 @@ function CheckboxTrackingPlugin(grid) { grid.getRowIdx = function (rowPath) { var path = Array.isArray(rowPath) ? JSON.stringify(rowPath) : rowPath; - return grid._rowPathsIndex[path]; + for (let i = 0; i < grid.getRowCount(); i ++) { + if (JSON.stringify(grid.getRow(i).rowPath) === path) { + return i; + } + } }; } @@ -584,7 +277,7 @@ function PerspectiveDataModel(grid) { grid.mixIn.call(grid.behavior.dataModel, { // Override setData - setData: function (dataPayload, schema, cache_update) { + setData: function (dataPayload, schema) { this.viewData = dataPayload; this.source.setData(dataPayload, schema); }, @@ -634,14 +327,6 @@ function PerspectiveDataModel(grid) { // Returns the number of rows for this dataset getRowCount: function () { - // let range = this.grid.getVisibleRows(); - // let s = range[1]; - // let e = range[range.length - 1]; - // if (range.length > 1 && (this.__cached_start !== s || this.__cached_end !== e)) { - // this.__cached_start = s; - // this.__cached_end = e; - // this.grid._cache_update(s, e); - // } return this.dataSource.data.length; }, @@ -725,19 +410,6 @@ function PerspectiveDataModel(grid) { }); } - -function mergePathsAndRows(rowPaths, rowData, rowLeaf) { - var mergedData = []; - var rowPathsIndex = {}; - for (var i = 0; i < rowData.length; i++) { - var row_data = { rowPath: rowPaths[i], rowData: rowData[i] }; - row_data['isLeaf'] = rowLeaf ? rowLeaf[i] : true; - mergedData.push(row_data); - rowPathsIndex[JSON.stringify(rowPaths[i])] = i; - } - return { data: mergedData, rowPathsIndex: rowPathsIndex }; -} - function convertToType(typ, val) { return ['object', 'boolean'].indexOf(typeof (typ)) > -1 ? JSON.parse(val) : (typ.constructor)(val); } @@ -761,6 +433,7 @@ function psp2hypergrid(data, schema) { columnTypes: [] } } + var is_tree = data[0].hasOwnProperty('__ROW_PATH__'); var columnPaths = Object.keys(data[0]) @@ -769,43 +442,40 @@ function psp2hypergrid(data, schema) { let flat_columns = columnPaths.map(col => col.join(",")); - let row_paths = []; let rows = []; - let row_leaves = []; for (let idx = 0; idx < data.length; idx++) { const row = data[idx] || {}; let new_row = []; + let row_path = []; + let row_leaf = true; if (is_tree) { if (row.__ROW_PATH__ === undefined) { row.__ROW_PATH__ = []; } - row_paths.push(["ROOT"].concat(row.__ROW_PATH__)); + row_path = ["ROOT"].concat(row.__ROW_PATH__); let name = row['__ROW_PATH__'][row['__ROW_PATH__'].length - 1]; if (name === undefined && idx === 0) name = "TOTAL" new_row = [name]; - row_leaves.push(row.__ROW_PATH__.length >= (data[idx + 1] ? data[idx + 1].__ROW_PATH__.length : 0)); - } else { - row_paths.push([]); + row_leaf = row.__ROW_PATH__.length >= (data[idx + 1] ? data[idx + 1].__ROW_PATH__.length : 0); } for (var col of flat_columns) { new_row.push(row[col]); } - rows.push(new_row); + rows.push({ + rowPath: row_path, + rowData: new_row, + rowLeaf: row_leaf + }); } var hg_data = { - rowPaths: row_paths, - data: rows, + rows: rows, isTree: is_tree, configuration: {}, columnPaths: (is_tree ? [[" "]] : []).concat(columnPaths), columnTypes: (is_tree ? ["str"] : []).concat(columnPaths.map(col => conv[schema[col[col.length - 1]]])) }; - if (is_tree) { - hg_data['rowLeaf'] = row_leaves; - } - return hg_data } @@ -917,16 +587,6 @@ async function fill_page(view, json, hidden, start_row, end_row) { return json; } -async function load_incrementally(view, schema, hidden, json, total, page) { - json = await fill_page(view, json, hidden, page * PAGE_SIZE, (page + 1) * PAGE_SIZE); - this.grid.set_data(json, schema); - this.grid.grid.canvas.resize(); - this.grid.grid.canvas.resize(); - if ((page + 1) * PAGE_SIZE < total) { - await load_incrementally.call(this, view, schema, hidden, json, total, page + 1); - } -} - async function grid(div, view, hidden) { let [nrows, json, schema] = await Promise.all([ view.num_rows(), @@ -943,14 +603,14 @@ async function grid(div, view, hidden) { } json.length = nrows; if (visible_rows.length > 0) { - json = await fill_page(view, json, hidden, visible_rows[1], visible_rows[visible_rows.length - 1] + 1); + json = await fill_page(view, json, hidden, visible_rows[1], visible_rows[visible_rows.length - 1] + 2); } if (!(document.contains ? document.contains(this.grid) : false)) { div.innerHTML = ""; div.appendChild(this.grid); } this.grid.grid._cache_update = async (s, e) => { - json = await fill_page(view, json, hidden, s, e + 1); + json = await fill_page(view, json, hidden, s, e + 1); this.grid.set_data(json, schema); } if (visible_rows.length > 0) { @@ -958,7 +618,6 @@ async function grid(div, view, hidden) { this.grid.grid.canvas.resize(); this.grid.grid.canvas.resize(); } - // await load_incrementally.call(this, view, schema, hidden, json, nrows, 0); } global.registerPlugin("hypergrid", { From f7e9662e3f6a236aa1ea85faa41c29be77e454ba Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Wed, 3 Jan 2018 13:39:10 -0500 Subject: [PATCH 13/29] Reduced iteration per page to only visible rows. --- package.json | 2 +- .../src/js/fixes.js | 2 +- .../src/js/hypergrid.js | 25 ++++++++++++------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/package.json b/package.json index 22e73e0753..2a8aca9091 100644 --- a/package.json +++ b/package.json @@ -3,7 +3,7 @@ "lerna": "^2.5.1" }, "scripts": { - "start": "lerna run start ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}}", + "start": "lerna run start ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}} --stream", "puppeteer": "docker run -it --rm --shm-size=2g -u root -e WRITE_TESTS=${WRITE_TESTS} -v $(pwd):/src -w /src/packages/${PACKAGE} zenato/puppeteer ./node_modules/.bin/jest --runInBand", "postinstall": "lerna bootstrap --hoist", "test": "npm run test_perspective && npm run test_viewer && npm run test_hypergrid && npm run test_highcharts", diff --git a/packages/perspective-viewer-hypergrid/src/js/fixes.js b/packages/perspective-viewer-hypergrid/src/js/fixes.js index 28dd21e08e..d2fa72c6e8 100644 --- a/packages/perspective-viewer-hypergrid/src/js/fixes.js +++ b/packages/perspective-viewer-hypergrid/src/js/fixes.js @@ -56,7 +56,7 @@ export function GridUIFixPlugin(grid) { let range = this.component.grid.getVisibleRows(); let s = range[1]; let e = range[range.length - 1]; - if (range.length > 1 && this.dirty && (this.__cached_start !== s || this.__cached_end !== e)) { + if (this.component.grid._cache_update && range.length > 1 && this.dirty && (this.__cached_start !== s || this.__cached_end !== e)) { if (this._updating_cache) { this._updating_cache.cancel(); } diff --git a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js index 7e8d190a59..f4747c9b09 100644 --- a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js +++ b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js @@ -146,9 +146,11 @@ function setPSP(payload) { col_settings['type'] = payload.columnTypes[i] === 'str' ? 'string' : payload.columnTypes[i]; processed_schema.push(col_settings); } + var old_schema = this.grid.behavior.subgrids.lookup.data.schema; this.schema_loaded = this.schema_loaded && _.isEqual(processed_schema, old_schema); this.schema = processed_schema; + if (this.schema_loaded) { this.grid.setData({ data: payload.rows, @@ -169,7 +171,7 @@ function setPSP(payload) { } } console.log('Setting up initial schema and data load into HyperGrid'); - this.grid.behavior.setData({ + this.grid.setData({ data: payload.rows, schema: this.schema }); @@ -279,7 +281,7 @@ function PerspectiveDataModel(grid) { // Override setData setData: function (dataPayload, schema) { this.viewData = dataPayload; - this.source.setData(dataPayload, schema); + this.source.setData(dataPayload, schema); }, // Is the grid view a tree @@ -422,7 +424,7 @@ var conv = { 'date': 'date' } -function psp2hypergrid(data, schema) { +function psp2hypergrid(data, schema, start = 0, end = undefined, length = undefined) { if (data.length === 0) { return { rowPaths: [], @@ -443,7 +445,10 @@ function psp2hypergrid(data, schema) { let flat_columns = columnPaths.map(col => col.join(",")); let rows = []; - for (let idx = 0; idx < data.length; idx++) { + if (length) { + rows.length = length; + } + for (let idx = start; idx < (end || data.length); idx++) { const row = data[idx] || {}; let new_row = []; let row_path = []; @@ -461,11 +466,11 @@ function psp2hypergrid(data, schema) { for (var col of flat_columns) { new_row.push(row[col]); } - rows.push({ + rows[idx] ={ rowPath: row_path, rowData: new_row, - rowLeaf: row_leaf - }); + isLeaf: row_leaf + }; } var hg_data = { @@ -610,8 +615,10 @@ async function grid(div, view, hidden) { div.appendChild(this.grid); } this.grid.grid._cache_update = async (s, e) => { - json = await fill_page(view, json, hidden, s, e + 1); - this.grid.set_data(json, schema); + json = await fill_page(view, json, hidden, s, e + 10); + let rows = psp2hypergrid(json, schema, s, Math.min(e + 10, nrows), nrows).rows; + rows[0] = this.grid.grid.behavior.dataModel.viewData[0]; + this.grid.grid.setData({data: rows}); } if (visible_rows.length > 0) { this.grid.set_data(json, schema); From 8c7566d178945da53fdb00d239f3692024dbd8e0 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Wed, 3 Jan 2018 13:39:49 -0500 Subject: [PATCH 14/29] Removed conversion of JS columns to vectors in C++ to reduce memory consumption. --- packages/perspective/src/cpp/main.cpp | 57 +++++++++++++++------------ 1 file changed, 31 insertions(+), 26 deletions(-) diff --git a/packages/perspective/src/cpp/main.cpp b/packages/perspective/src/cpp/main.cpp index 80f162f7fa..8ac3846b94 100644 --- a/packages/perspective/src/cpp/main.cpp +++ b/packages/perspective/src/cpp/main.cpp @@ -134,6 +134,33 @@ _get_aggspecs(val j_aggs) return aggspecs; } +/** + * + * + * Params + * ------ + * + * + * Returns + * ------- + * + */ +template +void +_fill_col(val dcol, t_col_sptr col, t_col_sptr key_col, bool fill_index) +{ + t_uint32 size = dcol["length"].as(); + for (auto i = 0; i < size; ++i) + { + auto elem = dcol[i].as(); + col->set_nth(i, elem); + if (fill_index) + { + key_col->set_nth(i, elem); + } + } +} + /** * * @@ -161,17 +188,12 @@ _fill_data(t_table_sptr tbl, auto name = ocolnames[cidx]; auto col = tbl->get_column(name); auto col_type = odt[cidx]; + auto fill_index = name == index; switch (col_type) { case DTYPE_INT32: { - std::vector dcol = - vecFromJSArray(data_cols[cidx]); - col->fill_vector(dcol); - if (name == index) - { - key_col->fill_vector(dcol); - } + _fill_col(data_cols[cidx], col, key_col, fill_index); } break; case DTYPE_BOOL: @@ -183,13 +205,7 @@ _fill_data(t_table_sptr tbl, break; case DTYPE_FLOAT64: { - std::vector dcol = - vecFromJSArray(data_cols[cidx]); - col->fill_vector(dcol); - if (name == index) - { - key_col->fill_vector(dcol); - } + _fill_col(data_cols[cidx], col, key_col, fill_index); } break; case DTYPE_TIME: @@ -206,18 +222,7 @@ _fill_data(t_table_sptr tbl, break; default: { - auto dcol = data_cols[cidx]; - t_uint32 size = dcol["length"].as(); - auto fill_index === name == index; - for (auto i = 0; i < size; ++i) - { - auto elem = dcol[i].as(); - col->set_nth(i, elem); - if (fill_index) - { - key_col->set_nth(i, elem); - } - } + _fill_col(data_cols[cidx], col, key_col, fill_index); } } } From bb7066f627b2eb3a5309384a5ab78e3ed222c520 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Fri, 5 Jan 2018 06:51:32 -0500 Subject: [PATCH 15/29] Removed bluebird. --- packages/perspective-common/common.config.js | 2 +- packages/perspective-common/index.js | 5 ----- packages/perspective-common/package.json | 3 --- packages/perspective-viewer/package.json | 1 - 4 files changed, 1 insertion(+), 10 deletions(-) diff --git a/packages/perspective-common/common.config.js b/packages/perspective-common/common.config.js index d925414b22..bc79b492ba 100644 --- a/packages/perspective-common/common.config.js +++ b/packages/perspective-common/common.config.js @@ -53,7 +53,7 @@ module.exports = function() { loader: "babel-loader", options: { presets: ['env'], - plugins: ['transform-promise-to-bluebird', 'transform-async-to-bluebird', 'transform-runtime', ["transform-es2015-for-of", { + plugins: ['transform-runtime', ["transform-es2015-for-of", { "loose": true }]] } diff --git a/packages/perspective-common/index.js b/packages/perspective-common/index.js index 36db446112..ecd66de0a6 100644 --- a/packages/perspective-common/index.js +++ b/packages/perspective-common/index.js @@ -7,11 +7,6 @@ * */ -require('bluebird').config({ - cancellation: true, - longStackTraces: false -}); - /** * Detect Internet Explorer. * diff --git a/packages/perspective-common/package.json b/packages/perspective-common/package.json index 7e7a6955bd..e1ab7bb32b 100644 --- a/packages/perspective-common/package.json +++ b/packages/perspective-common/package.json @@ -16,15 +16,12 @@ "author": "", "license": "Apache", "dependencies": { - "bluebird": "^3.5.1", "babel-runtime": "^6.26.0" }, "devDependencies": { "babel-core": "^6.26.0", "babel-loader": "^7.1.2", - "babel-plugin-transform-async-to-bluebird": "^1.1.1", "babel-plugin-transform-es2015-for-of": "^6.23.0", - "babel-plugin-transform-promise-to-bluebird": "^1.1.1", "babel-plugin-transform-runtime": "^6.23.0", "babel-polyfill": "^6.26.0", "babel-preset-env": "^1.6.0", diff --git a/packages/perspective-viewer/package.json b/packages/perspective-viewer/package.json index 0ba5f94cca..ce61ec36ee 100644 --- a/packages/perspective-viewer/package.json +++ b/packages/perspective-viewer/package.json @@ -43,7 +43,6 @@ "@jpmorganchase/perspective-common": "^0.0.1", "babel-core": "^6.26.0", "babel-loader": "^7.1.2", - "babel-plugin-transform-async-to-bluebird": "^1.1.1", "babel-plugin-transform-es2015-for-of": "^6.23.0", "babel-plugin-transform-promise-to-bluebird": "^1.1.1", "babel-plugin-transform-runtime": "^6.23.0", From 8e7a09c011706494f5ca3a1251b7fea865a67434 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Fri, 5 Jan 2018 06:55:31 -0500 Subject: [PATCH 16/29] Enable lazy mode only when a threshold is exceeded. --- .../src/js/fixes.js | 42 ++++++++++++----- .../src/js/hypergrid.js | 46 +++++++++++++------ 2 files changed, 62 insertions(+), 26 deletions(-) diff --git a/packages/perspective-viewer-hypergrid/src/js/fixes.js b/packages/perspective-viewer-hypergrid/src/js/fixes.js index d2fa72c6e8..4f3d57da20 100644 --- a/packages/perspective-viewer-hypergrid/src/js/fixes.js +++ b/packages/perspective-viewer-hypergrid/src/js/fixes.js @@ -50,23 +50,41 @@ export function GridUIFixPlugin(grid) { this.resizeNotification(); this.paintNow(); } + + function is_subrange(sub, sup) { + return !sup || (sup[0] <= sub[0] && sup[1] >= sub[1]); + } + + function estimate_range(grid) { + let range = Object.keys(grid.renderer.visibleRowsByDataRowIndex); + return [parseInt(range[0]), parseInt(range[range.length - 1])]; + } grid.canvas._tickPaint = grid.canvas.tickPaint; grid.canvas.tickPaint = async function (t) { - let range = this.component.grid.getVisibleRows(); - let s = range[1]; - let e = range[range.length - 1]; - if (this.component.grid._cache_update && range.length > 1 && this.dirty && (this.__cached_start !== s || this.__cached_end !== e)) { - if (this._updating_cache) { - this._updating_cache.cancel(); + if (this.component.grid._lazy_load) { + let range = estimate_range(this.component.grid); + if ( + this.component.grid._cache_update + && this.dirty + && !(this._updating_cache && is_subrange(range, this._updating_cache.range)) + ) { + this._updating_cache = this.component.grid._cache_update(...range); + this._updating_cache.range = range + await this._updating_cache; + let new_range = estimate_range(this.component.grid); + if (!is_subrange(new_range, range)) { + return; + } + this._cached_range = range; + this._updating_cache = undefined; + this.component.grid.canvas._tickPaint(t); + } else if (is_subrange(range, this._cached_range)) { + this.component.grid.canvas._tickPaint(t); } - this._updating_cache = this.component.grid._cache_update(s, e); - await this._updating_cache; - this._updateing_cache = undefined; - this.__cached_start = s; - this.__cached_end = e; + } else { + this.component.grid.canvas._tickPaint(t); } - this.component.grid.canvas._tickPaint(t); } grid._getGridCellFromMousePoint = grid.getGridCellFromMousePoint; diff --git a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js index f4747c9b09..3067534ee0 100644 --- a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js +++ b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js @@ -97,11 +97,11 @@ var light_theme_overrides = { ], hoverCellHighlight: { enabled: true, - backgroundColor: 'rgba(211, 221, 232, 0.85)' + backgroundColor: '#eeeeee' }, hoverRowHighlight: { enabled: true, - backgroundColor: 'rgba(211, 221, 232, 0.50)' + backgroundColor: '#f6f6f6' }, }; @@ -569,10 +569,9 @@ registerElement(TEMPLATE, { const PAGE_SIZE = 1000; -async function fill_page(view, json, hidden, start_row, end_row) { - let next_page = await view.to_json({start_row: start_row, end_row: end_row}); +function filter_hidden(hidden, json) { if (hidden.length > 0) { - let first = next_page[0]; + let first = json[0]; let to_delete = []; for (let key in first) { let split_key = key.split(','); @@ -580,18 +579,26 @@ async function fill_page(view, json, hidden, start_row, end_row) { to_delete.push(key); } } - for (let row of next_page) { + for (let row of json) { for (let h of to_delete) { delete row[h]; } } } + return json +} + +async function fill_page(view, json, hidden, start_row, end_row) { + let next_page = await view.to_json({start_row: start_row, end_row: end_row}); + next_page = filter_hidden(hidden, next_page); for (let idx = 0; idx < next_page.length; idx++) { json[start_row + idx] = next_page[idx]; } return json; } +const LAZY_THRESHOLD = 10000; + async function grid(div, view, hidden) { let [nrows, json, schema] = await Promise.all([ view.num_rows(), @@ -599,32 +606,43 @@ async function grid(div, view, hidden) { view.schema() ]); let visible_rows = []; + if (!this.grid) { this.grid = document.createElement('perspective-hypergrid'); visible_rows = [0, 0, 100]; } else if (this.grid.grid) { - this.grid.grid.canvas.stopResizing(); visible_rows = this.grid.grid.getVisibleRows(); } + json.length = nrows; - if (visible_rows.length > 0) { - json = await fill_page(view, json, hidden, visible_rows[1], visible_rows[visible_rows.length - 1] + 2); + let lazy_load = nrows > LAZY_THRESHOLD; + if (lazy_load) { + json = await fill_page(view, json, hidden, visible_rows[1], visible_rows[visible_rows.length - 1] + 2); + } else if (visible_rows.length > 0) { + json = await view.to_json(); + json = filter_hidden(hidden, json); } if (!(document.contains ? document.contains(this.grid) : false)) { div.innerHTML = ""; div.appendChild(this.grid); } + + await Promise.resolve(); + + this.grid.grid._lazy_load = false; + this.grid.grid._cache_update = async (s, e) => { json = await fill_page(view, json, hidden, s, e + 10); let rows = psp2hypergrid(json, schema, s, Math.min(e + 10, nrows), nrows).rows; rows[0] = this.grid.grid.behavior.dataModel.viewData[0]; this.grid.grid.setData({data: rows}); } - if (visible_rows.length > 0) { - this.grid.set_data(json, schema); - this.grid.grid.canvas.resize(); - this.grid.grid.canvas.resize(); - } + + this.grid.set_data(json, schema); + this.grid.grid.canvas.resize(); + this.grid.grid.canvas.resize(); + + this.grid.grid._lazy_load = lazy_load; } global.registerPlugin("hypergrid", { From e30f0f92a3d58dd40163e5103cd1d64f3829e62d Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Fri, 5 Jan 2018 06:56:01 -0500 Subject: [PATCH 17/29] Test for lazy view expansion. --- .../test/js/superstore.spec.js | 38 ++++++++++++++++++- .../test/results/results.json | 12 +++--- 2 files changed, 44 insertions(+), 6 deletions(-) diff --git a/packages/perspective-viewer-hypergrid/test/js/superstore.spec.js b/packages/perspective-viewer-hypergrid/test/js/superstore.spec.js index 5c9488f503..003b5ba4f6 100644 --- a/packages/perspective-viewer-hypergrid/test/js/superstore.spec.js +++ b/packages/perspective-viewer-hypergrid/test/js/superstore.spec.js @@ -7,4 +7,40 @@ * */ -require('@jpmorganchase/perspective-viewer/test/js/superstore.spec.js'); \ No newline at end of file +const utils = require('@jpmorganchase/perspective-viewer/test/js/utils.js'); + +const simple_tests = require('@jpmorganchase/perspective-viewer/test/js/simple_tests.js'); + +async function set_lazy(page) { + const viewer = await page.$('perspective-viewer'); + await page.evaluate(element => { + element.grid.grid.properties.repaintIntervalRate = 1; + Object.defineProperty(element.grid.grid, '_lazy_load', { + set: () => {}, + get: () => true + }); + }, viewer); +} + +utils.with_server({}, () => { + + describe.page("superstore.html", () => { + + simple_tests.default(); + + describe("lazy render mode", () => { + + test.capture("resets viewable area when the logical size expands.", async page => { + await set_lazy(page); + await page.click('#config_button'); + const viewer = await page.$('perspective-viewer'); + await page.evaluate(element => element.setAttribute('column-pivots', '["Category"]'), viewer); + await page.waitForSelector('perspective-viewer:not([updating])'); + await page.evaluate(element => element.setAttribute('row-pivots', '["City"]'), viewer); + }); + + }); + + }); + +}); diff --git a/packages/perspective-viewer-hypergrid/test/results/results.json b/packages/perspective-viewer-hypergrid/test/results/results.json index 0083cefd32..49d4945cd4 100644 --- a/packages/perspective-viewer-hypergrid/test/results/results.json +++ b/packages/perspective-viewer-hypergrid/test/results/results.json @@ -1,10 +1,12 @@ { "superstore.html/shows a grid without any settings applied.": "b09dc5301dbfe6afbfcd2dc58843a475", - "superstore.html/pivots by a row.": "3a0c709f1829207628aea8e7998fcfb3", - "superstore.html/pivots by two rows.": "61ecce517f57c45cdfcb894389059c42", - "superstore.html/pivots by a row and a column.": "17314167c41fb4f6683a75eeb4ca2e01", - "superstore.html/pivots by two rows and two columns.": "c33b4b735f79b209e8ef26ac5c48f6ce", + "superstore.html/pivots by a row.": "5260127d95553860fcc9ad56768ea427", + "superstore.html/pivots by two rows.": "f53fdf0be253dc4a3c5d86a03c50044f", + "superstore.html/pivots by a row and a column.": "47ea6ae4282c002c2da57d3e2008646d", + "superstore.html/pivots by two rows and two columns.": "c3939b8853d3db3d380fa3936dfb0559", "superstore.html/sorts by a numeric column.": "2eb4a45b85bedbdd2cef8bb8c712ed5c", "superstore.html/sorts by an alpha column.": "f99960fb04d633f96a190df6f55ed000", - "superstore.html/displays visible columns.": "95ea7a90e503ed69e4cc87b9f71adbb4" + "superstore.html/displays visible columns.": "95ea7a90e503ed69e4cc87b9f71adbb4", + "superstore.html/resets viewable area when the logical size expands.": "a7933b107656467730a115cec818d625", + "superstore.html/selecting a column does not log a context deleted error.": "2563581d3556c4576745443c8498bea9" } \ No newline at end of file From 4d470733134ae328665c7cf44ac4ba240bd87ad5 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Fri, 5 Jan 2018 06:56:32 -0500 Subject: [PATCH 18/29] Fix for regression in in-browser jasmine tests. --- packages/perspective/src/js/perspective.parallel.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/perspective/src/js/perspective.parallel.js b/packages/perspective/src/js/perspective.parallel.js index d6ab5de91c..aece0ea11e 100644 --- a/packages/perspective/src/js/perspective.parallel.js +++ b/packages/perspective/src/js/perspective.parallel.js @@ -233,7 +233,7 @@ let _initialized = false; worker.prototype._handle = function(e) { if (!this._worker.initialized.value) { - if (_initialized) { + if (!_initialized) { var event = document.createEvent("Event"); event.initEvent("perspective-ready", false, true); window.dispatchEvent(event); From 4597d49cd234b763196b51782978a2d0d8aad274 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Fri, 5 Jan 2018 06:57:08 -0500 Subject: [PATCH 19/29] Expose built assets rather than source to prevent babel conflicts. --- packages/perspective-viewer-highcharts/package.json | 2 +- packages/perspective-viewer-hypergrid/package.json | 2 +- packages/perspective-viewer/package.json | 2 +- packages/perspective-viewer/src/js/view.js | 2 +- packages/perspective/package.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/perspective-viewer-highcharts/package.json b/packages/perspective-viewer-highcharts/package.json index 9687439d47..43ffac266c 100644 --- a/packages/perspective-viewer-highcharts/package.json +++ b/packages/perspective-viewer-highcharts/package.json @@ -2,7 +2,7 @@ "name": "@jpmorganchase/perspective-viewer-highcharts", "version": "0.0.1", "description": "Perspective.js", - "main": "src/js/highcharts.js", + "main": "build/highcharts.plugin.js", "directories": { "test": "test" }, diff --git a/packages/perspective-viewer-hypergrid/package.json b/packages/perspective-viewer-hypergrid/package.json index 1e2860a1a4..066b23308d 100644 --- a/packages/perspective-viewer-hypergrid/package.json +++ b/packages/perspective-viewer-hypergrid/package.json @@ -2,7 +2,7 @@ "name": "@jpmorganchase/perspective-viewer-hypergrid", "version": "0.0.1", "description": "Perspective.js", - "main": "src/js/hypergrid.js", + "main": "build/hypergrid.plugin.js", "directories": { "test": "test" }, diff --git a/packages/perspective-viewer/package.json b/packages/perspective-viewer/package.json index ce61ec36ee..ff5f1d526a 100644 --- a/packages/perspective-viewer/package.json +++ b/packages/perspective-viewer/package.json @@ -2,7 +2,7 @@ "name": "@jpmorganchase/perspective-viewer", "version": "0.0.1", "description": "Perspective.js", - "main": "src/js/view.js", + "main": "build/perspective.view.js", "files": [ "src/*", "build/*" diff --git a/packages/perspective-viewer/src/js/view.js b/packages/perspective-viewer/src/js/view.js index 027194e709..f9176822d0 100644 --- a/packages/perspective-viewer/src/js/view.js +++ b/packages/perspective-viewer/src/js/view.js @@ -10,7 +10,7 @@ import _ from "underscore"; import {polyfill} from "mobile-drag-drop"; -import perspective from "@jpmorganchase/perspective"; +import perspective from "@jpmorganchase/perspective/src/js/perspective.parallel.js"; import {registerElement, importTemplate} from "@jpmorganchase/perspective-common"; import template from "../html/view.html"; diff --git a/packages/perspective/package.json b/packages/perspective/package.json index 01e857038f..3dc4694ef1 100644 --- a/packages/perspective/package.json +++ b/packages/perspective/package.json @@ -2,7 +2,7 @@ "name": "@jpmorganchase/perspective", "version": "0.0.1", "description": "Perspective.js", - "main": "src/js/perspective.parallel.js", + "main": "build/perspective.js", "publishConfig": { "access": "public" }, From 4b4ce9e67fe11ff558f40df58e5d34312ec4149b Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Fri, 5 Jan 2018 06:57:56 -0500 Subject: [PATCH 20/29] Removed reliance on bluebird cancel(). --- packages/perspective-viewer/src/js/view.js | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/packages/perspective-viewer/src/js/view.js b/packages/perspective-viewer/src/js/view.js index f9176822d0..a07e2cab60 100644 --- a/packages/perspective-viewer/src/js/view.js +++ b/packages/perspective-viewer/src/js/view.js @@ -123,12 +123,12 @@ function column_visibility_clicked(ev) { } let cols = this._view_columns('#column_names perspective-row:not(.off)'); this.setAttribute('columns', JSON.stringify(cols)); - update.call(this); + this._update(); } function column_aggregate_clicked() { this.setAttribute('aggregates', JSON.stringify(this._get_view_aggregates())); - update.call(this); + this._update(); } /****************************************************************************** @@ -271,7 +271,11 @@ function update() { this._debounced = setTimeout(() => { this._debounced = undefined; const t = performance.now(); - this._plugin.create.call(this, this._datavis, this._view, hidden, false).then(() => { + if (this._task) { + this._task.cancelled = true; + } + this._task = {cancelled: false} + this._plugin.create.call(this, this._datavis, this._view, hidden, false, this._task).then(() => { this.setAttribute('render_time', performance.now() - t); }); }, timeout || 0); @@ -282,9 +286,10 @@ function update() { const t = performance.now(); this._render_count = (this._render_count || 0) + 1; if (this._task) { - this._task.cancel(); + this._task.cancelled = true; } - this._task = this._plugin.create.call(this, this._datavis, this._view, hidden, true).then(() => { + this._task = {cancelled: false} + this._plugin.create.call(this, this._datavis, this._view, hidden, true, this._task).then(() => { if (!this.hasAttribute('render_time')) { this.dispatchEvent(new Event('loaded', {bubbles: true})); } From d7d868102f7ba698518ba61b17520de0da2b04e9 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Fri, 5 Jan 2018 06:58:44 -0500 Subject: [PATCH 21/29] Check for console.error() in tests. --- packages/perspective-viewer/test/js/utils.js | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/perspective-viewer/test/js/utils.js b/packages/perspective-viewer/test/js/utils.js index 3759ae2bad..6cee20b17a 100644 --- a/packages/perspective-viewer/test/js/utils.js +++ b/packages/perspective-viewer/test/js/utils.js @@ -129,6 +129,9 @@ test.capture = function capture(name, body, timeout = 60000) { if (process.env.DEBUG) private_console.log("---- " + name + " -----------------------------"); const page = await browser.newPage(); page.on('console', msg => { + if (msg.type === 'error') { + errors.push(msg.message); + } if (process.env.DEBUG) { private_console.log(msg.text); } @@ -147,7 +150,7 @@ test.capture = function capture(name, body, timeout = 60000) { // let animation run; await page.waitForSelector('perspective-viewer:not([updating])'); - await page.waitFor(300); + await page.waitFor(500); const screenshot = await page.screenshot(); await page.close(); From 451fa2d9ba593e2fe912a02bf87abc2f4cfb5398 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Fri, 5 Jan 2018 07:04:07 -0500 Subject: [PATCH 22/29] Fixed unicode output clobbering in minified package, which breaks servers which only support ascii. --- packages/perspective-common/common.config.js | 5 ++--- .../perspective-viewer-highcharts/test/results/results.json | 4 ++-- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/perspective-common/common.config.js b/packages/perspective-common/common.config.js index bc79b492ba..676303063d 100644 --- a/packages/perspective-common/common.config.js +++ b/packages/perspective-common/common.config.js @@ -6,9 +6,8 @@ const plugins = [] if (!process.env.PSP_NO_MINIFY) { plugins.push(new UglifyJSPlugin({ sourceMap: true, - uglifyOptions: { - sourceMap: true, - ecma: 5 + output: { + ascii_only: true } })); } diff --git a/packages/perspective-viewer-highcharts/test/results/results.json b/packages/perspective-viewer-highcharts/test/results/results.json index afd1c0c269..33608a084f 100644 --- a/packages/perspective-viewer-highcharts/test/results/results.json +++ b/packages/perspective-viewer-highcharts/test/results/results.json @@ -19,14 +19,14 @@ "bar.html/pivots by a row.": "667024057e32ac7dcff85f714e991345", "bar.html/pivots by two rows.": "38985a461a70412e91ba04ed08630f56", "bar.html/pivots by a row and a column.": "d3c12fafa6aa6e48a65fb772633db007", - "bar.html/pivots by two rows and two columns.": "62d79c966449d7baeb024991038dbce9", + "bar.html/pivots by two rows and two columns.": "f8541acf149555a07db51b75fe25b3de", "bar.html/sorts by a numeric column.": "89ca3dd689ac395e44e77520845055f4", "bar.html/sorts by an alpha column.": "9116319b7b31aed9beba0632e474ddf0", "line.html/shows a grid without any settings applied.": "5c7d75d2872635968019ef08bc505ff3", "line.html/pivots by a row.": "a5c448c1f00f22657dae1ce3e059b075", "line.html/pivots by two rows.": "1944115018252f8dceb03f84fab2ebf4", "line.html/pivots by a row and a column.": "79c490c3f843b24af104b6681685b82f", - "line.html/pivots by two rows and two columns.": "76b7c6b1c0854e3fc295fb4ad2bc8e86", + "line.html/pivots by two rows and two columns.": "2199f0cb57cdab991f9a62c5c277e3b3", "line.html/sorts by a numeric column.": "604f906365754839e95d0c7f01f15241", "line.html/sorts by an alpha column.": "a9cf05d78de91c435055049ed538d1ad", "line.html/displays visible columns.": "010890a5ebf5e4bd32557191a6b11c27", From d09ac64190bd730e7ac179b3f5b97b1ce05507e1 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sat, 6 Jan 2018 19:13:55 -0500 Subject: [PATCH 23/29] Handle perspective errors in worker mode. --- packages/perspective/src/js/perspective.js | 10 ++++++++++ packages/perspective/src/js/perspective.parallel.js | 10 +++++++--- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index 42509b3c91..193eb41afb 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -949,6 +949,11 @@ if (typeof self !== "undefined" && self.addEventListener) { data: data }); } + }).catch(error => { + self.postMessage({ + id: msg.id, + error: error + }); }); } else { self.postMessage({ @@ -974,6 +979,11 @@ if (typeof self !== "undefined" && self.addEventListener) { id: msg.id, data: result }); + }).catch(error => { + self.postMessage({ + id: msg.id, + error: error + }); }); } break; diff --git a/packages/perspective/src/js/perspective.parallel.js b/packages/perspective/src/js/perspective.parallel.js index aece0ea11e..eddc6eecd5 100644 --- a/packages/perspective/src/js/perspective.parallel.js +++ b/packages/perspective/src/js/perspective.parallel.js @@ -62,8 +62,8 @@ function subscribe(method, cmd) { function async_queue(method, cmd) { return function() { var args = Array.prototype.slice.call(arguments, 0, arguments.length); - return new Promise(function(resolve) { - this._worker.handlers[++this._worker.msg_id] = resolve; + return new Promise(function(resolve, reject) { + this._worker.handlers[++this._worker.msg_id] = {resolve, reject}; var msg = { id: this._worker.msg_id, cmd: cmd || 'view_method', @@ -250,7 +250,11 @@ worker.prototype._handle = function(e) { if (e.data.id) { var handler = this._worker.handlers[e.data.id]; if (handler) { - handler(e.data.data); + if (e.data.error) { + handler.reject(e.data.error); + } else { + handler.resolve(e.data.data); + } if (!handler.keep_alive) { delete this._worker.handlers[e.data.id]; } From 32c612938cbdf17e7a1d0cec92892913f058f802 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sat, 6 Jan 2018 19:14:30 -0500 Subject: [PATCH 24/29] Fixed test failures when error messages are logged. --- packages/perspective-viewer/test/js/utils.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/perspective-viewer/test/js/utils.js b/packages/perspective-viewer/test/js/utils.js index 6cee20b17a..8c76f07d50 100644 --- a/packages/perspective-viewer/test/js/utils.js +++ b/packages/perspective-viewer/test/js/utils.js @@ -130,7 +130,7 @@ test.capture = function capture(name, body, timeout = 60000) { const page = await browser.newPage(); page.on('console', msg => { if (msg.type === 'error') { - errors.push(msg.message); + errors.push(msg.text); } if (process.env.DEBUG) { private_console.log(msg.text); From 2734977c1d58b8fffc750063584ecc9000ea6f0b Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sat, 6 Jan 2018 19:15:01 -0500 Subject: [PATCH 25/29] HIghcharts task cancellation. --- packages/perspective-viewer-highcharts/src/js/highcharts.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/perspective-viewer-highcharts/src/js/highcharts.js b/packages/perspective-viewer-highcharts/src/js/highcharts.js index 501acb781c..0e96a818f0 100644 --- a/packages/perspective-viewer-highcharts/src/js/highcharts.js +++ b/packages/perspective-viewer-highcharts/src/js/highcharts.js @@ -221,13 +221,17 @@ function* visible_rows(json, hidden) { } export function draw(mode) { - return async function(el, view, hidden, redraw) { + return async function(el, view, hidden, redraw, task) { var row_pivots = this._view_columns('#row_pivots perspective-row:not(.off)'); var col_pivots = this._view_columns('#column_pivots perspective-row:not(.off)'); var aggregates = this._get_view_aggregates(); let [js, schema] = await Promise.all([view.to_json(), view.schema()]); + if (task.cancelled) { + return; + } + let [series, top, colorRange] = _make_series.call(this, js, row_pivots, col_pivots, mode, hidden); var colors = COLORS_20; From 0a024440333e7db72ce0a5221ec6692885eac45b Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sat, 6 Jan 2018 19:16:45 -0500 Subject: [PATCH 26/29] Fixed cache population to prevent unnecessary updates. --- .../src/js/fixes.js | 37 ------- .../src/js/hypergrid.js | 100 ++++++++++++++---- .../test/js/superstore.spec.js | 9 ++ .../test/results/results.json | 5 +- 4 files changed, 93 insertions(+), 58 deletions(-) diff --git a/packages/perspective-viewer-hypergrid/src/js/fixes.js b/packages/perspective-viewer-hypergrid/src/js/fixes.js index 4f3d57da20..9d68147e7a 100644 --- a/packages/perspective-viewer-hypergrid/src/js/fixes.js +++ b/packages/perspective-viewer-hypergrid/src/js/fixes.js @@ -50,42 +50,6 @@ export function GridUIFixPlugin(grid) { this.resizeNotification(); this.paintNow(); } - - function is_subrange(sub, sup) { - return !sup || (sup[0] <= sub[0] && sup[1] >= sub[1]); - } - - function estimate_range(grid) { - let range = Object.keys(grid.renderer.visibleRowsByDataRowIndex); - return [parseInt(range[0]), parseInt(range[range.length - 1])]; - } - - grid.canvas._tickPaint = grid.canvas.tickPaint; - grid.canvas.tickPaint = async function (t) { - if (this.component.grid._lazy_load) { - let range = estimate_range(this.component.grid); - if ( - this.component.grid._cache_update - && this.dirty - && !(this._updating_cache && is_subrange(range, this._updating_cache.range)) - ) { - this._updating_cache = this.component.grid._cache_update(...range); - this._updating_cache.range = range - await this._updating_cache; - let new_range = estimate_range(this.component.grid); - if (!is_subrange(new_range, range)) { - return; - } - this._cached_range = range; - this._updating_cache = undefined; - this.component.grid.canvas._tickPaint(t); - } else if (is_subrange(range, this._cached_range)) { - this.component.grid.canvas._tickPaint(t); - } - } else { - this.component.grid.canvas._tickPaint(t); - } - } grid._getGridCellFromMousePoint = grid.getGridCellFromMousePoint; grid.getGridCellFromMousePoint = function(mouse) { @@ -101,7 +65,6 @@ export function GridUIFixPlugin(grid) { } }; - grid.renderer._paintGridlines = grid.renderer.paintGridlines; grid.renderer.paintGridlines = function(gc) { var visibleColumns = this.visibleColumns, C = visibleColumns.length, diff --git a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js index 3067534ee0..e90d2413cc 100644 --- a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js +++ b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js @@ -501,6 +501,47 @@ function null_formatter(formatter, null_value = '') { } return formatter } + +function is_subrange(sub, sup) { + if (!sup) { + return false; + } + return sup[0] <= sub[0] && sup[1] >= sub[1]; +} + +function estimate_range(grid) { + let range = Object.keys(grid.renderer.visibleRowsByDataRowIndex); + return [parseInt(range[0]), parseInt(range[range.length - 1]) + 2]; +} + +function CachedRendererPlugin(grid) { + grid.canvas._paintNow = grid.canvas.paintNow; + grid.canvas.paintNow = async function (t) { + if (this.component.grid._lazy_load) { + let range = estimate_range(this.component.grid); + if ( + this.component.grid._cache_update + && ( + (this.component.grid._updating_cache && !is_subrange(range, this.component.grid._updating_cache.range)) + || (!this.component.grid._updating_cache && !is_subrange(range, this.component.grid._cached_range)) + ) + ) { + this.component.grid._updating_cache = this.component.grid._cache_update(...range); + this.component.grid._updating_cache.range = range + let updated = await this.component.grid._updating_cache; + if (updated) { + this.component.grid._updating_cache = undefined; + this.component.grid._cached_range = range; + this.component.grid.canvas._paintNow(t); + } + } else if (is_subrange(range, this.component.grid._cached_range)) { + this.component.grid.canvas._paintNow(t); + } + } else { + this.component.grid.canvas._paintNow(t); + } + } +} registerElement(TEMPLATE, { @@ -530,7 +571,13 @@ registerElement(TEMPLATE, { host.setAttribute('hidden', true); this.grid = new Hypergrid(host, { Behavior: Behaviors.JSON }); host.removeAttribute('hidden'); - this.grid.installPlugins([GridUIFixPlugin, PerspectiveDataModel, CheckboxTrackingPlugin ]); + + this.grid.installPlugins([ + GridUIFixPlugin, + PerspectiveDataModel, + CheckboxTrackingPlugin, + CachedRendererPlugin + ]); var grid_properties = generateGridProperties(light_theme_overrides); grid_properties['showRowNumbers'] = grid_properties['showCheckboxes'] || grid_properties['showRowNumbers']; @@ -599,50 +646,65 @@ async function fill_page(view, json, hidden, start_row, end_row) { const LAZY_THRESHOLD = 10000; -async function grid(div, view, hidden) { +async function grid(div, view, hidden, redraw, task) { + let [nrows, json, schema] = await Promise.all([ view.num_rows(), view.to_json({end_row: 1}), view.schema() ]); - let visible_rows = []; + + let visible_rows; if (!this.grid) { this.grid = document.createElement('perspective-hypergrid'); - visible_rows = [0, 0, 100]; - } else if (this.grid.grid) { - visible_rows = this.grid.grid.getVisibleRows(); } json.length = nrows; + let lazy_load = nrows > LAZY_THRESHOLD; - if (lazy_load) { - json = await fill_page(view, json, hidden, visible_rows[1], visible_rows[visible_rows.length - 1] + 2); - } else if (visible_rows.length > 0) { - json = await view.to_json(); - json = filter_hidden(hidden, json); + + if (!lazy_load) { + json = view.to_json().then(json => filter_hidden(hidden, json)); + } else { + json = Promise.resolve(json); } + if (!(document.contains ? document.contains(this.grid) : false)) { div.innerHTML = ""; div.appendChild(this.grid); + await new Promise(resolve => setTimeout(resolve)); } - await Promise.resolve(); + json = await json; + if (task.cancelled) { + return; + } - this.grid.grid._lazy_load = false; + + this.grid.grid._lazy_load = lazy_load; + this.grid.grid._cached_range = undefined; this.grid.grid._cache_update = async (s, e) => { - json = await fill_page(view, json, hidden, s, e + 10); - let rows = psp2hypergrid(json, schema, s, Math.min(e + 10, nrows), nrows).rows; - rows[0] = this.grid.grid.behavior.dataModel.viewData[0]; - this.grid.grid.setData({data: rows}); + json = await fill_page(view, json, hidden, s, e); + let new_range = estimate_range(this.grid.grid); + if (is_subrange(new_range, [s, e])) { + let rows = psp2hypergrid(json, schema, s, Math.min(e, nrows), nrows).rows; + rows[0] = this.grid.grid.behavior.dataModel.viewData[0]; + this.grid.grid.setData({data: rows}); + return true; + } else { + return false; + } } - + this.grid.set_data(json, schema); this.grid.grid.canvas.resize(); this.grid.grid.canvas.resize(); - this.grid.grid._lazy_load = lazy_load; + if (this._updating_cache) { + await this._updating_cache; + } } global.registerPlugin("hypergrid", { diff --git a/packages/perspective-viewer-hypergrid/test/js/superstore.spec.js b/packages/perspective-viewer-hypergrid/test/js/superstore.spec.js index 003b5ba4f6..ccb7bf34c8 100644 --- a/packages/perspective-viewer-hypergrid/test/js/superstore.spec.js +++ b/packages/perspective-viewer-hypergrid/test/js/superstore.spec.js @@ -39,6 +39,15 @@ utils.with_server({}, () => { await page.evaluate(element => element.setAttribute('row-pivots', '["City"]'), viewer); }); + test.capture("resets viewable area when the physical size expands.", async page => { + await set_lazy(page); + await page.click('#config_button'); + const viewer = await page.$('perspective-viewer'); + await page.evaluate(element => element.setAttribute('row-pivots', '["Category"]'), viewer); + await page.waitForSelector('perspective-viewer:not([updating])'); + await page.evaluate(element => element.setAttribute('row-pivots', '[]'), viewer); + await page.click('#config_button'); + }); }); }); diff --git a/packages/perspective-viewer-hypergrid/test/results/results.json b/packages/perspective-viewer-hypergrid/test/results/results.json index 49d4945cd4..40f4c367ab 100644 --- a/packages/perspective-viewer-hypergrid/test/results/results.json +++ b/packages/perspective-viewer-hypergrid/test/results/results.json @@ -7,6 +7,7 @@ "superstore.html/sorts by a numeric column.": "2eb4a45b85bedbdd2cef8bb8c712ed5c", "superstore.html/sorts by an alpha column.": "f99960fb04d633f96a190df6f55ed000", "superstore.html/displays visible columns.": "95ea7a90e503ed69e4cc87b9f71adbb4", - "superstore.html/resets viewable area when the logical size expands.": "a7933b107656467730a115cec818d625", - "superstore.html/selecting a column does not log a context deleted error.": "2563581d3556c4576745443c8498bea9" + "superstore.html/resets viewable area when the logical size expands.": "6b8f4fa0dc02fe60f6fc14c4c648a66d", + "superstore.html/selecting a column does not log a context deleted error.": "2563581d3556c4576745443c8498bea9", + "superstore.html/resets viewable area when the physical size expands.": "60689232fd5405439ef4d3f2e8dc68e2" } \ No newline at end of file From e7b7c290529f83a5cb1ec5ebf21571adcff35ca2 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Sat, 6 Jan 2018 23:04:03 -0500 Subject: [PATCH 27/29] Fixed timing bug with async hypergrid render updates. --- packages/perspective-viewer-hypergrid/src/js/fixes.js | 2 +- .../perspective-viewer-hypergrid/src/js/hypergrid.js | 11 ++++------- .../test/results/results.json | 4 ++-- packages/perspective/src/js/perspective.parallel.js | 5 ++--- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/packages/perspective-viewer-hypergrid/src/js/fixes.js b/packages/perspective-viewer-hypergrid/src/js/fixes.js index 9d68147e7a..bde89c861e 100644 --- a/packages/perspective-viewer-hypergrid/src/js/fixes.js +++ b/packages/perspective-viewer-hypergrid/src/js/fixes.js @@ -11,7 +11,7 @@ import rectangular from 'rectangular'; export function GridUIFixPlugin(grid) { - grid.canvas.resize = function() { + grid.canvas.resize = async function() { var box = this.size = this.div.getBoundingClientRect(); this.width = Math.floor(this.div.clientWidth); diff --git a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js index e90d2413cc..702a2e6a52 100644 --- a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js +++ b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js @@ -681,7 +681,6 @@ async function grid(div, view, hidden, redraw, task) { return; } - this.grid.grid._lazy_load = lazy_load; this.grid.grid._cached_range = undefined; @@ -699,12 +698,10 @@ async function grid(div, view, hidden, redraw, task) { } this.grid.set_data(json, schema); - this.grid.grid.canvas.resize(); - this.grid.grid.canvas.resize(); - - if (this._updating_cache) { - await this._updating_cache; - } + await this.grid.grid.canvas.resize(); + await this.grid.grid._updating_cache; + await this.grid.grid.canvas.resize(); + await this.grid.grid._updating_cache; } global.registerPlugin("hypergrid", { diff --git a/packages/perspective-viewer-hypergrid/test/results/results.json b/packages/perspective-viewer-hypergrid/test/results/results.json index 40f4c367ab..32d1c9ca23 100644 --- a/packages/perspective-viewer-hypergrid/test/results/results.json +++ b/packages/perspective-viewer-hypergrid/test/results/results.json @@ -7,7 +7,7 @@ "superstore.html/sorts by a numeric column.": "2eb4a45b85bedbdd2cef8bb8c712ed5c", "superstore.html/sorts by an alpha column.": "f99960fb04d633f96a190df6f55ed000", "superstore.html/displays visible columns.": "95ea7a90e503ed69e4cc87b9f71adbb4", - "superstore.html/resets viewable area when the logical size expands.": "6b8f4fa0dc02fe60f6fc14c4c648a66d", + "superstore.html/resets viewable area when the logical size expands.": "a7933b107656467730a115cec818d625", "superstore.html/selecting a column does not log a context deleted error.": "2563581d3556c4576745443c8498bea9", - "superstore.html/resets viewable area when the physical size expands.": "60689232fd5405439ef4d3f2e8dc68e2" + "superstore.html/resets viewable area when the physical size expands.": "75b1b246d3c2612c45fc6c5bc7c3660a" } \ No newline at end of file diff --git a/packages/perspective/src/js/perspective.parallel.js b/packages/perspective/src/js/perspective.parallel.js index eddc6eecd5..f4494bbf4d 100644 --- a/packages/perspective/src/js/perspective.parallel.js +++ b/packages/perspective/src/js/perspective.parallel.js @@ -39,10 +39,9 @@ if (detectIE() && window.location.href.indexOf(__SCRIPT_PATH__.host()) === -1) { function subscribe(method, cmd) { return function() { - var handler = arguments[arguments.length - 1]; + var resolve = arguments[arguments.length - 1]; var args = Array.prototype.slice.call(arguments, 0, arguments.length - 1); - handler.keep_alive = true; - this._worker.handlers[++this._worker.msg_id] = handler; + this._worker.handlers[++this._worker.msg_id] = {resolve, reject: () => {}, keep_alive: true}; var msg = { id: this._worker.msg_id, cmd: cmd || 'view_method', From fa2a210e69f3417cdd3c4ef739c27c62779b51f2 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Mon, 8 Jan 2018 14:42:26 -0500 Subject: [PATCH 28/29] Clean up perspective C++ memory on table allocation failure. --- packages/perspective-examples/package.json | 1 + packages/perspective/src/cpp/main.cpp | 33 ++++++++++++---------- packages/perspective/src/js/perspective.js | 26 +++++++++++------ 3 files changed, 37 insertions(+), 23 deletions(-) diff --git a/packages/perspective-examples/package.json b/packages/perspective-examples/package.json index c61a07f54b..f6fef2d05f 100644 --- a/packages/perspective-examples/package.json +++ b/packages/perspective-examples/package.json @@ -17,6 +17,7 @@ "start": "npm run copy", "copy": "mkdir -p build && cp -r node_modules/@jpmorganchase/perspective/build/* build && cp -r node_modules/@jpmorganchase/perspective-viewer/build/* build && cp -r node_modules/@jpmorganchase/perspective-viewer-hypergrid/build/* build && cp -r node_modules/@jpmorganchase/perspective-viewer-highcharts/build/* build && cp src/html/* build && cp src/json/* build && cp src/csv/* build && cp src/css/* build && cp src/js/* build", "host": "http-server build/", + "remove": "find build \\( -name \"CMake*\" -o -name \"Makefile\" -o -name \"cmake_install.cmake\" -o -name \"compile_commands.json\" -o -name \"psp.js\" \\) -d -exec rm -r \"{}\" \\;", "clean": "find build -mindepth 1 -delete" }, "repository": { diff --git a/packages/perspective/src/cpp/main.cpp b/packages/perspective/src/cpp/main.cpp index 8ac3846b94..d1eec98367 100644 --- a/packages/perspective/src/cpp/main.cpp +++ b/packages/perspective/src/cpp/main.cpp @@ -83,10 +83,6 @@ _get_fterms(t_schema schema, val j_filters) case DTYPE_FLOAT64: term = mktscalar(filter[2].as()); break; - // case DTYPE_STR: - // term = - // mktscalar(filter[2].as().c_str()); - // break; case DTYPE_BOOL: term = mktscalar(filter[2].as()); break; @@ -161,6 +157,22 @@ _fill_col(val dcol, t_col_sptr col, t_col_sptr key_col, bool fill_index) } } +template<> +void +_fill_col(val dcol, t_col_sptr col, t_col_sptr key_col, bool fill_index) +{ + t_uint32 size = dcol["length"].as(); + for (auto i = 0; i < size; ++i) + { + auto elem = static_cast(dcol[i].as()); + col->set_nth(i, elem); + if (fill_index) + { + key_col->set_nth(i, elem); + } + } +} + /** * * @@ -198,9 +210,7 @@ _fill_data(t_table_sptr tbl, break; case DTYPE_BOOL: { - std::vector dcol = - vecFromJSArray(data_cols[cidx]); - col->fill_vector(dcol); + _fill_col(data_cols[cidx], col, key_col, fill_index); } break; case DTYPE_FLOAT64: @@ -210,14 +220,7 @@ _fill_data(t_table_sptr tbl, break; case DTYPE_TIME: { - auto dcol_ = vecFromJSArray(data_cols[cidx]); - std::vector dcol(dcol_.size()); - t_uindex count = 0; - for (t_uindex count=0; count < dcol_.size(); ++count) - { - dcol[count] = dcol_[count]; - } - col->fill_vector(dcol); + _fill_col(data_cols[cidx], col, key_col, fill_index); } break; default: diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index 193eb41afb..f0b3362e07 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -1046,18 +1046,28 @@ const perspective = { } else { tindex = __MODULE__.t_dtype.DTYPE_UINT32; } - let gnode = __MODULE__.make_gnode(pdata.names, pdata.types, tindex); - let pool = new __MODULE__.t_pool({_update_callback: function() {} } ); - let id = pool.register_gnode(gnode); + + let tbl, gnode, pool; + try { - let tbl = __MODULE__.make_table(data.length || 0, pdata.names, pdata.types, pdata.cdata, 0, options.index, tindex); + gnode = __MODULE__.make_gnode(pdata.names, pdata.types, tindex); + pool = new __MODULE__.t_pool({_update_callback: function() {} } ); + let id = pool.register_gnode(gnode); + tbl = __MODULE__.make_table(data.length || 0, pdata.names, pdata.types, pdata.cdata, 0, options.index, tindex); __MODULE__.fill(id, tbl, gnode, pool); - tbl.delete(); return new table(id, gnode, pool, options.index, tindex); } catch (e) { - console.error("Failed to create table"); - console.error(pdata); - return; + if (gnode) { + gnode.delete(); + } + if (pool) { + pool.delete(); + } + throw e; + } finally { + if (tbl) { + tbl.delete(); + } } } } From f4ddc32825b6e0a7a59faaea325aec322438a198 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Mon, 8 Jan 2018 14:50:01 -0500 Subject: [PATCH 29/29] v0.0.2 --- lerna.json | 2 +- packages/perspective-common/package.json | 2 +- packages/perspective-examples/package.json | 10 +++++----- packages/perspective-viewer-highcharts/package.json | 10 +++++----- packages/perspective-viewer-hypergrid/package.json | 10 +++++----- packages/perspective-viewer/package.json | 8 ++++---- packages/perspective/package.json | 6 +++--- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/lerna.json b/lerna.json index 08e3a6172e..ec8b5c85f8 100644 --- a/lerna.json +++ b/lerna.json @@ -3,5 +3,5 @@ "packages": [ "packages/*" ], - "version": "0.0.1" + "version": "0.0.2" } diff --git a/packages/perspective-common/package.json b/packages/perspective-common/package.json index e1ab7bb32b..7c085e9b2b 100644 --- a/packages/perspective-common/package.json +++ b/packages/perspective-common/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-common", - "version": "0.0.1", + "version": "0.0.2", "description": "Perspective.js", "main": "index.js", "files": [ diff --git a/packages/perspective-examples/package.json b/packages/perspective-examples/package.json index f6fef2d05f..0d7c1477cf 100644 --- a/packages/perspective-examples/package.json +++ b/packages/perspective-examples/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-examples", - "version": "0.0.1", + "version": "0.0.2", "description": "Perspective.js", "main": "index.js", "directories": { @@ -27,10 +27,10 @@ "author": "", "license": "Apache", "dependencies": { - "@jpmorganchase/perspective": "^0.0.1", - "@jpmorganchase/perspective-viewer": "^0.0.1", - "@jpmorganchase/perspective-viewer-highcharts": "^0.0.1", - "@jpmorganchase/perspective-viewer-hypergrid": "^0.0.1", + "@jpmorganchase/perspective": "^0.0.2", + "@jpmorganchase/perspective-viewer": "^0.0.2", + "@jpmorganchase/perspective-viewer-highcharts": "^0.0.2", + "@jpmorganchase/perspective-viewer-hypergrid": "^0.0.2", "babel-runtime": "^6.26.0", "query-string": "^5.0.1", "rectangular": "1.0.1", diff --git a/packages/perspective-viewer-highcharts/package.json b/packages/perspective-viewer-highcharts/package.json index 43ffac266c..eb05475671 100644 --- a/packages/perspective-viewer-highcharts/package.json +++ b/packages/perspective-viewer-highcharts/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-viewer-highcharts", - "version": "0.0.1", + "version": "0.0.2", "description": "Perspective.js", "main": "build/highcharts.plugin.js", "directories": { @@ -33,7 +33,7 @@ "author": "", "license": "Apache", "dependencies": { - "@jpmorganchase/perspective-common": "^0.0.1", + "@jpmorganchase/perspective-common": "^0.0.2", "babel-runtime": "^6.26.0", "chroma-js": "^1.3.4", "highcharts": "5.0.14", @@ -42,9 +42,9 @@ "highcharts-more": "^0.1.2" }, "devDependencies": { - "@jpmorganchase/perspective": "^0.0.1", - "@jpmorganchase/perspective-common": "^0.0.1", - "@jpmorganchase/perspective-viewer": "^0.0.1", + "@jpmorganchase/perspective": "^0.0.2", + "@jpmorganchase/perspective-common": "^0.0.2", + "@jpmorganchase/perspective-viewer": "^0.0.2", "babel-core": "^6.26.0", "babel-loader": "^7.1.2", "babel-plugin-transform-es2015-for-of": "^6.23.0", diff --git a/packages/perspective-viewer-hypergrid/package.json b/packages/perspective-viewer-hypergrid/package.json index 066b23308d..cb1688e94a 100644 --- a/packages/perspective-viewer-hypergrid/package.json +++ b/packages/perspective-viewer-hypergrid/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-viewer-hypergrid", - "version": "0.0.1", + "version": "0.0.2", "description": "Perspective.js", "main": "build/hypergrid.plugin.js", "directories": { @@ -33,16 +33,16 @@ "author": "", "license": "Apache", "dependencies": { - "@jpmorganchase/perspective-common": "^0.0.1", + "@jpmorganchase/perspective-common": "^0.0.2", "babel-polyfill": "^6.26.0", "babel-runtime": "^6.26.0", "fin-hypergrid": "2.0.2", "underscore": "^1.8.3" }, "devDependencies": { - "@jpmorganchase/perspective": "^0.0.1", - "@jpmorganchase/perspective-common": "^0.0.1", - "@jpmorganchase/perspective-viewer": "^0.0.1", + "@jpmorganchase/perspective": "^0.0.2", + "@jpmorganchase/perspective-common": "^0.0.2", + "@jpmorganchase/perspective-viewer": "^0.0.2", "babel-core": "^6.26.0", "babel-jest": "^22.0.4", "babel-loader": "^7.1.2", diff --git a/packages/perspective-viewer/package.json b/packages/perspective-viewer/package.json index ff5f1d526a..55a64779c6 100644 --- a/packages/perspective-viewer/package.json +++ b/packages/perspective-viewer/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-viewer", - "version": "0.0.1", + "version": "0.0.2", "description": "Perspective.js", "main": "build/perspective.view.js", "files": [ @@ -30,8 +30,8 @@ "author": "", "license": "Apache", "dependencies": { - "@jpmorganchase/perspective": "^0.0.1", - "@jpmorganchase/perspective-common": "^0.0.1", + "@jpmorganchase/perspective": "^0.0.2", + "@jpmorganchase/perspective-common": "^0.0.2", "babel-polyfill": "^6.26.0", "babel-runtime": "^6.26.0", "bluebird": "^3.5.1", @@ -40,7 +40,7 @@ "underscore": "^1.8.3" }, "devDependencies": { - "@jpmorganchase/perspective-common": "^0.0.1", + "@jpmorganchase/perspective-common": "^0.0.2", "babel-core": "^6.26.0", "babel-loader": "^7.1.2", "babel-plugin-transform-es2015-for-of": "^6.23.0", diff --git a/packages/perspective/package.json b/packages/perspective/package.json index 3dc4694ef1..42433bbcf8 100644 --- a/packages/perspective/package.json +++ b/packages/perspective/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective", - "version": "0.0.1", + "version": "0.0.2", "description": "Perspective.js", "main": "build/perspective.js", "publishConfig": { @@ -38,7 +38,7 @@ "author": "", "license": "Apache", "dependencies": { - "@jpmorganchase/perspective-common": "^0.0.1", + "@jpmorganchase/perspective-common": "^0.0.2", "babel-runtime": "^6.26.0", "bluebird": "^3.5.1", "d3-array": "^1.2.1", @@ -47,7 +47,7 @@ "underscore": "^1.8.3" }, "devDependencies": { - "@jpmorganchase/perspective-common": "^0.0.1", + "@jpmorganchase/perspective-common": "^0.0.2", "babel-core": "^6.26.0", "babel-loader": "^7.1.2", "babel-plugin-transform-async-to-bluebird": "^1.1.1",