Merge pull request #190 from jpmorganchase/remote-performance

Remote performance
finos · Aug 14, 2018 · 7c92d4a · 7c92d4a
2 parents 6b67811 + a608c19
commit 7c92d4a
Show file tree

Hide file tree

Showing 11 changed files with 116 additions and 47 deletions.
diff --git a/packages/perspective-viewer-hypergrid/src/js/PerspectiveDataModel.js b/packages/perspective-viewer-hypergrid/src/js/PerspectiveDataModel.js
@@ -116,7 +116,13 @@ module.exports = require('datasaur-local').extend('PerspectiveDataModel', {
             return this._view.num_rows();
         }).then(nrows => {
             let rects = getSubrects.call(this.grid.renderer, nrows);
-            resolve(!!rects.find(uncachedRow, this));
+            if (!!rects.find(uncachedRow, this)) {
+                this.grid.renderer.dataWindow.left = rects[0].left;
+                this.grid.renderer.dataWindow.top = rects[0].top;
+                this.grid.renderer.dataWindow.width = rects[0].width;
+                this.grid.renderer.dataWindow.height = rects[0].height;
+            }
+            resolve(false);
         }).catch(() => {
             resolve(true);
         }).finally(() => {

diff --git a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js
@@ -15,7 +15,7 @@ const Range = require('./Range');
 const perspectivePlugin = require('./perspective-plugin');
 const PerspectiveDataModel = require('./PerspectiveDataModel');
 const treeLineRendererPaint = require('./hypergrid-tree-cell-renderer').treeLineRendererPaint;
-const psp2hypergrid = require('./psp-to-hypergrid');
+const {psp2hypergrid} = require('./psp-to-hypergrid');
 
 import {bindTemplate} from "@jpmorganchase/perspective-viewer/src/js/utils.js";
 
@@ -315,7 +315,7 @@ async function grid_create(div, view, task) {
     const colPivots = JSON.parse(this.getAttribute('column-pivots'));
     const [nrows, json, schema, tschema] = await Promise.all([
         view.num_rows(),
-        view.to_json(Range.create(0, colPivots.length + 1)),
+        view.to_columns(Range.create(0, colPivots.length + 1)),
         view.schema(),
         this._table.schema()
     ]);
@@ -339,7 +339,7 @@ async function grid_create(div, view, task) {
     dataModel._view = view;
 
     dataModel.pspFetch = async function (range) {
-        let next_page = await view.to_json(range);
+        let next_page = await view.to_columns(range);
         this.data = [];
         const rows = psp2hypergrid(next_page, hidden, schema, tschema, rowPivots).rows;
         const data = this.data

diff --git a/packages/perspective-viewer-hypergrid/src/js/psp-to-hypergrid.js b/packages/perspective-viewer-hypergrid/src/js/psp-to-hypergrid.js
@@ -10,27 +10,19 @@
 const TREE_COLUMN_INDEX = require('fin-hypergrid/src/behaviors/Behavior').prototype.treeColumnIndex;
 
 function filter_hidden(hidden, json) {
-    if (hidden.length > 0) {
-        const first = json[0];
-        const to_delete = [];
-        for (let key in first) {
-            const split_key = key.split(',');
-            if (hidden.indexOf(split_key[split_key.length - 1].trim()) >= 0) {
-                to_delete.push(key);
-            }
-        }
-        for (let row of json) {
-            for (let h of to_delete) {
-                delete row[h];
-            }
+    for (let key of Object.keys(json)) {
+        const split_key = key.split(',');
+        if (hidden.indexOf(split_key[split_key.length - 1].trim()) >= 0) {
+            delete json[key];
         }
     }
     return json;
 }
 
-module.exports = function psp2hypergrid(data, hidden, schema, tschema, row_pivots) {
+function psp2hypergrid(data, hidden, schema, tschema, row_pivots) {
     data = filter_hidden(hidden, data);
-    if (data.length === 0) {
+    const firstcol = Object.keys(data)[0];
+    if (data[firstcol].length === 0) {
         let columns = Object.keys(schema);
         return {
             rows: [],
@@ -43,40 +35,35 @@ module.exports = function psp2hypergrid(data, hidden, schema, tschema, row_pivot
 
     var is_tree = !!row_pivots.length;
 
-    var flat_columns = Object.keys(data[0]).filter(row => row !== '__ROW_PATH__');
+    var flat_columns = Object.keys(data).filter(row => row !== '__ROW_PATH__');
     var columnPaths = flat_columns.map(row => row.split(','));
 
-    let rows = data.map(function (row, idx) {
-        // `dataRow` (element of `dataModel.data`) keys will be `index` here rather than
-        // `columnName` because pivoted data have obscure column names of little use to developer.
-        // This also allows us to override `dataModel.getValue` with a slightly more efficient version
-        // that doesn't require mapping the name through `dataModel.dataSource.schema` to get the index.
+    let rows = [];
+
+    for (let idx = 0; idx < data[firstcol].length; idx++) {
+
         let dataRow = flat_columns.reduce(function (dataRow, columnName, index) {
-            dataRow[index] = row[columnName];
+            dataRow[index] = data[columnName][idx];
             return dataRow;
         }, {});
+        rows.push(dataRow);
 
         if (is_tree) {
-            if (row.__ROW_PATH__ === undefined) {
-                row.__ROW_PATH__ = [];
+            if (data['__ROW_PATH__'][idx] === undefined) {
+                data['__ROW_PATH__'][idx] = [];
             }
 
-            let name = row.__ROW_PATH__[row.__ROW_PATH__.length - 1];
+            let name = data['__ROW_PATH__'][idx][data['__ROW_PATH__'][idx].length - 1];
             if (name === undefined && idx === 0) {
                 name = 'TOTAL';
             }
-
-            // Following stores the tree column under [-1] rather than ['Tree'] so our `getValue`
-            // override can access it using the tree column index rather than the tree column name.
             dataRow[TREE_COLUMN_INDEX] = {
                 rollup: name,
-                rowPath: ['ROOT'].concat(row.__ROW_PATH__),
-                isLeaf: row.__ROW_PATH__.length >= row_pivots.length
+                rowPath: ['ROOT'].concat(data['__ROW_PATH__'][idx]),
+                isLeaf: data['__ROW_PATH__'][idx].length >= row_pivots.length
             };
         }
-
-        return dataRow;
-    });
+    }
 
     return {
         rows: rows,
@@ -87,3 +74,5 @@ module.exports = function psp2hypergrid(data, hidden, schema, tschema, row_pivot
             .concat(columnPaths.map(col => schema[col[col.length - 1]]))
     };
 };
+
+module.exports = {psp2hypergrid}
diff --git a/packages/perspective-viewer/src/js/view.js b/packages/perspective-viewer/src/js/view.js
@@ -194,7 +194,7 @@ function drop(ev) {
     }
 
     // Deselect the dropped column
-    if (this._plugin.deselectMode === "pivots" && this._visible_column_count() > 1 && name !== "sort") {
+    if (this._plugin.deselectMode === "pivots" && this._visible_column_count() > 1 && name !== "sort" && name !== "filter") {
         for (let x of this.querySelectorAll("#active_columns perspective-row")) {
             if (x.getAttribute('name') === data[0]) {
                  this._active_columns.removeChild(x);

diff --git a/packages/perspective/src/config/perspective.node.config.js b/packages/perspective/src/config/perspective.node.config.js
@@ -12,4 +12,9 @@ module.exports = Object.assign({}, common(), {
         path: path.resolve(__dirname, '../../build'),
         libraryTarget: "umd"
     }
+});
+
+module.exports.module.rules.push({
+    test: /\.wasm$/,
+    loader: "arraybuffer-loader"
 });
diff --git a/packages/perspective/src/js/api.js b/packages/perspective/src/js/api.js
@@ -54,6 +54,8 @@ function view(worker, table_name, config) {
 
 view.prototype.to_json = async_queue('to_json');
 
+view.prototype.to_columns = async_queue('to_columns');
+
 view.prototype.to_csv = async_queue('to_csv');
 
 view.prototype.schema = async_queue('schema');

diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js
@@ -563,7 +563,7 @@ const to_format = async function (options, formatter) {
                 if (this.config.row_pivot[0] !== 'psp_okey') {
                     let col_name = "__ROW_PATH__";
                     let row_path = this.ctx.unity_get_row_path(start_row + ridx);
-                    formatter.initColumnValue(row, col_name)
+                    formatter.initColumnValue(data, row, col_name)
                     for (let i = 0; i < row_path.size(); i++) {
                         const value = __MODULE__.scalar_vec_to_val(row_path, i);
                         formatter.addColumnValue(data, row, col_name, value);
@@ -586,8 +586,36 @@ const to_format = async function (options, formatter) {
 
     return formatter.formatData(data, options.config)
 }
+
+/**
+ * Serializes this view to JSON data in a column-oriented format.
+ *
+ * @async
+ *
+ * @param {Object} [options] An optional configuration object.
+ * @param {number} options.start_row The starting row index from which
+ * to serialize.
+ * @param {number} options.end_row The ending row index from which
+ * to serialize.
+ * @param {number} options.start_col The starting column index from which
+ * to serialize.
+ * @param {number} options.end_col The ending column index from which
+ * to serialize.
+ *
+ * @returns {Promise<Array>} A Promise resolving to An array of Objects
+ * representing the rows of this {@link view}.  If this {@link view} had a
+ * "row_pivots" config parameter supplied when constructed, each row Object
+ * will have a "__ROW_PATH__" key, whose value specifies this row's
+ * aggregated path.  If this {@link view} had a "column_pivots" config
+ * parameter supplied, the keys of this object will be comma-prepended with
+ * their comma-separated column paths.
+ */
+view.prototype.to_columns = async function (options) {
+    return to_format.call(this, options, formatters.jsonTableFormatter);
+}
+
 /**
- * Serializes this view to JSON data in a standard format.
+ * Serializes this view to JSON data in a row-oriented format.
  *
  * @async
  *
@@ -609,7 +637,7 @@ const to_format = async function (options, formatter) {
  * parameter supplied, the keys of this object will be comma-prepended with
  * their comma-separated column paths.
  */
-view.prototype.to_json = async function(options) {
+view.prototype.to_json = async function (options) {
     return to_format.call(this, options, formatters.jsonFormatter);
 }
 

diff --git a/packages/perspective/src/js/perspective.node.js b/packages/perspective/src/js/perspective.node.js
@@ -7,18 +7,18 @@
  *
  */
 
+import buffer from "../../build/wasm_async/psp.wasm";
+
 const perspective = require('./perspective.js');
 
 const fs = require('fs');
 
 const WebSocket = require('ws');
 
-let Module;
-
 const load_perspective = require("../../build/wasm_sync/psp.js").load_perspective;
-const wasm = fs.readFileSync('./build/wasm_sync/psp.wasm');
-Module = load_perspective({
-    wasmBinary: wasm,
+
+let Module = load_perspective({
+    wasmBinary: buffer,
     wasmJSMethod: 'native-wasm',
     ENVIRONMENT: "NODE"
 });    

diff --git a/packages/perspective/src/js/view_formatters.js b/packages/perspective/src/js/view_formatters.js
@@ -12,7 +12,7 @@ import papaparse from "papaparse";
 const jsonFormatter = {
     initDataValue: () => [],
     initRowValue: () => ({}),
-    initColumnValue: (row, colName) => row[colName] = [],
+    initColumnValue: (data, row, colName) => row[colName] = [],
     setColumnValue: (data, row, colName, value) => row[colName] = value,
     addColumnValue: (data, row, colName, value) => row[colName].unshift(value),
     addRow: (data, row) => data.push(row),
@@ -23,7 +23,27 @@ const csvFormatter = Object.assign({}, jsonFormatter, {
     formatData: (data, config) => papaparse.unparse(data, config)
 });
 
+const jsonTableFormatter = {
+    initDataValue: () => new Object(),
+    initRowValue: () => {},
+    setColumnValue: (data, row, colName, value) => {
+        data[colName] = data[colName] || [];
+        data[colName].push(value)
+    },
+    addColumnValue: (data, row, colName, value) => {
+        data[colName] = data[colName] || [];
+        data[colName][data[colName].length - 1].unshift(value);
+    },
+    initColumnValue: (data, row, colName) => {
+        data[colName] = data[colName] || [];
+        data[colName].push([]);
+    },
+    addRow: (data, row) => {},
+    formatData: data => data
+}
+
 export default {
     jsonFormatter,
+    jsonTableFormatter,
     csvFormatter
 };
diff --git a/packages/perspective/test/config/test_node.config.js b/packages/perspective/test/config/test_node.config.js
@@ -12,4 +12,9 @@ module.exports = Object.assign({}, common(), {
         path: path.resolve(__dirname, '../../build'),
         libraryTarget: 'umd'
     }
+});
+
+module.exports.module.rules.push({
+    test: /\.wasm$/,
+    loader: "arraybuffer-loader"
 });
diff --git a/packages/perspective/test/js/constructors.js b/packages/perspective/test/js/constructors.js
@@ -44,6 +44,13 @@ var data_3 = [
     {'w': 4.5, 'x': 4, 'y':'d', 'z': false}
 ];
 
+var data_7 = {
+    'w': [1.5, 2.5, 3.5, 4.5],
+    'x': [1, 2, 3, 4],
+    'y': ['a', 'b', 'c', 'd'],
+    'z': [true, false, true, false]
+};
+
 var meta_3 = {
     'w': 'float',
     'x': "integer",
@@ -176,6 +183,13 @@ module.exports = (perspective) => {
             expect(answer).toEqual(result2);
         });
 
+        it("Serializes a simple view to column-oriented JSON", async function () {
+            var table = perspective.table(data_3);
+            var view = table.view({});
+            let result2 = await view.to_columns();
+            expect(data_7).toEqual(result2);
+        });
+
     });
 
     describe("Constructors", function() {