From 3d6a10b79a3648af0f8d6df21be4ba8575e3ef26 Mon Sep 17 00:00:00 2001 From: Sergio Garcia Date: Mon, 9 Jul 2018 11:08:32 -0400 Subject: [PATCH 1/5] Add to_csv to view object --- packages/perspective/src/js/perspective.js | 111 +++++++++++++-------- 1 file changed, 69 insertions(+), 42 deletions(-) diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index daab26a256..36f74d641e 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -12,9 +12,10 @@ import {Table} from "@apache-arrow/es5-esm/table"; import {TypeVisitor} from "@apache-arrow/es5-esm/visitor"; import {Precision} from "@apache-arrow/es5-esm/type"; import {is_valid_date, DateParser} from "./date_parser.js"; - +import {formatters} from "./view_formatters"; import {TYPE_AGGREGATES, AGGREGATE_DEFAULTS, TYPE_FILTERS, FILTER_DEFAULTS, SORT_ORDERS} from "./defaults.js"; + // IE fix - chrono::steady_clock depends on performance.now() which does not exist in IE workers if (global.performance === undefined) { global.performance || {now: Date.now}; @@ -496,31 +497,7 @@ view.prototype.schema = async function() { return new_schema; } -/** - * Serializes this view to JSON data in a standard format. - * - * @async - * - * @param {Object} [options] An optional configuration object. - * @param {number} options.start_row The starting row index from which - * to serialize. - * @param {number} options.end_row The ending row index from which - * to serialize. - * @param {number} options.start_col The starting column index from which - * to serialize. - * @param {number} options.end_col The ending column index from which - * to serialize. - * - * @returns {Promise} A Promise resolving to An array of Objects - * representing the rows of this {@link view}. If this {@link view} had a - * "row_pivots" config parameter supplied when constructed, each row Object - * will have a "__ROW_PATH__" key, whose value specifies this row's - * aggregated path. If this {@link view} had a "column_pivots" config - * parameter supplied, the keys of this object will be comma-prepended with - * their comma-separated column paths. - */ -view.prototype.to_json = async function(options) { - +to_format = async function(options, formatter) { options = options || {}; let viewport = this.config.viewport ? this.config.viewport : {}; let start_row = options.start_row || (viewport.top ? viewport.top : 0); @@ -541,12 +518,6 @@ view.prototype.to_json = async function(options) { let data; - if (options.format && options.format === "table") { - data = {}; - } else { - data = []; - } - let col_names = [[]].concat(this._column_names()); let row, prev_row; let depth = []; @@ -555,38 +526,94 @@ view.prototype.to_json = async function(options) { let cidx = idx % (end_col - start_col); if (cidx === 0) { if (row) { - data.push(row); + formatter.addRow(data, row); } - row = {}; + row = formatter.initRowValue(); ridx ++; } if (this.sides() === 0) { let col_name = col_names[start_col + cidx + 1]; - row[col_name] = slice[idx]; + formatter.setColumnValue(row, col_name, slice[idx]) } else { if (cidx === 0) { if (this.config.row_pivot[0] !== 'psp_okey') { let col_name = "__ROW_PATH__"; let row_path = this.ctx.unity_get_row_path(start_row + ridx); - row[col_name] = []; + formatter.initColumnValue(row, col_name) for (let i = 0; i < row_path.size(); i++) { - row[col_name].unshift(__MODULE__.scalar_vec_to_val(row_path, i)); + const value = __MODULE__.scalar_vec_to_val(row_path, i); + formatter.addColumnValue(row, value, col_name); } row_path.delete(); } } else { let col_name = col_names[start_col + cidx]; - row[col_name] = slice[idx]; + formatter.setColumnValue(row, col_name, slice[idx]) } } } - if (row) data.push(row); + if (row) { + formatter.addRow(data, row); + } if (this.config.row_pivot[0] === 'psp_okey') { - return data.slice(this.config.column_pivot.length); - } else { - return data; + data = data.slice(this.config.column_pivot.length); } + + return formatter.formatData(data) +} +/** + * Serializes this view to JSON data in a standard format. + * + * @async + * + * @param {Object} [options] An optional configuration object. + * @param {number} options.start_row The starting row index from which + * to serialize. + * @param {number} options.end_row The ending row index from which + * to serialize. + * @param {number} options.start_col The starting column index from which + * to serialize. + * @param {number} options.end_col The ending column index from which + * to serialize. + * + * @returns {Promise} A Promise resolving to An array of Objects + * representing the rows of this {@link view}. If this {@link view} had a + * "row_pivots" config parameter supplied when constructed, each row Object + * will have a "__ROW_PATH__" key, whose value specifies this row's + * aggregated path. If this {@link view} had a "column_pivots" config + * parameter supplied, the keys of this object will be comma-prepended with + * their comma-separated column paths. + */ +view.prototype.to_json = async function(options) { + return to_format(options, formatters.jsonFormatter); +} + +/** + * Serializes this view to CSV data in a standard format. + * + * @async + * + * @param {Object} [options] An optional configuration object. + * @param {number} options.start_row The starting row index from which + * to serialize. + * @param {number} options.end_row The ending row index from which + * to serialize. + * @param {number} options.start_col The starting column index from which + * to serialize. + * @param {number} options.end_col The ending column index from which + * to serialize. + * + * @returns {Promise} A Promise resolving to a string in CSV format + * representing the rows of this {@link view}. If this {@link view} had a + * "row_pivots" config parameter supplied when constructed, each row + * will have prepended those values specified by this row's + * aggregated path. If this {@link view} had a "column_pivots" config + * parameter supplied, the keys of this object will be comma-prepended with + * their comma-separated column paths. + */ +view.prototype.to_csv = async function(options) { + return to_format(options, formatters.csvFormatter); } /** From e0f044dd05fca96e7c8213853995c039b71e002e Mon Sep 17 00:00:00 2001 From: Sergio Garcia Date: Mon, 9 Jul 2018 11:28:16 -0400 Subject: [PATCH 2/5] fix called to addColumnValue --- packages/perspective/src/js/perspective.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index 36f74d641e..f30cfcd400 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -542,7 +542,7 @@ to_format = async function(options, formatter) { formatter.initColumnValue(row, col_name) for (let i = 0; i < row_path.size(); i++) { const value = __MODULE__.scalar_vec_to_val(row_path, i); - formatter.addColumnValue(row, value, col_name); + formatter.addColumnValue(row, col_name, value); } row_path.delete(); } From 91ccfb5e3d92a8307b1a8104d3b8a4f7cd3e6f49 Mon Sep 17 00:00:00 2001 From: Sergio Garcia Date: Mon, 9 Jul 2018 11:43:35 -0400 Subject: [PATCH 3/5] add column headers --- packages/perspective/src/js/perspective.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index f30cfcd400..7f7fb58d37 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -516,7 +516,7 @@ to_format = async function(options, formatter) { slice = __MODULE__.get_data_two(this.ctx, start_row, end_row, start_col, end_col); } - let data; + let data = formatter.initDataValue; let col_names = [[]].concat(this._column_names()); let row, prev_row; @@ -533,7 +533,7 @@ to_format = async function(options, formatter) { } if (this.sides() === 0) { let col_name = col_names[start_col + cidx + 1]; - formatter.setColumnValue(row, col_name, slice[idx]) + formatter.setColumnValue(data, row, col_name, slice[idx]) } else { if (cidx === 0) { if (this.config.row_pivot[0] !== 'psp_okey') { @@ -542,13 +542,13 @@ to_format = async function(options, formatter) { formatter.initColumnValue(row, col_name) for (let i = 0; i < row_path.size(); i++) { const value = __MODULE__.scalar_vec_to_val(row_path, i); - formatter.addColumnValue(row, col_name, value); + formatter.addColumnValue(data, row, col_name, value); } row_path.delete(); } } else { let col_name = col_names[start_col + cidx]; - formatter.setColumnValue(row, col_name, slice[idx]) + formatter.setColumnValue(data, row, col_name, slice[idx]) } } } From 09b731453be54de9ac804e112a9aab9f73ef52c4 Mon Sep 17 00:00:00 2001 From: Sergio Garcia Date: Mon, 9 Jul 2018 11:44:43 -0400 Subject: [PATCH 4/5] added missing formatters file --- .../perspective/src/js/view_formatters.js | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 packages/perspective/src/js/view_formatters.js diff --git a/packages/perspective/src/js/view_formatters.js b/packages/perspective/src/js/view_formatters.js new file mode 100644 index 0000000000..2ffbcbc02d --- /dev/null +++ b/packages/perspective/src/js/view_formatters.js @@ -0,0 +1,32 @@ +const jsonFormatter = { + initDataValue: () => [], + initRowValue: () => ({}), + initColumnValue: (row, colName) => row[colName] = {}, + setColumnValue: (row, colName, value) => row[colName] = [value], + addColumnValue: (row, colName, value) => row[colName].unshift(value), + addRow: (data, row) => data.push(row) + formatData: (data) => data +}; + +const csvFormatter = { + initDataValue: () => [''], + initRowValue: () => ({}), + initColumnValue: (row, colName) => void, + setColumnValue: (data, row, colName, value) => { + row.push(value); + //append header + data[0] = data[0] + "," + value; + }, + addColumnValue: (data, row, colName, value) => { + row.unshift(value); + //prepend header + data[0] = value + "," + data[0]; + }, + addRow: (data, row) => data.push(row.toString()), + formatData: (data) => data.join('\r\n') +}; + +export default { + jsonFormatter, + csvFormatter +}; From 689cce2b7141c0ca664076cdabbf0ec4a72c415a Mon Sep 17 00:00:00 2001 From: texodus Date: Mon, 9 Jul 2018 18:28:01 -0400 Subject: [PATCH 5/5] Fixed bugs; added papaparse-based csvFormatter; added tests --- packages/perspective/src/js/perspective.js | 18 ++++---- .../src/js/perspective.parallel.js | 2 + .../perspective/src/js/view_formatters.js | 41 +++++++++---------- packages/perspective/test/js/constructors.js | 35 ++++++++++++++++ 4 files changed, 66 insertions(+), 30 deletions(-) diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index 7f7fb58d37..5138b6e3b2 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -12,7 +12,7 @@ import {Table} from "@apache-arrow/es5-esm/table"; import {TypeVisitor} from "@apache-arrow/es5-esm/visitor"; import {Precision} from "@apache-arrow/es5-esm/type"; import {is_valid_date, DateParser} from "./date_parser.js"; -import {formatters} from "./view_formatters"; +import formatters from "./view_formatters"; import {TYPE_AGGREGATES, AGGREGATE_DEFAULTS, TYPE_FILTERS, FILTER_DEFAULTS, SORT_ORDERS} from "./defaults.js"; @@ -497,7 +497,7 @@ view.prototype.schema = async function() { return new_schema; } -to_format = async function(options, formatter) { +const to_format = async function (options, formatter) { options = options || {}; let viewport = this.config.viewport ? this.config.viewport : {}; let start_row = options.start_row || (viewport.top ? viewport.top : 0); @@ -516,7 +516,7 @@ to_format = async function(options, formatter) { slice = __MODULE__.get_data_two(this.ctx, start_row, end_row, start_col, end_col); } - let data = formatter.initDataValue; + let data = formatter.initDataValue(); let col_names = [[]].concat(this._column_names()); let row, prev_row; @@ -560,7 +560,7 @@ to_format = async function(options, formatter) { data = data.slice(this.config.column_pivot.length); } - return formatter.formatData(data) + return formatter.formatData(data, options.config) } /** * Serializes this view to JSON data in a standard format. @@ -586,7 +586,7 @@ to_format = async function(options, formatter) { * their comma-separated column paths. */ view.prototype.to_json = async function(options) { - return to_format(options, formatters.jsonFormatter); + return to_format.call(this, options, formatters.jsonFormatter); } /** @@ -603,6 +603,8 @@ view.prototype.to_json = async function(options) { * to serialize. * @param {number} options.end_col The ending column index from which * to serialize. + * @param {Object} options.config A config object for the Papaparse {@link https://www.papaparse.com/docs#json-to-csv} + * config object. * * @returns {Promise} A Promise resolving to a string in CSV format * representing the rows of this {@link view}. If this {@link view} had a @@ -612,8 +614,8 @@ view.prototype.to_json = async function(options) { * parameter supplied, the keys of this object will be comma-prepended with * their comma-separated column paths. */ -view.prototype.to_csv = async function(options) { - return to_format(options, formatters.csvFormatter); +view.prototype.to_csv = async function (options) { + return to_format.call(this, options, formatters.csvFormatter); } /** @@ -625,7 +627,7 @@ view.prototype.to_csv = async function(options) { * * @returns {Promise} The number of aggregated rows. */ -view.prototype.num_rows = async function() { +view.prototype.num_rows = async function () { return this.ctx.get_row_count(); } diff --git a/packages/perspective/src/js/perspective.parallel.js b/packages/perspective/src/js/perspective.parallel.js index 93559198db..6af670bff5 100644 --- a/packages/perspective/src/js/perspective.parallel.js +++ b/packages/perspective/src/js/perspective.parallel.js @@ -99,6 +99,8 @@ function view(table_name, worker, config) { view.prototype.to_json = async_queue('to_json'); +view.prototype.to_csv = async_queue('to_csv'); + view.prototype.schema = async_queue('schema'); view.prototype.num_columns = async_queue('num_columns'); diff --git a/packages/perspective/src/js/view_formatters.js b/packages/perspective/src/js/view_formatters.js index 2ffbcbc02d..d9044e3652 100644 --- a/packages/perspective/src/js/view_formatters.js +++ b/packages/perspective/src/js/view_formatters.js @@ -1,30 +1,27 @@ +/****************************************************************************** + * + * Copyright (c) 2017, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ + +import papaparse from "papaparse"; + const jsonFormatter = { initDataValue: () => [], initRowValue: () => ({}), - initColumnValue: (row, colName) => row[colName] = {}, - setColumnValue: (row, colName, value) => row[colName] = [value], - addColumnValue: (row, colName, value) => row[colName].unshift(value), - addRow: (data, row) => data.push(row) - formatData: (data) => data + initColumnValue: (row, colName) => row[colName] = [], + setColumnValue: (data, row, colName, value) => row[colName] = value, + addColumnValue: (data, row, colName, value) => row[colName].unshift(value), + addRow: (data, row) => data.push(row), + formatData: data => data }; -const csvFormatter = { - initDataValue: () => [''], - initRowValue: () => ({}), - initColumnValue: (row, colName) => void, - setColumnValue: (data, row, colName, value) => { - row.push(value); - //append header - data[0] = data[0] + "," + value; - }, - addColumnValue: (data, row, colName, value) => { - row.unshift(value); - //prepend header - data[0] = value + "," + data[0]; - }, - addRow: (data, row) => data.push(row.toString()), - formatData: (data) => data.join('\r\n') -}; +const csvFormatter = Object.assign({}, jsonFormatter, { + formatData: (data, config) => papaparse.unparse(data, config) +}); export default { jsonFormatter, diff --git a/packages/perspective/test/js/constructors.js b/packages/perspective/test/js/constructors.js index 7e663f5e5a..79f1977bb8 100644 --- a/packages/perspective/test/js/constructors.js +++ b/packages/perspective/test/js/constructors.js @@ -137,6 +137,41 @@ module.exports = (perspective) => { }); + describe("Formatters", function () { + + it("Serializes a simple view to CSV", async function () { + var table = perspective.table(data); + var view = table.view({}); + var answer = `x,y,z\r\n1,a,true\r\n2,b,false\r\n3,c,true\r\n4,d,false`; + let result2 = await view.to_csv(); + expect(answer).toEqual(result2); + }); + + it("Serializes 1 sided view to CSV", async function () { + var table = perspective.table(data); + var view = table.view({ + row_pivot: ['z'], + aggregate: [{op: 'sum', column:'x'}], + }); + var answer = `__ROW_PATH__,x\r\n,10\r\nfalse,6\r\ntrue,4`; + let result2 = await view.to_csv(); + expect(answer).toEqual(result2); + }); + + it("Serializes a 2 sided view to CSV", async function () { + var table = perspective.table(data); + var view = table.view({ + row_pivot: ['z'], + column_pivot: ['y'], + aggregate: [{op: 'sum', column:'x'}], + }); + var answer = `__ROW_PATH__,\"a,x\",\"b,x\",\"c,x\",\"d,x\"\r\n,1,2,3,4\r\nfalse,,2,,4\r\ntrue,1,,3,`; + let result2 = await view.to_csv(); + expect(answer).toEqual(result2); + }); + + }); + describe("Constructors", function() { it("JSON constructor", async function () {