Merge pull request #150 from jpmorganchase/to-csv

To csv
finos · Jul 10, 2018 · 187c697 · 187c697
2 parents 708887c + 689cce2
commit 187c697
Show file tree

Hide file tree

Showing 4 changed files with 139 additions and 44 deletions.
diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js
@@ -12,9 +12,10 @@ import {Table} from "@apache-arrow/es5-esm/table";
 import {TypeVisitor} from "@apache-arrow/es5-esm/visitor";
 import {Precision} from "@apache-arrow/es5-esm/type";
 import {is_valid_date, DateParser} from "./date_parser.js";
-
+import formatters from "./view_formatters";
 import {TYPE_AGGREGATES, AGGREGATE_DEFAULTS, TYPE_FILTERS, FILTER_DEFAULTS, SORT_ORDERS} from "./defaults.js";
 
+
 // IE fix - chrono::steady_clock depends on performance.now() which does not exist in IE workers
 if (global.performance === undefined) {
     global.performance || {now: Date.now};
@@ -496,31 +497,7 @@ view.prototype.schema = async function() {
     return new_schema;
 }
 
-/**
- * Serializes this view to JSON data in a standard format.
- *
- * @async
- *
- * @param {Object} [options] An optional configuration object.
- * @param {number} options.start_row The starting row index from which
- * to serialize.
- * @param {number} options.end_row The ending row index from which
- * to serialize.
- * @param {number} options.start_col The starting column index from which
- * to serialize.
- * @param {number} options.end_col The ending column index from which
- * to serialize.
- *
- * @returns {Promise<Array>} A Promise resolving to An array of Objects
- * representing the rows of this {@link view}.  If this {@link view} had a
- * "row_pivots" config parameter supplied when constructed, each row Object
- * will have a "__ROW_PATH__" key, whose value specifies this row's
- * aggregated path.  If this {@link view} had a "column_pivots" config
- * parameter supplied, the keys of this object will be comma-prepended with
- * their comma-separated column paths.
- */
-view.prototype.to_json = async function(options) {
-
+const to_format = async function (options, formatter) {
     options = options || {};
     let viewport = this.config.viewport ? this.config.viewport : {};
     let start_row = options.start_row || (viewport.top ? viewport.top : 0);
@@ -539,13 +516,7 @@ view.prototype.to_json = async function(options) {
         slice = __MODULE__.get_data_two(this.ctx, start_row, end_row, start_col, end_col);
     }
 
-    let data;
-
-    if (options.format && options.format === "table") {
-        data = {};
-    } else {
-        data = [];
-    }
+    let data = formatter.initDataValue();
 
     let col_names = [[]].concat(this._column_names());
     let row, prev_row;
@@ -555,38 +526,96 @@ view.prototype.to_json = async function(options) {
         let cidx = idx % (end_col - start_col);
         if (cidx === 0) {
             if (row) {
-                data.push(row);
+                formatter.addRow(data, row);
             }
-            row = {};
+            row = formatter.initRowValue();
             ridx ++;
         }
         if (this.sides() === 0) {
             let col_name = col_names[start_col + cidx + 1];
-            row[col_name] = slice[idx];
+            formatter.setColumnValue(data, row, col_name, slice[idx])
         } else {
             if (cidx === 0) {
                 if (this.config.row_pivot[0] !== 'psp_okey') {
                     let col_name = "__ROW_PATH__";
                     let row_path = this.ctx.unity_get_row_path(start_row + ridx);
-                    row[col_name] = [];
+                    formatter.initColumnValue(row, col_name)
                     for (let i = 0; i < row_path.size(); i++) {
-                        row[col_name].unshift(__MODULE__.scalar_vec_to_val(row_path, i));
+                        const value = __MODULE__.scalar_vec_to_val(row_path, i);
+                        formatter.addColumnValue(data, row, col_name, value);
                     }
                     row_path.delete();
                 }
             } else {
                 let col_name = col_names[start_col + cidx];
-                row[col_name] = slice[idx];
+                formatter.setColumnValue(data, row, col_name, slice[idx])
             }
         }
     }
 
-    if (row) data.push(row);
+    if (row) {
+        formatter.addRow(data, row);
+    }
     if (this.config.row_pivot[0] === 'psp_okey') {
-        return data.slice(this.config.column_pivot.length);
-    } else {
-        return data;
+        data = data.slice(this.config.column_pivot.length);
     }
+
+    return formatter.formatData(data, options.config)
+}
+/**
+ * Serializes this view to JSON data in a standard format.
+ *
+ * @async
+ *
+ * @param {Object} [options] An optional configuration object.
+ * @param {number} options.start_row The starting row index from which
+ * to serialize.
+ * @param {number} options.end_row The ending row index from which
+ * to serialize.
+ * @param {number} options.start_col The starting column index from which
+ * to serialize.
+ * @param {number} options.end_col The ending column index from which
+ * to serialize.
+ *
+ * @returns {Promise<Array>} A Promise resolving to An array of Objects
+ * representing the rows of this {@link view}.  If this {@link view} had a
+ * "row_pivots" config parameter supplied when constructed, each row Object
+ * will have a "__ROW_PATH__" key, whose value specifies this row's
+ * aggregated path.  If this {@link view} had a "column_pivots" config
+ * parameter supplied, the keys of this object will be comma-prepended with
+ * their comma-separated column paths.
+ */
+view.prototype.to_json = async function(options) {
+    return to_format.call(this, options, formatters.jsonFormatter);
+}
+
+/**
+ * Serializes this view to CSV data in a standard format.
+ *
+ * @async
+ *
+ * @param {Object} [options] An optional configuration object.
+ * @param {number} options.start_row The starting row index from which
+ * to serialize.
+ * @param {number} options.end_row The ending row index from which
+ * to serialize.
+ * @param {number} options.start_col The starting column index from which
+ * to serialize.
+ * @param {number} options.end_col The ending column index from which
+ * to serialize.
+ * @param {Object} options.config A config object for the Papaparse {@link https://www.papaparse.com/docs#json-to-csv}
+ * config object.
+ *
+ * @returns {Promise<string>} A Promise resolving to a string in CSV format
+ * representing the rows of this {@link view}.  If this {@link view} had a
+ * "row_pivots" config parameter supplied when constructed, each row 
+ * will have prepended those values specified by this row's
+ * aggregated path.  If this {@link view} had a "column_pivots" config
+ * parameter supplied, the keys of this object will be comma-prepended with
+ * their comma-separated column paths.
+ */
+view.prototype.to_csv = async function (options) {
+    return to_format.call(this, options, formatters.csvFormatter);
 }
 
 /**
@@ -598,7 +627,7 @@ view.prototype.to_json = async function(options) {
  *
  * @returns {Promise<number>} The number of aggregated rows.
  */
-view.prototype.num_rows = async function() {
+view.prototype.num_rows = async function () {
     return this.ctx.get_row_count();
 }
 

diff --git a/packages/perspective/src/js/perspective.parallel.js b/packages/perspective/src/js/perspective.parallel.js
@@ -99,6 +99,8 @@ function view(table_name, worker, config) {
 
 view.prototype.to_json = async_queue('to_json');
 
+view.prototype.to_csv = async_queue('to_csv');
+
 view.prototype.schema = async_queue('schema');
 
 view.prototype.num_columns = async_queue('num_columns');

diff --git a/packages/perspective/src/js/view_formatters.js b/packages/perspective/src/js/view_formatters.js
@@ -0,0 +1,29 @@
+/******************************************************************************
+ *
+ * Copyright (c) 2017, the Perspective Authors.
+ *
+ * This file is part of the Perspective library, distributed under the terms of
+ * the Apache License 2.0.  The full license can be found in the LICENSE file.
+ *
+ */
+
+import papaparse from "papaparse";
+
+const jsonFormatter = {
+    initDataValue: () => [],
+    initRowValue: () => ({}),
+    initColumnValue: (row, colName) => row[colName] = [],
+    setColumnValue: (data, row, colName, value) => row[colName] = value,
+    addColumnValue: (data, row, colName, value) => row[colName].unshift(value),
+    addRow: (data, row) => data.push(row),
+    formatData: data => data
+};
+
+const csvFormatter = Object.assign({}, jsonFormatter, {
+    formatData: (data, config) => papaparse.unparse(data, config)
+});
+
+export default {
+    jsonFormatter,
+    csvFormatter
+};
diff --git a/packages/perspective/test/js/constructors.js b/packages/perspective/test/js/constructors.js
@@ -137,6 +137,41 @@ module.exports = (perspective) => {
 
     });  
 
+    describe("Formatters", function () {
+
+        it("Serializes a simple view to CSV", async function () {
+            var table = perspective.table(data);
+            var view = table.view({});
+            var answer = `x,y,z\r\n1,a,true\r\n2,b,false\r\n3,c,true\r\n4,d,false`;
+            let result2 = await view.to_csv();
+            expect(answer).toEqual(result2);
+        });
+
+        it("Serializes 1 sided view to CSV", async function () {
+            var table = perspective.table(data);
+            var view = table.view({
+                row_pivot: ['z'],
+                aggregate: [{op: 'sum', column:'x'}],
+            });
+            var answer = `__ROW_PATH__,x\r\n,10\r\nfalse,6\r\ntrue,4`;
+            let result2 = await view.to_csv();
+            expect(answer).toEqual(result2);
+        });
+
+        it("Serializes a 2 sided view to CSV", async function () {
+            var table = perspective.table(data);
+            var view = table.view({
+                row_pivot: ['z'],
+                column_pivot: ['y'],
+                aggregate: [{op: 'sum', column:'x'}],
+            });
+            var answer = `__ROW_PATH__,\"a,x\",\"b,x\",\"c,x\",\"d,x\"\r\n,1,2,3,4\r\nfalse,,2,,4\r\ntrue,1,,3,`;
+            let result2 = await view.to_csv();
+            expect(answer).toEqual(result2);
+        });
+
+    });
+
     describe("Constructors", function() {
 
         it("JSON constructor", async function () {