add 2D columnar charting

finos · texodus · Sep 27, 2018 · Sep 6, 2018 · Sep 7, 2018 · Sep 13, 2018
commit e6f8b11baf34ac9439d834a7e72eb7896e810518
diff --git a/packages/perspective-viewer-highcharts/src/js/draw.js b/packages/perspective-viewer-highcharts/src/js/draw.js
@@ -13,7 +13,7 @@ import "../less/highcharts.less";
 
 import {COLORS_10, COLORS_20} from "./externals.js";
 import {color_axis} from "./color_axis.js";
-import {make_tree_data, make_y_data, make_xy_data, make_xyz_data} from "./series.js";
+import {make_tree_data, make_y_data, make_xy_data, make_xyz_data, make_xy_column_data} from "./series.js";
 import {set_boost, set_category_axis, set_both_axis, default_config, set_tick_size} from "./config.js";
 
 export const draw = mode =>
@@ -35,32 +35,35 @@ export const draw = mode =>
             this._charts = [];
         }
 
-        if (this.hasAttribute("updating") && this._charts.length > 0) {
-            for (let chart of this._charts) {
-                try {
-                    chart.destroy();
-                } catch (e) {
-                    console.warn("Scatter plot destroy() call failed - this is probably leaking memory");
-                }
-            }
-            this._charts = [];
-        }
-
-        let configs = [],
-            xaxis_name = aggregates.length > 0 ? aggregates[0].column : undefined,
-            xaxis_type = schema[xaxis_name],
-            yaxis_name = aggregates.length > 1 ? aggregates[1].column : undefined,
-            yaxis_type = schema[yaxis_name],
-            xtree_name = row_pivots.length > 0 ? row_pivots[row_pivots.length - 1] : undefined,
-            xtree_type = tschema[xtree_name],
-            ytree_name = col_pivots.length > 0 ? col_pivots[col_pivots.length - 1] : undefined,
-            ytree_type = tschema[ytree_name],
-            num_aggregates = aggregates.length - hidden.length;
+    let configs = [],
+        xaxis_name = aggregates.length > 0 ? aggregates[0].column : undefined,
+        xaxis_type = schema[xaxis_name],
+        yaxis_name = aggregates.length > 1 ? aggregates[1].column : undefined,
+        yaxis_type = schema[yaxis_name],
+        xtree_name = row_pivots.length > 0 ? row_pivots[row_pivots.length - 1] : undefined,
+        xtree_type = tschema[xtree_name],
+        ytree_name = col_pivots.length > 0 ? col_pivots[col_pivots.length - 1] : undefined,
+        ytree_type = tschema[ytree_name],
+        num_aggregates = aggregates.length - hidden.length;
 
     if (mode === 'scatter') {
-        js = await view.to_json();
-        let config = configs[0] = default_config.call(this, aggregates, mode, js, col_pivots);
-        let [series, xtop, colorRange, ytop] = make_xy_data(js, schema, aggregates.map(x => x.column), row_pivots, col_pivots, hidden);
+        let s;
+        let config = configs[0] = default_config.call(this, aggregates, mode);
+
+        // determine whether to use column/row data
+        if (col_pivots.length === 0) {
+            const cols = await view.to_columns();
+            s = await make_xy_column_data(cols, schema, aggregates.map(x => x.column), row_pivots, col_pivots, hidden);
+        } else {  
+            js = await view.to_json();
+            s = await make_xy_data(js, schema, aggregates.map(x => x.column), row_pivots, col_pivots, hidden);
+        } 
+
+        const series = s[0];
+        const xtop = s[1];
+        const colorRange = s[2];
+        const ytop = s[3];
+
         config.legend.floating = series.length <= 20;
         config.legend.enabled = col_pivots.length > 0;
         config.series = series;
@@ -184,9 +187,21 @@ export const draw = mode =>
             configs.push(config);
         }
     } else if (mode === 'line') {
-        js = await view.to_json();
+        let s;
         let config = configs[0] = default_config.call(this, aggregates, mode, js, col_pivots);
-        let [series, xtop, , ytop] = make_xy_data(js, schema, aggregates.map(x => x.column), row_pivots, col_pivots, hidden);
+
+        if (col_pivots.length === 0) {
+            const cols = await view.to_columns();
+            s = await make_xy_column_data(cols, schema, aggregates.map(x => x.column), row_pivots, col_pivots, hidden);
+        } else {  
+            js = await view.to_json();
+            s = await make_xy_data(js, schema, aggregates.map(x => x.column), row_pivots, col_pivots, hidden);
+        } 
+
+        const series = s[0];
+        const xtop = s[1];
+        const ytop = s[3];
+
         const colors = series.length <= 10 ? COLORS_10 : COLORS_20;
         config.legend.floating = series.length <= 20;
         config.legend.enabled = col_pivots.length > 0;
@@ -222,12 +237,35 @@ export const draw = mode =>
                 labels: {overflow: 'justify'}
             }
 
-            this._charts = this._charts.map(x => x());
+    if (this.hasAttribute('updating') && this._charts.length > 0) {
+        for (let chart of this._charts) {
+            try {
+                chart.destroy();
+            } catch (e) {
+                console.warn("Scatter plot destroy() call failed - this is probably leaking memory");
+            }
         }
+        this._charts = [];
+    }
 
-        if (!this._charts.every(x => document.contains(x.renderTo))) {
-            for (let e of Array.prototype.slice.call(el.children)) {
-                el.removeChild(e);
+    if (this._charts.length > 0) {
+        let idx = 0;
+        for (let chart of this._charts) {
+            let config = configs[idx++];
+            if (mode === 'scatter') {
+                let conf = {
+                    series: config.series,
+                    plotOptions: {}
+                };
+                set_tick_size.call(this, conf);
+                chart.update(conf);
+            } else if (mode.indexOf('line') > -1) {
+                chart.update({
+                    series: config.series
+                });
+            } else {
+                let opts = {series: config.series, xAxis: config.xAxis, yAxis: config.yAxis};
+                chart.update(opts);
             }
             this._charts.map(x => el.appendChild(x.renderTo));
         }
@@ -237,3 +275,21 @@ export const draw = mode =>
             this._charts.map(x => x.reflow());
         }
     };
+
+        for (let i = 0; i < this._charts.length; i++) {
+            this._charts[i] = this._charts[i]();
+        }
+        //this._charts = this._charts.map(x => x());
+    }
+
+    if (!this._charts.every(x => document.contains(x.renderTo))) {
+        for (let e of Array.prototype.slice.call(el.children)) { el.removeChild(e); }
+        this._charts.map(x => el.appendChild(x.renderTo));
+    }
+
+    // TODO resize bug in Highcharts?
+    if (configs.length > 1) {
+       this._charts.map(x => x.reflow());
+    }
+}
+
diff --git a/packages/perspective-viewer-highcharts/src/js/series.js b/packages/perspective-viewer-highcharts/src/js/series.js
@@ -38,7 +38,7 @@ function column_to_series(data, sname, gname) {
     let s = {
         name: sname,
         connectNulls: true,
-        data: data.map(val => (val === undefined || val === "" ? null : val))
+        data
     };
 
     if (gname) {
@@ -185,10 +185,13 @@ class ColumnIterator {
         for (let name of this.column_names) {
             let data = this.columns[name];
             if (this.columns.__ROW_PATH__) {
-                data = data.filter(
-                    (_, idx) => {
-                        return this.columns.__ROW_PATH__[idx].length === this.pivot_length;
-                    });        
+                let filtered_data = [];
+                for (let i = 0; i < data.length; i++) {
+                    if (this.columns.__ROW_PATH__[i].length === this.pivot_length) {
+                        filtered_data.push(data[i]);
+                    }
+                }
+                data = filtered_data; 
             }
             yield {name, data};
         }
@@ -207,7 +210,10 @@ export function make_y_data(cols, pivots, hidden) {
         } else {
             sname = sname.slice(0, sname.length - 1).join(", ") || " ";
         }
-        let s = column_to_series(col.data, sname, gname);
+        let s = column_to_series(
+            col.data.map(val => val === undefined || val === "" ? null : val), 
+            sname, 
+            gname);
         series.push(s);
     }
 
@@ -243,6 +249,7 @@ export function make_y_heatmap_data(js, pivots, hidden) {
  * XY
  */
 
+// TODO: rewrite
 class TickClean {
     constructor(type) {
         this.dict = {};
@@ -271,6 +278,7 @@ class TickClean {
 class MakeTick {
     constructor(schema, columns) {
         this.schema = schema;
+        // TODO: rewrite
         this.xaxis_clean = new TickClean(schema[columns[0]]);
         this.yaxis_clean = new TickClean(schema[columns[1]]);
         this.color_clean = new TickClean(schema[columns[2]]);
@@ -315,10 +323,147 @@ class MakeTick {
         }
         return tick;
     }
+
+    make_col(cols, col_names, num_cols, pivot_length, row_path, color_range) {
+        let ticks = [];
+        let data = cols;
+
+        if (cols.length === 0) {
+            return ticks;
+        }
+
+        if (cols.length === undefined) {
+            // Dealing with a ColumnIterator object - must map data to 2D array properly
+            data = [];
+            for (let name of col_names) {
+                data.push(cols[name]);
+            }
+        }
+
+        for (let i = 0; i < data[0].length; i++) {
+            if(data[0][i] === null || data[0][i] === undefined || data[0][i] === "") {
+                data[0][i] = null;
+                continue;
+            }
+
+            let tick = {};
+
+            if (row_path) {
+                if (row_path[i].length !== pivot_length) {
+                    continue;
+                }
+
+                tick.name = row_path[i].join(", ");
+            }
+
+            // set x-axis
+            tick.x = data[0][i];
+            tick.x = this.xaxis_clean.clean(tick.x);
+
+            if (num_cols > 1) {
+                // set y-axis
+                tick.y = data[1][i];
+                tick.y = this.yaxis_clean.clean(tick.y);
+            }
+
+            if (num_cols > 2) {
+                // set color
+                let color = data[2][i];
+                if (this.schema[col_names[2]] === "string") {
+                    let color_index = this.color_clean.clean(color);
+                    tick.marker = {
+                        lineColor: color_index,
+                        fillColor: color_index
+                    };
+                } else {
+                    if (!isNaN(color)) {
+                        color_range[0] = Math.min(color_range[0], color);
+                        color_range[1] = Math.max(color_range[1], color);
+                    }
+                    tick.colorValue = color;
+                }
+            }
+
+            if (num_cols > 3) {
+                // set size
+                let size = data[3][i];
+                tick.z = isNaN(size) ? 1 : size;
+            }
+
+            ticks.push(tick);
+        }
+
+        return ticks;
+    }    
 }
 
-export function make_xy_data(js, schema, columns, pivots, col_pivots, hidden) {
-    // TODO: use column data
+export async function make_xy_column_data(cols, schema, aggs, pivots, col_pivots, hidden) {
+    const columns = new ColumnIterator(cols, hidden, pivots.length);
+    let series = [];
+    let color_range = [Infinity, -Infinity];
+    let make_tick = new MakeTick(schema, columns.column_names);
+
+    let row_path = columns.columns.__ROW_PATH__;
+
+    if (col_pivots.length === 0) {
+        let ticks = make_tick.make_col(
+            columns.columns,
+            columns.column_names,
+            aggs.length,
+            columns.pivot_length,
+            row_path,
+            color_range
+        );
+
+        let s = column_to_series(ticks, ' ');   
+        series.push(s); 
+    } else {
+        let groups = {};
+
+        if (row_path) {
+            // remove all total rows
+            let clean_row_path = [];
+
+            for (let i = 0; i < row_path.length; i++) {
+                if (row_path[i].length === columns.pivot_length) {
+                    clean_row_path.push(row_path[i]);
+                }
+            }
+
+            row_path = clean_row_path;
+        }
+
+        for (let col of columns) {
+            let column_levels = col.name.split(COLUMN_SEPARATOR_STRING);
+            let group_name = column_levels.slice(0, column_levels.length - 1).join(", ") || " ";
+
+            if (groups[group_name] === undefined) {
+                groups[group_name] = [];
+            }
+
+            groups[group_name].push(col.data);
+        }
+
+        // FIXME: this is the heaviest loop
+        for (let name in groups) {
+            let ticks = make_tick.make_col(
+                groups[name], 
+                aggs, 
+                aggs.length,
+                columns.pivot_length,
+                row_path, 
+                color_range,
+            );
+
+            let s = column_to_series(ticks, name);
+            series.push(s);
+        }
+    }
+
+    return [series, {categories: make_tick.xaxis_clean.names}, color_range, {categories: make_tick.yaxis_clean.names}];
+}
+
+export async function make_xy_data(js, schema, columns, pivots, col_pivots, hidden) {
     let rows = new TreeAxisIterator(pivots.length, js);
     let rows2 = new RowIterator(rows, hidden);
     let series = [];
@@ -344,7 +489,7 @@ export function make_xy_data(js, schema, columns, pivots, col_pivots, hidden) {
         });
         for (let prop of cols) {
             let column_levels = prop.split(COLUMN_SEPARATOR_STRING);
-            let group_name = column_levels.slice(0, column_levels.length - 1).join(",") || " ";
+            let group_name = column_levels.slice(0, column_levels.length - 1).join(", ") || " ";
             if (prev === undefined) {
                 prev = group_name;
             }
@@ -368,8 +513,9 @@ export function make_xy_data(js, schema, columns, pivots, col_pivots, hidden) {
                 s.data.push(tick);
             }
         }
-    }
-    return [series, {categories: make_tick.xaxis_clean.names}, colorRange, {categories: make_tick.yaxis_clean.names}];
+    }   
+
+    return [series, {categories: make_tick.xaxis_clean.names}, colorRange, {categories: make_tick.yaxis_clean.names}];   
 }
 
 /******************************************************************************