From ae080ca97e6ac03648c113137e33cd814a35dc68 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Mon, 6 Apr 2020 01:50:50 -0400 Subject: [PATCH 1/2] Fix column resizing when column pivots are applied --- .../src/js/datagrid.js | 3 +- .../src/js/events.js | 35 +++++--- .../src/js/scroll_panel.js | 5 +- .../src/js/table.js | 74 ++++++++-------- .../src/js/tbody.js | 16 ++-- .../src/js/thead.js | 88 +++++++++++++------ .../src/js/tree_row_header.js | 2 +- .../src/js/view_model.js | 4 + .../src/less/material.less | 10 ++- .../test/results/linux.docker.json | 12 +-- 10 files changed, 152 insertions(+), 97 deletions(-) diff --git a/packages/perspective-viewer-datagrid/src/js/datagrid.js b/packages/perspective-viewer-datagrid/src/js/datagrid.js index 16360c2bc7..ddccb8a0e3 100644 --- a/packages/perspective-viewer-datagrid/src/js/datagrid.js +++ b/packages/perspective-viewer-datagrid/src/js/datagrid.js @@ -106,8 +106,7 @@ export class DatagridViewModel extends DatagridViewEventModel { this._virtual_scrolling_disabled = _render_element.hasAttribute("disable-virtual-datagrid"); this.create_shadow_dom(); this._column_sizes = {auto: {}, override: {}, indices: []}; - this.table_model = new DatagridTableViewModel(this._table_clip, this._column_sizes); - this._sticky_container.appendChild(this.table_model.table); + this.table_model = new DatagridTableViewModel(this._table_clip, this._column_sizes, this._sticky_container); if (!this.table_model) return; if (this._render_element) { if (this._render_element !== this.table_model.table.parentElement) { diff --git a/packages/perspective-viewer-datagrid/src/js/events.js b/packages/perspective-viewer-datagrid/src/js/events.js index 94de849deb..d7ff7017c5 100644 --- a/packages/perspective-viewer-datagrid/src/js/events.js +++ b/packages/perspective-viewer-datagrid/src/js/events.js @@ -82,12 +82,12 @@ export class DatagridViewEventModel extends DatagridVirtualTableViewModel { delete this._column_sizes.override[metadata.size_key]; delete this._column_sizes.auto[metadata.size_key]; delete this._column_sizes.indices[metadata.cidx]; - element.style.minWidth = "0"; - element.style.maxWidth = "none"; + element.style.minWidth = ""; + element.style.maxWidth = ""; for (const row of this.table_model.body.cells) { const td = row[metadata.cidx]; - td.style.minWidth = "0"; - td.style.maxWidth = "none"; + td.style.minWidth = ""; + td.style.maxWidth = ""; td.classList.remove("pd-cell-clip"); } await this.draw({invalid_viewport: true}); @@ -103,6 +103,10 @@ export class DatagridViewEventModel extends DatagridVirtualTableViewModel { * @memberof DatagridVirtualTableViewModel */ async _on_click(event) { + if (event.button !== 0) { + return; + } + let element = event.target; while (element.tagName !== "TD" && element.tagName !== "TH") { element = element.parentElement; @@ -110,6 +114,7 @@ export class DatagridViewEventModel extends DatagridVirtualTableViewModel { return; } } + const is_button = event.target.classList.contains("pd-row-header-icon"); const is_resize = event.target.classList.contains("pd-column-resize"); const metadata = METADATA_MAP.get(element); @@ -134,7 +139,8 @@ export class DatagridViewEventModel extends DatagridVirtualTableViewModel { */ _on_resize_column(event, element, metadata) { const start = event.pageX; - const width = element.offsetWidth; + element = this.table_model.header.get_column_header(metadata.vcidx); + const width = this._column_sizes.indices[metadata.cidx]; const move = event => this._on_resize_column_move(event, element, start, width, metadata); const up = async () => { document.removeEventListener("mousemove", move); @@ -163,17 +169,20 @@ export class DatagridViewEventModel extends DatagridVirtualTableViewModel { const diff = event.pageX - start; const override_width = width + diff; this._column_sizes.override[metadata.size_key] = override_width; - th.style.minWidth = override_width + "px"; - th.style.maxWidth = override_width + "px"; - const auto_width = this._column_sizes.auto[metadata.size_key]; - for (const row of this.table_model.body.cells) { - const td = row[metadata.cidx]; - td.style.maxWidth = td.style.minWidth = override_width + "px"; - td.classList.toggle("pd-cell-clip", auto_width > override_width); - } + // If the column is smaller, new columns may need to be fetched, so + // redraw, else just update the DOM widths as if redrawn. if (diff < 0) { await this.draw({invalid_viewport: true, preserve_width: true}); + } else { + th.style.minWidth = override_width + "px"; + th.style.maxWidth = override_width + "px"; + const auto_width = this._column_sizes.auto[metadata.size_key]; + for (const row of this.table_model.body.cells) { + const td = row[metadata.vcidx]; + td.style.maxWidth = td.style.minWidth = override_width + "px"; + td.classList.toggle("pd-cell-clip", auto_width > override_width); + } } } diff --git a/packages/perspective-viewer-datagrid/src/js/scroll_panel.js b/packages/perspective-viewer-datagrid/src/js/scroll_panel.js index aa2779795f..8cccad4cef 100644 --- a/packages/perspective-viewer-datagrid/src/js/scroll_panel.js +++ b/packages/perspective-viewer-datagrid/src/js/scroll_panel.js @@ -212,7 +212,7 @@ export class DatagridVirtualTableViewModel extends HTMLElement { let max_scroll_column = this._view_cache.column_paths.length; while (width < this._container_size.width && max_scroll_column >= 0) { max_scroll_column--; - width += this._column_sizes.indices[max_scroll_column] || 100; + width += this._column_sizes.indices[max_scroll_column] || 60; } const psp_offset = this._view_cache.config.row_pivots.length > 0; return Math.min(this._view_cache.column_paths.length - (psp_offset ? 2 : 1), max_scroll_column + (psp_offset ? 0 : 1)); @@ -301,7 +301,7 @@ export class DatagridVirtualTableViewModel extends HTMLElement { let cidx = 0, virtual_width = 0; while (cidx < max_scroll_column) { - virtual_width += this._column_sizes.indices[cidx] || 100; + virtual_width += this._column_sizes.indices[cidx] || 60; cidx++; } const panel_width = this._container_size.width + virtual_width; @@ -318,7 +318,6 @@ export class DatagridVirtualTableViewModel extends HTMLElement { */ _update_virtual_panel_height(nrows) { const {row_height = 19} = this._column_sizes; - //const header_levels = this._view_cache.config.column_pivots.length + 1; const virtual_panel_px_size = Math.min(BROWSER_MAX_HEIGHT, nrows * row_height); this._virtual_panel.style.height = `${virtual_panel_px_size}px`; } diff --git a/packages/perspective-viewer-datagrid/src/js/table.js b/packages/perspective-viewer-datagrid/src/js/table.js index ebf75f8192..35779a6602 100644 --- a/packages/perspective-viewer-datagrid/src/js/table.js +++ b/packages/perspective-viewer-datagrid/src/js/table.js @@ -9,7 +9,7 @@ import {DatagridHeaderViewModel} from "./thead"; import {DatagridBodyViewModel} from "./tbody"; -import {column_path_2_type} from "./utils"; +import {column_path_2_type, html} from "./utils"; /** * view model. In order to handle unknown column width when `draw()` @@ -20,16 +20,15 @@ import {column_path_2_type} from "./utils"; * @class DatagridTableViewModel */ export class DatagridTableViewModel { - constructor(table_clip, column_sizes) { - const table = document.createElement("table"); - table.setAttribute("cellspacing", 0); - - const thead = document.createElement("thead"); - table.appendChild(thead); - - const tbody = document.createElement("tbody"); - table.appendChild(tbody); - + constructor(table_clip, column_sizes, element) { + element.innerHTML = html` +
+ + +
+ `; + const [table] = element.children; + const [thead, tbody] = table.children; this.table = table; this._column_sizes = column_sizes; this.header = new DatagridHeaderViewModel(column_sizes, table_clip, thead); @@ -61,31 +60,33 @@ export class DatagridTableViewModel { } } - async draw(container_size, view_cache, selected_id, is_resize, viewport) { + async draw(container_size, view_cache, selected_id, preserve_width, viewport) { const {width: container_width, height: container_height} = container_size; const {view, config, column_paths, schema, table_schema} = view_cache; const visible_columns = column_paths.slice(viewport.start_col); const columns_data = await view.to_columns(viewport); - const {start_col: sidx} = viewport; + const {start_row: ridx_offset, start_col: cidx_offset} = viewport; + const depth = config.row_pivots.length; const id_column = columns_data["__ID__"]; - let row_height = this._column_sizes.row_height; + const view_state = {viewport_width: 0, selected_id, depth, ridx_offset, cidx_offset, row_height: this._column_sizes.row_height}; + let cont_body, - cont_head, cidx = 0, - width = 0, last_cells = []; if (column_paths[0] === "__ROW_PATH__") { - const alias = config.row_pivots.join(","); - const types = config.row_pivots.map(x => table_schema[x]); - cont_head = this.header.draw(config, alias, "", types); - cont_body = this.body.draw(container_height, alias, 0, columns_data["__ROW_PATH__"], id_column, selected_id, types, config.row_pivots.length, viewport.start_row, 0, row_height); - selected_id = false; - if (!is_resize) { + const column_name = config.row_pivots.join(","); + const type = config.row_pivots.map(x => table_schema[x]); + const column_data = columns_data["__ROW_PATH__"]; + const column_state = {column_name, cidx: 0, column_data, id_column, type}; + const cont_head = this.header.draw(config, column_name, "", type, 0); + cont_body = this.body.draw(container_height, column_state, {...view_state, cidx_offset: 0}); + view_state.selected_id = false; + view_state.viewport_width += this._column_sizes.indices[0] || cont_body.td?.offsetWidth || cont_head.th.offsetWidth; + view_state.row_height = view_state.row_height || cont_body.row_height; + cidx++; + if (!preserve_width) { last_cells.push([cont_body.td || cont_head.th, cont_body.metadata || cont_head.metadata]); } - row_height = row_height || cont_body.row_height; - width += this._column_sizes.indices[0] || cont_body.td?.offsetWidth || cont_head.th.offsetWidth; - cidx++; } try { @@ -103,19 +104,20 @@ export class DatagridTableViewModel { columns_data[column_name] = new_col[column_name]; } - const column_data = columns_data[column_name]; const type = column_path_2_type(schema, column_name); - cont_head = this.header.draw(config, undefined, column_name, type, config.sort, cont_head); - cont_body = this.body.draw(container_height, column_name, cidx, column_data, id_column, selected_id, type, undefined, viewport.start_row, sidx, cont_body?.row_height); - selected_id = false; - width += this._column_sizes.indices[cidx + sidx] || cont_body.td?.offsetWidth || cont_head.th.offsetWidth; - if (!is_resize) { + const column_data = columns_data[column_name]; + const column_state = {column_name, cidx, column_data, id_column, type}; + const cont_head = this.header.draw(config, undefined, column_name, type, cidx + cidx_offset); + cont_body = this.body.draw(container_height, column_state, view_state); + view_state.selected_id = false; + view_state.viewport_width += this._column_sizes.indices[cidx + cidx_offset] || cont_body.td?.offsetWidth || cont_head.th.offsetWidth; + view_state.row_height = view_state.row_height || cont_body.row_height; + cidx++; + if (!preserve_width) { last_cells.push([cont_body.td || cont_head.th, cont_body.metadata || cont_head.metadata]); } - row_height = row_height || cont_body.row_height; - cidx++; - if (width > container_width) { + if (view_state.viewport_width > container_width) { break; } } @@ -123,9 +125,7 @@ export class DatagridTableViewModel { return last_cells; } finally { this.body.clean({ridx: cont_body?.ridx || 0, cidx}); - if (cont_head) { - this.header.clean(cont_head); - } + this.header.clean(); } } } diff --git a/packages/perspective-viewer-datagrid/src/js/tbody.js b/packages/perspective-viewer-datagrid/src/js/tbody.js index 42fbf088f5..8828f5199f 100644 --- a/packages/perspective-viewer-datagrid/src/js/tbody.js +++ b/packages/perspective-viewer-datagrid/src/js/tbody.js @@ -16,7 +16,7 @@ import {ViewModel} from "./view_model"; * @class DatagridBodyViewModel */ export class DatagridBodyViewModel extends ViewModel { - _draw_td(ridx, cidx, column, val, id, selected_id, type, depth, is_open, ridx_offset, cidx_offset) { + _draw_td(ridx, val, id, is_open, {cidx, column_name, type}, {selected_id, depth, ridx_offset, cidx_offset}) { const {tr, row_container} = this._get_row(ridx); if (selected_id !== false) { const is_selected = isEqual(id, selected_id); @@ -29,8 +29,8 @@ export class DatagridBodyViewModel extends ViewModel { metadata.id = id; metadata.cidx = cidx + cidx_offset; metadata.type = type; - metadata.column = column; - metadata.size_key = `${column}|${type}`; + metadata.column = column_name; + metadata.size_key = `${column_name}|${type}`; metadata.ridx = ridx + ridx_offset; td.className = `pd-${type}`; const override_width = this._column_sizes.override[metadata.size_key]; @@ -41,8 +41,8 @@ export class DatagridBodyViewModel extends ViewModel { td.style.maxWidth = override_width + "px"; } else { td.classList.remove("pd-cell-clip"); - td.style.minWidth = "0"; - td.style.maxWidth = "none"; + td.style.minWidth = ""; + td.style.maxWidth = ""; } const formatter = this._format(type); if (val === undefined || val === null) { @@ -62,13 +62,15 @@ export class DatagridBodyViewModel extends ViewModel { return {td, metadata}; } - draw(container_height, column_name, cidx, column_data, id_column, selected_id, type, depth, ridx_offset, cidx_offset, row_height) { + draw(container_height, column_state, view_state) { + const {cidx, column_data, id_column} = column_state; + let {row_height} = view_state; let ridx = 0; let td, metadata; for (const val of column_data) { const next = column_data[ridx + 1]; const id = id_column?.[ridx]; - const obj = this._draw_td(ridx++, cidx, column_name, val, id, selected_id, type, depth, next?.length > val?.length, ridx_offset, cidx_offset); + const obj = this._draw_td(ridx++, val, id, next?.length > val?.length, column_state, view_state); td = obj.td; metadata = obj.metadata; row_height = row_height || td.offsetHeight; diff --git a/packages/perspective-viewer-datagrid/src/js/thead.js b/packages/perspective-viewer-datagrid/src/js/thead.js index 08c38d7d38..76b6c04cf6 100644 --- a/packages/perspective-viewer-datagrid/src/js/thead.js +++ b/packages/perspective-viewer-datagrid/src/js/thead.js @@ -9,6 +9,7 @@ import {ViewModel} from "./view_model"; import {ICON_MAP} from "./constants"; +import {html} from "./utils.js"; /** * view model. This model accumulates state in the form of @@ -21,20 +22,31 @@ export class DatagridHeaderViewModel extends ViewModel { _draw_group_th(offset_cache, d, column, sort_dir) { const {tr, row_container} = this._get_row(d); const th = this._get_cell("th", row_container, offset_cache[d], tr); - const cidx = offset_cache[d]; offset_cache[d] += 1; th.className = ""; th.removeAttribute("colspan"); th.style.minWidth = "0"; - const metadata = this._get_or_create_metadata(th); - metadata.cidx = cidx; - if (sort_dir?.length === 0) { - th.innerHTML = "" + column + ``; + th.innerHTML = html` + ${column} + + `; } else { - const sort_txt = sort_dir?.map(x => ICON_MAP[x]); - th.innerHTML = "" + column + `${sort_txt}`; + const sort_txt = sort_dir + ?.map(x => { + const icon = ICON_MAP[x]; + return html` + ${icon} + `; + }) + .join(""); + th.innerHTML = html` + ${column} + ${sort_txt} + + `; } + return th; } @@ -56,8 +68,8 @@ export class DatagridHeaderViewModel extends ViewModel { metadata.column_name = column_name; metadata.column_type = type; metadata.is_column_header = false; - metadata.size_key = `${column}|${type}`; th.className = ""; + return metadata; } _draw_th(column, column_name, type, th) { @@ -76,38 +88,58 @@ export class DatagridHeaderViewModel extends ViewModel { th.style.maxWidth = override_width + "px"; } else if (auto_width) { th.classList.remove("pd-cell-clip"); - th.style.maxWidth = "none"; + th.style.maxWidth = ""; th.style.minWidth = auto_width + "px"; } + return metadata; } - draw(config, alias, column, type, sort, {group_header_cache = [], offset_cache = []} = {}) { + get_column_header(cidx) { + const {tr, row_container} = this._get_row(this.rows.length - 1); + return this._get_cell("th", row_container, cidx, tr); + } + + _group_header_cache = []; + _offset_cache = []; + + draw(config, alias, column_path, type, cidx) { const header_levels = config.column_pivots.length + 1; - let parts = column.split?.("|"); + let parts = column_path.split?.("|"); let th, column_name, is_new_group = false; for (let d = 0; d < header_levels; d++) { column_name = parts[d] ? parts[d] : ""; - group_header_cache[d] = group_header_cache[d] || []; - offset_cache[d] = offset_cache[d] || 0; + this._offset_cache[d] = this._offset_cache[d] || 0; if (d < header_levels - 1) { - if (group_header_cache[d][0] === column_name) { - th = group_header_cache[d][1]; - th.setAttribute("colspan", (parseInt(th.getAttribute("colspan")) || 1) + 1); + if (this._group_header_cache?.[d]?.[0]?.column_name === column_name) { + th = this._group_header_cache[d][1]; + this._group_header_cache[d][2] += 1; + th.setAttribute("colspan", this._group_header_cache[d][2]); } else { - th = this._draw_group_th(offset_cache, d, column_name, []); - this._draw_group(column, column_name, type, th); - group_header_cache[d] = [column_name, th]; + th = this._draw_group_th(this._offset_cache, d, column_name, []); + const metadata = this._draw_group(column_path, column_name, type, th); + this._group_header_cache[d] = [metadata, th, 1]; is_new_group = true; } } else { if (is_new_group) { - this._redraw_previous(offset_cache, d); + this._redraw_previous(this._offset_cache, d); + } + const vcidx = this._offset_cache[d]; + const sort_dir = config.sort?.filter(x => x[0] === column_name).map(x => x[1]); + th = this._draw_group_th(this._offset_cache, d, column_name, sort_dir); + + // Update the group header's metadata such that each group + // header has the same metadata coordinates of its rightmost column. + const metadata = this._draw_th(alias || column_path, column_name, type, th); + metadata.vcidx = vcidx; + metadata.cidx = cidx; + for (const [group_meta] of this._group_header_cache) { + group_meta.cidx = cidx; + group_meta.vcidx = vcidx; + group_meta.size_key = metadata.size_key; } - const sort_dir = sort?.filter(x => x[0] === column_name).map(x => x[1]); - th = this._draw_group_th(offset_cache, d, column_name, sort_dir); - this._draw_th(alias || column, column_name, type, th); } } @@ -115,11 +147,13 @@ export class DatagridHeaderViewModel extends ViewModel { th.classList.add("pd-group-header"); } const metadata = this._get_or_create_metadata(th); - this._clean_rows(offset_cache.length); - return {group_header_cache, offset_cache, th, metadata}; + this._clean_rows(this._offset_cache.length); + return {th, metadata}; } - clean({offset_cache}) { - this._clean_columns(offset_cache); + clean() { + this._clean_columns(this._offset_cache); + this._offset_cache = []; + this._group_header_cache = []; } } diff --git a/packages/perspective-viewer-datagrid/src/js/tree_row_header.js b/packages/perspective-viewer-datagrid/src/js/tree_row_header.js index 01cfa4dde0..7fe743fcfe 100644 --- a/packages/perspective-viewer-datagrid/src/js/tree_row_header.js +++ b/packages/perspective-viewer-datagrid/src/js/tree_row_header.js @@ -36,7 +36,7 @@ function _tree_header_levels(path, is_open, is_leaf) { export function tree_header(td, path, types, is_leaf, is_open) { const type = types[path.length - 1]; - const name = path[path.length - 1] || "TOTAL"; + const name = path.length === 0 ? "TOTAL" : path[path.length - 1]; const header_classes = _tree_header_classes.call(this, name, type, is_leaf); const tree_levels = _tree_header_levels(path, is_open, is_leaf); const header_text = this._format_text(type)(name); diff --git a/packages/perspective-viewer-datagrid/src/js/view_model.js b/packages/perspective-viewer-datagrid/src/js/view_model.js index 344467751b..eecd79ff96 100644 --- a/packages/perspective-viewer-datagrid/src/js/view_model.js +++ b/packages/perspective-viewer-datagrid/src/js/view_model.js @@ -31,6 +31,10 @@ export class ViewModel { return this.cells.length; } + _set_metadata(td, metadata) { + METADATA_MAP.set(td, metadata); + } + _get_or_create_metadata(td) { if (METADATA_MAP.has(td)) { return METADATA_MAP.get(td); diff --git a/packages/perspective-viewer-datagrid/src/less/material.less b/packages/perspective-viewer-datagrid/src/less/material.less index 6748749176..9b639170b2 100644 --- a/packages/perspective-viewer-datagrid/src/less/material.less +++ b/packages/perspective-viewer-datagrid/src/less/material.less @@ -9,7 +9,7 @@ @row-height: 19px; -perspective-viewer div { +perspective-viewer > div { position: absolute; overflow: hidden; top: 12px; @@ -25,6 +25,12 @@ perspective-viewer, :host { text-align: center; } + // Header groups should overflow and not contribute to auto-sizing. + thead tr:not(:last-child) th { + overflow: hidden; + max-width: 0px; + } + thead tr:last-child .pd-float, tbody .pd-float { text-align: right; @@ -138,6 +144,8 @@ perspective-viewer, :host { span.pd-column-header-icon { font-size: 10px; padding-left: 3px; + display: inline-block; + width: 10px; font-family: "Material Icons"; } diff --git a/packages/perspective-viewer-datagrid/test/results/linux.docker.json b/packages/perspective-viewer-datagrid/test/results/linux.docker.json index 971d0fad77..eedda910ac 100644 --- a/packages/perspective-viewer-datagrid/test/results/linux.docker.json +++ b/packages/perspective-viewer-datagrid/test/results/linux.docker.json @@ -1,16 +1,16 @@ { "superstore_replaces_all_rows_": "7f95361aa2a268a8ee842c4632513578", - "__GIT_COMMIT__": "6f77fcc6c0aeea74aa445ded6969e7ee233c3a8b", + "__GIT_COMMIT__": "de9b20ab4e3651612156b5bdd54c745e34e1a798", "empty_empty_grids_do_not_explode": "deb15d155a84145e67da078c803e3eec", "regressions_Updates_should_not_render_an_extra_row_for_column_only_views": "676bcaac0bc0d63a2bfa3bd254443b0c", "regressions_Updates_regular_updates": "86561ee0ebe91f656f56941e7ff1cf35", "regressions_Updates_saving_a_computed_column_does_not_interrupt_update_rendering": "17f79bc23ea6ea8b54aea4ee367aedd0", "superstore_shows_a_grid_without_any_settings_applied_": "4fb387275da7c37f5aaf25af52d9e8d5", - "superstore_pivots_by_a_row_": "ca590844917a4f75623519654b230155", - "superstore_pivots_by_two_rows_": "0a72fd22ebbfa4cd1222650fc215371e", + "superstore_pivots_by_a_row_": "4e0997491754ffd9d4886154be4a8638", + "superstore_pivots_by_two_rows_": "5d4895fb6286057cd676f17284c31ba7", "superstore_pivots_by_a_column_": "ba5ffdc9676ffb53c22f1f5f40bd5c3d", - "superstore_pivots_by_a_row_and_a_column_": "af58a85fe204cb96e7200d139c552311", - "superstore_pivots_by_two_rows_and_two_columns_": "46a86389a587977f005c44455b2511b6", + "superstore_pivots_by_a_row_and_a_column_": "ab8a4999f675788a5403f399832dd063", + "superstore_pivots_by_two_rows_and_two_columns_": "e0780ec8476857bcd0c0986b47c04332", "superstore_sorts_by_a_hidden_column_": "f0beae68b6506d75f3126db285d03290", "superstore_sorts_by_a_numeric_column_": "b598f9af4aa390f51435e8a8f1fc3020", "superstore_filters_by_a_numeric_column_": "045c98d9feb94929e228310b755fe5a2", @@ -18,6 +18,6 @@ "superstore_highlights_invalid_filter_": "311401a388e8cf89d0f9a5c5a1eb9146", "superstore_sorts_by_an_alpha_column_": "bc2c035e1c21eab541ae2e1119289913", "superstore_displays_visible_columns_": "f7ab6c498375d5b98b47ef582105ea25", - "superstore_resets_viewable_area_when_the_logical_size_expands_": "f54da69699014b26ac63c2dea6e6ff94", + "superstore_resets_viewable_area_when_the_logical_size_expands_": "e2eafa3c5eedb70bc538e759234d5122", "superstore_resets_viewable_area_when_the_physical_size_expands_": "4fb387275da7c37f5aaf25af52d9e8d5" } \ No newline at end of file From d71c65efea6c8b348bd2e7a2b24a9a517ef86eb5 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Mon, 6 Apr 2020 01:52:41 -0400 Subject: [PATCH 2/2] Updates docs and examples --- .../2018-10-08-nyc-citibike-analysis-1.md | 360 ------------------ docs/pages/en/index.js | 35 +- examples/simple/superstore-custom-grid.html | 28 +- 3 files changed, 33 insertions(+), 390 deletions(-) delete mode 100644 docs/blog/2018-10-08-nyc-citibike-analysis-1.md diff --git a/docs/blog/2018-10-08-nyc-citibike-analysis-1.md b/docs/blog/2018-10-08-nyc-citibike-analysis-1.md deleted file mode 100644 index 2f4b59bc25..0000000000 --- a/docs/blog/2018-10-08-nyc-citibike-analysis-1.md +++ /dev/null @@ -1,360 +0,0 @@ ---- -title: NYC Citibike Analytics in Real-Time, Part 1 -author: Andrew Stein -authorURL: http://github.com/texodus -authorImageURL: https://avatars3.githubusercontent.com/u/60666?s=400&v=4 ---- - -There are many well-known perks to living in New York City - the world class -restaurants, museums, two mediocre NFL teams (taken together, that's one pretty -good NFL team!). Seldom mentioned, though, is the city's world-class Open Data! - -In this post, we'll be taking advantage of NYC's abundance of Open Data and -the Perspective library to build some real-time analytics visualizations that -seek to answer one burning question: Where are all of New York's -[Citibikes](https://www.citibikenyc.com/)? - - - - - - -Here's what we'll be building, a real-time map of NYC Citibike stations colored -by the number of bikes availble at each. There is no server involved, all data -subscriptions, analytics logics, and visualization is done entirely in your -browser with Perspective. This examples is live - go ahead and -try it out! - -
-
- - - - -
- -## Setup - -We're going to need a few libraries, which perspective-heads will no doubt -already be quite familiar with (Yes, perspective-heads is a real term that real -people other than me use): - -```html - - - - -``` - -You're also going to need a relatively up-to-date browser, as this example makes use -of several [ES2017 features](https://developer.mozilla.org/en-US/docs/Web/JavaScript/New_in_JavaScript/ECMAScript_Next_support_in_Mozilla) -like `async`/`await`. - -## Getting the data - -The [NYC Citibike program](https://www.citibikenyc.com/) gratiously provides -access to their [real-time station data](https://www.citibikenyc.com/system-data) -via a pretty stand REST/JSON API, and they've even done us the favor of properly -configuring [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS)! -This means we should have no problem fetching and reading this data directly in -a Web Browser, no server required! From skimming the -[General Bikeshare Feed Specification](https://github.com/NABSA/gbfs/blob/master/gbfs.md), -it looks like the feeds we're interested in are: - -* [station_information.json](https://gbfs.citibikenyc.com/gbfs/en/station_information.json) Mostly static list of all stations, their capacities and locations. Required of systems utilizing docks. - -* [station_status.json](https://gbfs.citibikenyc.com/gbfs/en/station_status.json) Number of available bikes and docks at each station and station availability. Required of systems utilizing docks. - -First thing we're going to want is some rote-standard code to fetch these feeds -via [`XMLHttpRequest`](https://developer.mozilla.org/en-US/docs/Web/API/XMLHttpRequest). -As we're going to be making lots of Asynchronous calls in this study, it will be convenient -to write a quick wrapper function `get()` which wraps up the common boilerplate -for `XMLHttpRequest` in a `Promise`; this will allow us to later call `get()` -with simpler `async`/`await` semantics, even in deeply nested asynchronous code, -greatly simplifying ... well everything, really. This will also compose nicely -with our `perspective` API calls we'll be making, as they also exclusively -return `Promise`s. - -```javascript -function get(url) { - return new Promise(resolve => { - const xhr = new XMLHttpRequest(); - xhr.open("GET", url, true); - xhr.responseType = "json"; - xhr.onload = () => resolve(xhr.response); - xhr.send(null); - }); -} -``` - -Of course, we'd like a function for getting feeds specifically, which -share some common strucutre we can encapsulate in a function, so let's go ahead and make -a specialized function `get_feed()` just for this purpose. It will need to take -a `feedname` to parameterize the URL, and a `callback` parameter for when we intend -to have the feed polled; when this parameter is present, we will -continuously `get()` the feed at some interval and invoke `callback` with the -results, rather than `return` them. - -From the specification, we know that the data we're interested in is in the -`stations` field of the object in the `data` field. We're also conveniently -provided a TTL value at the top level of each object, which we can use to -calculate a poll frequency for the feed, as we'll want to update it as often -as it is available. - -```javascript -async function get_feed(feedname, callback) { - const url = `https://gbfs.citibikenyc.com/gbfs/en/${feedname}.json`; - const {data: {stations}, ttl} = await get(url); - if (typeof callback === "function") { - callback(stations); - setTimeout(() => get_feed(feedname, callback), ttl * 1000); - } else { - return stations; - } -} -``` - -## Inferring a feed's schema - -From looking at the schemas for these feeds, it looks like we're going to want -to join these two feeds on `station_id`, which means we're going to need to -give Perspective a schema so it knows what column data to expect, as each -row update will only have some fields depending on which feed it is coming from. -Unfortunately, the schemas presented in the specification are not JSON, nor do -they provide their types (both requirements of Perspective); more -unfortunately, I have the patience of a small child, so spending 10 minutes -manually writing a schema is simply out of the question. Time to give up! - -... - -Just kidding - we can trivially utilize Perspective's own column inference to do -this for us! All we have to do is load the JSON data into a Perspective -`table()`, then call its `schema()` method. First thing's first, though - -Perspective is designed to run in a -[Web Worker](https://developer.mozilla.org/en-US/docs/Web/API/Web_Workers_API), -so we'll need to create an instance of one that we can then create `table()`s -in. - -```javascript -const worker = perspective.worker(); -``` - -Now the engine is instantiated and we can call methods on it to create & -transform data in a separate Web Worker process. Let's now use it to make a -function `get_schema()` for inferring schemas from feeds, since we'll -want to re-use this logic for both of our feeds. - -```javascript -async function get_schema(feed) { - const table = worker.table(feed); - const schema = await table.schema(); - table.delete(); - return schema; -} -``` - -The call to `table.delete()` is important! Perspective uses Web Assembly to -achieve its barnstorming performance, and unfortunately as of today, -[Web Assembly does not support VM Garbage Collection](https://github.com/WebAssembly/design/issues/1079); -without this call then, the `table()` instantiated in this method would -leak when it falls out of scope, leaving the underlying table memory -allocated forever. - -With `get_schema()` in hand, it's easy to create the ultimate merged schema we -seek. - -```javascript -async function merge_schemas(feeds) { - const schemas = await Promise.all(feeds.map(get_schema)); - return Object.assign({}, ...schemas); -} -``` - -Testing this out in the Chrome Javascript Console is easy! The Console supports -`await` directly (unlike Javascript, where this keyword cannot exist outside an -`async` function block), so all we need to do is copy/paste our function -invocation, exaclty as we'll use it in our finished script. - -```javascript -const feednames = ["station_status", "station_information"]; -const feeds = await Promise.all(feednames.map(get_feed)); -const schema = await merge_schemas(feeds); -``` - -This returns exactly what we were looking for - the merged schema for -the `station_status` and `station_information` feeds, with types as inferred by -the Perspective inferrence algorithm directly from the data itself. - -```json -{ - "station_id": "float", - "num_bikes_available": "integer", - "num_ebikes_available": "float", - "num_bikes_disabled": "integer", - "num_docks_available": "integer", - "num_docks_disabled": "float", - "is_installed": "integer", - "is_renting": "integer", - "is_returning": "integer", - "last_reported": "float", - "eightd_has_available_keys": "boolean", - "eightd_active_station_services": "float", - "name": "string", - "short_name": "float", - "lat": "float", - "lon": "float", - "region_id": "integer", - "rental_methods": "float", - "capacity": "integer", - "rental_url": "string", - "eightd_has_key_dispenser": "boolean", - "eightd_station_services": "float", - "has_kiosk": "boolean" -} -``` - -## Creating a `table` by joining feeds with an index - -Now that we have our schema, and a convenient `get_feed()` method for -fetching our feeds, the next step is to load the feeds we've fetched into -Perspective. The basic data primitive in Perspective is the `table()` object, -and creating one from a schema is easy - we just pass the `schema` object -we created above to the `table()` constructor on the `worker`, just like we did -for the table we used to infer the constituent schemas in our -`get_schema()` function. - -```javascript -const table = worker.table(schema, {index: "station_id"}); -for (let feed of feeds) { - table.update(feed); -} -``` - -Ah, but wait - this is just the state of Citibike at the moment of page load! -What about the real-time support we were promised in the title, and diligently -prepared for in our `get_feed()` function? Turns out, that diligent -preparation indeed makes this pretty trivial - we just dispatch our `callback` -parameter to our `table.update()` method. The index field of the -options object passed as the second parameter to -`table()` makes sure that each updated row overwrites existing rows -joined by `station_id`, and Perspective's support for -[partial updates](https://jpmorganchase.github.io/perspective/docs/usage.html#partial-i-row-i-updates-via-undefined) -means only the fields actually defined in the `station_status` feed are updated, -while the `station_information` fields are left alone. Without this property, -rows added via the `table.update()` method would simply append, and we'd -need to join the `station_status` fields with the `station_information` fields -via a much more complex `` configuration. - -```javascript -get_feed("station_status", table.update); -``` - -## Loading a `table` in a `` - -Now that we have a `table()` with our Citibike subscription all wired up, the -last thing to do is view it! For this, we need a ``. We -could create one via Javascript through the standard DOM APIs such as -`document.createElement()`, but one of the nice features of -[Web Components]() is that they can be used declaritively directly in your -application's HTML without any special pre-processing. - -```html - -``` - - Next, we capture references to all ``s on our page through - the standard DOM APIs, such that we may call their side-effecting - methods such as `load()`, which we'll use to bind our Citibike `table()`. - We can freely share this `table()` among as many ``s as - we need, so we'll just iterate through all of them - even our `update()` calls - will be shared, causing all bound ``s to re-render when - the `table()` changes. - -```javascript -for (viewer of document.getElementsByTagName("perspective-viewer")) { - viewer.load(table); -} -``` - -And just like that, we have our live, joined dataset loaded in a fully interactive -``, ready to slice & dice! (This isn't a screenshot, its -a live perspective viewer with real data, so go ahead and play around!) - -
-
- - -
- -## Where are the Citibikes? - -While we can see our data now and transform/visualize it to our heart's content, -the default view on initial load is not super interesting, so let's kick it -up a notch! Wait, no, that's probably going to get us in trouble ... er, -let's put some Perspective on it! - -Regarding our instigating question, there are a number of good potential -visualizations that may help us understand the answer, including -an incredibly obvious and easy one - a list of Citibike stations ordered by -the `"num_bikes_available"`. We can make this the default view on a -`` easily in HTML, through its -[Attribute API](https://jpmorganchase.github.io/perspective/docs/usage.html#setting-reading-viewer-configuration-via-attributes). - -In this case, we'll want to set the `columns` attribute to our column set, -`["num_bikes_available"]`, and the `sort` attribute to a list of sort -descriptors `[["num_bikes_available","desc"]]`, or in other words, that we -want the list arranged with the values of the `"num_bikes_available"` column -descending. We also provide the `"name"` field to the `row-pivots` property, -though there will only be one row in the Citibike data per `"name"`. While we -could have accomplished something similar by leaving this view un-pivoted and -instead added `"name"` to the `columns` attribute, making it a `row-pivot` -gives us a pretty tree axis indicating visually that `"name"` is the intended -axis of the view, as well as creating a `TOTAL` aggregate row showing -the sum total of all `"num_bikes_available"`. - -```html - - - -``` - -Because our `table()` are shared, we can easily make another view on the -same data by just configuring another `` - how about a -heat map of station availability? The schema conveniently has `lon` and `lat` -fields, which one surmise stand for Longitude and Latitude. -`` uses hard-coded mappings for displaying a `view()` -configuration, and for Scatter Charts, it maps selected -`columns` to "X Axis", "Y Axis", "Color" and "Size" in that order. Sure enough, -setting columns to `lon` and `lat`, plus an interesting field like -`num_bikes_available`, as well as the `view` attribute to `xy_scatter`, -should give us something that looks roughly like the classic profile of the Five -Bouroughs, colored by Citibike availability: - -```html - - - -``` - -Finally, at long last, we have our live & ticking Citibike Analytics Dashboard: - -
-
- - - - -
- -## Appendix - the Entire Application - -For your covenience, the entire Javascript application at once is available -[in the `examples/` directory of the Perspective github repository](https://github.com/jpmorganchase/perspective/blob/master/examples/simple/citibike.html), as well as -[in a JSFiddle](https://jsfiddle.net/texodus/m2rwz690) \ No newline at end of file diff --git a/docs/pages/en/index.js b/docs/pages/en/index.js index bba73d0c9c..769f80549b 100755 --- a/docs/pages/en/index.js +++ b/docs/pages/en/index.js @@ -82,12 +82,12 @@ class HomeSplash extends React.Component { - - - - - - + + + + + + @@ -183,16 +183,19 @@ const FeatureCallout = props => ( const DESCRIPTION_TEXT = ` # What is Perspective? -Originally developed for J.P. Morgan's trading business, Perspective is -an interactive visualization component for large, real-time -datasets. Use it to build reports, dashboards, notebooks and applications. -Perspective comes with: -* A fast, memory efficient streaming query engine, written in C++ and compiled to [WebAssembly](https://webassembly.org/), with read/write/stream support for [Apache Arrow](). -* A framework-agnostic query configuration UI component, based on [Web Components](https://www.webcomponents.org/), and a WebWorker and/or WebSocket data engine host for stable interactivity at high frequency. -* A customizable HTML Data Grid plugin, and a Charts plugin built on [D3FC](https://d3fc.io/). -* Integration with Jupyterlab, both natively in a Python kernel, and as a notebook Widget. -* Cross-language streaming & virtualization to the browser via [Apache Arrow](https://arrow.apache.org/). -* Runtimes for the browser, Python, and Node.js. +Perspective is an interactive visualization component for large, real-time +datasets. Originally developed for J.P. Morgan's trading business, Perspective +makes it simple to build real-time & user configurable analytics entirely in the +browser, or in concert with Python and/or +[Jupyterlab](https://jupyterlab.readthedocs.io/en/stable/). +Use it to create reports, dashboards, notebooks and applications, with static +data or streaming updates via [Apache Arrow](https://arrow.apache.org/). +- A fast, memory efficient streaming query engine, written in C++ and compiled to [WebAssembly](https://webassembly.org/), with read/write/stream support for [Apache Arrow](). +- A framework-agnostic query configuration UI component, based on [Web Components](https://www.webcomponents.org/), and a WebWorker and/or WebSocket data engine host for stable interactivity at high frequency. +- A customizable HTML Data Grid plugin, and a Chart plugin built on [D3FC](https://d3fc.io/). +- Integration with [Jupyterlab](https://jupyterlab.readthedocs.io/en/stable/), both natively in a Python kernel, and as a notebook Widget. +- Cross-language streaming and/or virtualization to the browser via [Apache Arrow](https://arrow.apache.org/). +- Runtimes for the browser, Python, and Node.js. `; const Description = props => ( diff --git a/examples/simple/superstore-custom-grid.html b/examples/simple/superstore-custom-grid.html index 51ce2042c8..b1d89046c8 100644 --- a/examples/simple/superstore-custom-grid.html +++ b/examples/simple/superstore-custom-grid.html @@ -214,57 +214,57 @@