diff --git a/README.md b/README.md index 152acd0f2e..ba987642de 100644 --- a/README.md +++ b/README.md @@ -273,4 +273,21 @@ document.addEventListener("WebComponentsReady", function () { }); ``` +Views can share a table by instancing it separately and passing it to the +`load()` method. + +```javascript +var view1 = document.getElementById('view1'); +var view2 = document.getElementById('view2'); + +// Use the default Web Worker instance +var tbl = view1.worker.table(data); + +view1.load(tbl); +view2.load(tbl); + +tbl.update([{'x': 5, 'y': 'e', 'z': true}]); +``` + + See [API Docs](https://github.com/jpmorganchase/perspective/tree/master/packages/perspective) for more details. diff --git a/lerna.json b/lerna.json index 4b52e4ee36..c8ac83657a 100644 --- a/lerna.json +++ b/lerna.json @@ -3,5 +3,5 @@ "packages": [ "packages/*" ], - "version": "0.1.4" + "version": "0.1.5" } diff --git a/package.json b/package.json index 9c7553ec42..97f7a8da39 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,7 @@ "scripts": { "start": "lerna run start ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}} --stream", "travis_start": "lerna run start ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}} --stream && lerna run compile_test --stream", - "puppeteer": "docker run -it --rm --shm-size=2g -u root -e WRITE_TESTS=${WRITE_TESTS} -v $(pwd):/src -w /src/packages/${PACKAGE} perspective/puppeteer ./node_modules/.bin/jest --silent --runInBand", + "puppeteer": "docker run -it --rm --shm-size=2g -u root -e WRITE_TESTS=${WRITE_TESTS} -v $(pwd):/src -w /src/packages/${PACKAGE} perspective/puppeteer ./node_modules/.bin/jest --silent", "postinstall": "lerna bootstrap --hoist", "test": "lerna run compile_test --stream && npm run test_perspective && npm run test_viewer && npm run test_hypergrid && npm run test_highcharts", "test_perspective": "PACKAGE=perspective npm run puppeteer", diff --git a/packages/perspective-common/common.config.js b/packages/perspective-common/common.config.js index 3efcbb010b..b6e148406d 100644 --- a/packages/perspective-common/common.config.js +++ b/packages/perspective-common/common.config.js @@ -1,7 +1,10 @@ const UglifyJSPlugin = require('uglifyjs-webpack-plugin'); +const webpack = require('webpack'); -const plugins = [] +const plugins = [ + new webpack.ContextReplacementPlugin(/moment[\/\\]locale$/, /(en|es|fr)$/) +]; if (!process.env.PSP_NO_MINIFY && !process.env.PSP_DEBUG) { plugins.push(new UglifyJSPlugin({ diff --git a/packages/perspective-common/package.json b/packages/perspective-common/package.json index 4156f8429d..1c090aa10b 100644 --- a/packages/perspective-common/package.json +++ b/packages/perspective-common/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-common", - "version": "0.1.2", + "version": "0.1.5", "description": "Perspective.js", "main": "index.js", "files": [ diff --git a/packages/perspective-examples/package.json b/packages/perspective-examples/package.json index 96f928bf9d..8cd9d6810c 100644 --- a/packages/perspective-examples/package.json +++ b/packages/perspective-examples/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-examples", - "version": "0.1.4", + "version": "0.1.5", "description": "Perspective.js", "main": "index.js", "directories": { @@ -27,10 +27,10 @@ "author": "", "license": "Apache", "dependencies": { - "@jpmorganchase/perspective": "^0.1.4", - "@jpmorganchase/perspective-viewer": "^0.1.4", - "@jpmorganchase/perspective-viewer-highcharts": "^0.1.4", - "@jpmorganchase/perspective-viewer-hypergrid": "^0.1.4", + "@jpmorganchase/perspective": "^0.1.5", + "@jpmorganchase/perspective-viewer": "^0.1.5", + "@jpmorganchase/perspective-viewer-highcharts": "^0.1.5", + "@jpmorganchase/perspective-viewer-hypergrid": "^0.1.5", "babel-runtime": "^6.26.0", "query-string": "^5.0.1", "rectangular": "1.0.1", diff --git a/packages/perspective-jupyterlab/package.json b/packages/perspective-jupyterlab/package.json index 855b3904f2..8e8e87bed1 100644 --- a/packages/perspective-jupyterlab/package.json +++ b/packages/perspective-jupyterlab/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-jupyterlab", - "version": "0.1.4", + "version": "0.1.5", "description": "Perspective.js", "files": [ "build/*.d.ts", @@ -24,9 +24,9 @@ "watch": "tsc -w" }, "dependencies": { - "@jpmorganchase/perspective-viewer": "^0.1.4", - "@jpmorganchase/perspective-viewer-highcharts": "^0.1.4", - "@jpmorganchase/perspective-viewer-hypergrid": "^0.1.4", + "@jpmorganchase/perspective-viewer": "^0.1.5", + "@jpmorganchase/perspective-viewer-highcharts": "^0.1.5", + "@jpmorganchase/perspective-viewer-hypergrid": "^0.1.5", "@jupyterlab/application": "^0.15.0", "@jupyterlab/rendermime-interfaces": "^1.0.0", "@jupyterlab/services": "^1.1.1-0", diff --git a/packages/perspective-viewer-highcharts/package.json b/packages/perspective-viewer-highcharts/package.json index cdd3028460..834f21c3a5 100644 --- a/packages/perspective-viewer-highcharts/package.json +++ b/packages/perspective-viewer-highcharts/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-viewer-highcharts", - "version": "0.1.4", + "version": "0.1.5", "description": "Perspective.js", "main": "build/highcharts.plugin.js", "directories": { @@ -33,7 +33,7 @@ "author": "", "license": "Apache", "dependencies": { - "@jpmorganchase/perspective-common": "^0.1.2", + "@jpmorganchase/perspective-common": "^0.1.5", "babel-runtime": "^6.26.0", "chroma-js": "^1.3.4", "highcharts": "5.0.14", @@ -41,9 +41,9 @@ "highcharts-more": "^0.1.2" }, "devDependencies": { - "@jpmorganchase/perspective": "^0.1.4", - "@jpmorganchase/perspective-common": "^0.1.2", - "@jpmorganchase/perspective-viewer": "^0.1.4", + "@jpmorganchase/perspective": "^0.1.5", + "@jpmorganchase/perspective-common": "^0.1.5", + "@jpmorganchase/perspective-viewer": "^0.1.5", "babel-core": "^6.26.0", "babel-loader": "^7.1.2", "babel-plugin-transform-es2015-for-of": "^6.23.0", diff --git a/packages/perspective-viewer-highcharts/src/js/heatmap.js b/packages/perspective-viewer-highcharts/src/js/heatmap.js index d50e3133a6..671cd837a3 100644 --- a/packages/perspective-viewer-highcharts/src/js/heatmap.js +++ b/packages/perspective-viewer-highcharts/src/js/heatmap.js @@ -10,6 +10,7 @@ import highcharts from 'highcharts'; import highchartsMore from 'highcharts-more'; import heatmap from 'highcharts/modules/heatmap'; +import boost from 'highcharts/modules/boost'; import grouped_categories from 'highcharts-grouped-categories'; import chroma from 'chroma-js'; @@ -27,6 +28,7 @@ let axisProto = Highcharts.Axis.prototype, highchartsMore(highcharts); heatmap(highcharts); grouped_categories(highcharts); +boost(highcharts); export const COLORS_10 = ['#1f77b4','#ff7f0e','#2ca02c','#d62728','#9467bd','#8c564b','#e377c2','#7f7f7f','#bcbd22','#17becf']; export const COLORS_20 = [ diff --git a/packages/perspective-viewer-highcharts/src/js/highcharts.js b/packages/perspective-viewer-highcharts/src/js/highcharts.js index 6fc02ffeda..1f45587db7 100644 --- a/packages/perspective-viewer-highcharts/src/js/highcharts.js +++ b/packages/perspective-viewer-highcharts/src/js/highcharts.js @@ -267,7 +267,7 @@ export function draw(mode) { let new_radius = 0; if (mode === 'scatter') { - new_radius = Math.min(8, Math.max(4, Math.floor((this.clientWidth + this.clientHeight) / 240))); + new_radius = Math.min(8, Math.max(4, Math.floor((this.clientWidth + this.clientHeight) / Math.max(300, series[0].data.length / 3)))); } var config = { @@ -323,6 +323,7 @@ export function draw(mode) { }, series: { animation: false, + boostThreshold: Infinity, 'turboThreshold': 60000, borderWidth: 0, connectNulls: true, @@ -396,6 +397,16 @@ export function draw(mode) { } }); } + if (aggregates.length < 3) { + Object.assign(config, { + boost: { + useGPUTranslations: true, + usePreAllocated: true + } + }); + config.plotOptions.series.boostThreshold = 5000; + config.plotOptions.series.turboThreshold = Infinity; + } Object.assign(config, { colors: [ colors[0] @@ -436,6 +447,15 @@ export function draw(mode) { colorRange = [-cmax, cmax]; } + Object.assign(config, { + boost: { + useGPUTranslations: true, + usePreAllocated: true + } + }); + config.plotOptions.series.boostThreshold = 5000; + config.plotOptions.series.turboThreshold = Infinity; + // Calculate ylabel nesting let ylabels = series.map(function (s) { return s.name.split(','); }) let ytop = {name: null, depth: 0, categories: []}; @@ -469,6 +489,11 @@ export function draw(mode) { data: data, nullColor: '#999' }], + + boost: { + useGPUTranslations: true + }, + colorAxis: { min: colorRange[0], max: colorRange[1], @@ -553,7 +578,7 @@ export function draw(mode) { if (this._chart) { if (mode === 'scatter') { - let new_radius = Math.min(8, Math.max(4, Math.floor((this._chart.chartWidth + this._chart.chartHeight) / 240))); + let new_radius = Math.min(6, Math.max(2, Math.floor((this._chart.chartWidth + this._chart.chartHeight) / Math.max(300, config.series[0].data.length / 3)))); this._chart.update({ series: config.series, plotOptions: { diff --git a/packages/perspective-viewer-highcharts/test/results/results.json b/packages/perspective-viewer-highcharts/test/results/results.json index dd87263342..a2e107d214 100644 --- a/packages/perspective-viewer-highcharts/test/results/results.json +++ b/packages/perspective-viewer-highcharts/test/results/results.json @@ -7,13 +7,13 @@ "superstore.html/sorts by a numeric column.": "375062fbd50e09afa7b6df7e6b4a3a5f", "superstore.html/sorts by an alpha column.": "2643e0b5787b6e8e85f122b1713d01cd", "superstore.html/displays visible columns.": "1da296527f149393be8f3f99e34b3e4a", - "scatter.html/shows a grid without any settings applied.": "d06e736acfd0512d7edb77f226581565", - "scatter.html/pivots by a row.": "10b9610b3457b590b4836b975e03c068", - "scatter.html/pivots by two rows.": "41e6302d8087af19d3d2a0f09ae5aa0a", - "scatter.html/pivots by a row and a column.": "3b056b59fa8f765d8cf64e0850b2cd11", - "scatter.html/pivots by two rows and two columns.": "30d8e6c029ad7867ea5f1f0155222039", - "scatter.html/sorts by a numeric column.": "0ed4d9b8999afcc636926eda8858785b", - "scatter.html/sorts by an alpha column.": "27a0820e9bc3dfcca56690ca28da4e64", + "scatter.html/shows a grid without any settings applied.": "0ef3d2e80e4959067d650491d93adcd5", + "scatter.html/pivots by a row.": "514b5ca8d31fe6e8d7550bc27ea15927", + "scatter.html/pivots by two rows.": "c9f18a0977f49b1e154c58c72ba9d8d3", + "scatter.html/pivots by a row and a column.": "525eb9dee38854f95eddcd7b3bd54395", + "scatter.html/pivots by two rows and two columns.": "89cfce914b11bb99b16dc03e2d2f8ca5", + "scatter.html/sorts by a numeric column.": "1743fdf24da7aed53faf74de40b0a82f", + "scatter.html/sorts by an alpha column.": "233139e79430cd9894e5cd6f88d955c7", "scatter.html/displays visible columns.": "aecfc8890cf0d63020f40235fe9329c5", "bar.html/shows a grid without any settings applied.": "026d2eaebb0fbac962eff07e51aede89", "bar.html/pivots by a row.": "f4a8a1d1c8ab73a8a0adc3ca2e3fafb7", @@ -31,10 +31,10 @@ "line.html/sorts by an alpha column.": "cb5289fe91482191ff60a96c2b38fd1e", "line.html/displays visible columns.": "58cffc507ef835e319616bca7243f735", "bar.html/displays visible columns.": "60815e32387730d65b3b304e9faa1f7d", - "scatter.html/sorts by a hidden column.": "ca7bf4886caf2ea11b152101de74761a", + "scatter.html/sorts by a hidden column.": "fca2080cc47f4652d5a5dcadf5d39df7", "line.html/sorts by a hidden column.": "c5fe3821967162ebd9a45aa4f6875cc4", "bar.html/sorts by a hidden column.": "df74e27ca165fece457e36d9d72ba8d9", - "scatter.html/filters by a numeric column.": "4076e985f519a7d565de860eba47f533", + "scatter.html/filters by a numeric column.": "65e362590f05e931a3c4207b17723aca", "line.html/filters by a numeric column.": "22c80d25e38429af1ce9456826fe917f", "bar.html/filters by a numeric column.": "81c8f07989a6cd72726b584b395e9764", "heatmap.html/shows a grid without any settings applied.": "8557a845ef78bea79ba024d6a5c1653e", diff --git a/packages/perspective-viewer-hypergrid/package.json b/packages/perspective-viewer-hypergrid/package.json index 8bd4a74a78..9339dea277 100644 --- a/packages/perspective-viewer-hypergrid/package.json +++ b/packages/perspective-viewer-hypergrid/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-viewer-hypergrid", - "version": "0.1.4", + "version": "0.1.5", "description": "Perspective.js", "main": "build/hypergrid.plugin.js", "directories": { @@ -33,16 +33,16 @@ "author": "", "license": "Apache", "dependencies": { - "@jpmorganchase/perspective-common": "^0.1.2", + "@jpmorganchase/perspective-common": "^0.1.5", "babel-polyfill": "^6.26.0", "babel-runtime": "^6.26.0", "fin-hypergrid": "2.0.2", "underscore": "^1.8.3" }, "devDependencies": { - "@jpmorganchase/perspective": "^0.1.4", - "@jpmorganchase/perspective-common": "^0.1.2", - "@jpmorganchase/perspective-viewer": "^0.1.4", + "@jpmorganchase/perspective": "^0.1.5", + "@jpmorganchase/perspective-common": "^0.1.5", + "@jpmorganchase/perspective-viewer": "^0.1.5", "babel-core": "^6.26.0", "babel-jest": "^22.0.4", "babel-loader": "^7.1.2", diff --git a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js index c486ba3dc4..feeb5e1d22 100644 --- a/packages/perspective-viewer-hypergrid/src/js/hypergrid.js +++ b/packages/perspective-viewer-hypergrid/src/js/hypergrid.js @@ -118,10 +118,6 @@ function generateGridProperties(overrides) { } function setPSP(payload) { - if (payload.rows.length === 0) { - this.grid.setData({data: []}); - return; - }; if (payload.isTree) { this.grid.renderer.properties.fixedColumnCount = 1; } else { @@ -427,13 +423,15 @@ var conv = { function psp2hypergrid(data, schema, start = 0, end = undefined, length = undefined) { if (data.length === 0) { + let columns = Object.keys(schema); return { + rows: [], rowPaths: [], data: [], isTree: false, configuration: {}, - columnPaths: [], - columnTypes: [] + columnPaths: columns.map(col => [col]), + columnTypes: columns.map(col => conv[schema[col]]) } } diff --git a/packages/perspective-viewer/package.json b/packages/perspective-viewer/package.json index cc2b7b2c7a..07639280a0 100644 --- a/packages/perspective-viewer/package.json +++ b/packages/perspective-viewer/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-viewer", - "version": "0.1.4", + "version": "0.1.5", "description": "Perspective.js", "main": "build/perspective.view.js", "files": [ @@ -30,8 +30,8 @@ "author": "", "license": "Apache", "dependencies": { - "@jpmorganchase/perspective": "^0.1.4", - "@jpmorganchase/perspective-common": "^0.1.2", + "@jpmorganchase/perspective": "^0.1.5", + "@jpmorganchase/perspective-common": "^0.1.5", "babel-polyfill": "^6.26.0", "babel-runtime": "^6.26.0", "bluebird": "^3.5.1", @@ -40,7 +40,7 @@ "underscore": "^1.8.3" }, "devDependencies": { - "@jpmorganchase/perspective-common": "^0.1.2", + "@jpmorganchase/perspective-common": "^0.1.5", "babel-core": "^6.26.0", "babel-loader": "^7.1.2", "babel-plugin-transform-es2015-for-of": "^6.23.0", diff --git a/packages/perspective-viewer/src/js/view.js b/packages/perspective-viewer/src/js/view.js index d94b695400..d20b940efb 100644 --- a/packages/perspective-viewer/src/js/view.js +++ b/packages/perspective-viewer/src/js/view.js @@ -69,9 +69,13 @@ function calc_index(event) { if (this._active_columns.children.length == 0) { return 0; } else { - let {offsetHeight, offsetTop} = this._active_columns.children[0]; - let column_offset = offsetTop - this._active_columns.offsetTop; - return Math.max(0, Math.floor((event.offsetY + this._active_columns.scrollTop - column_offset) / offsetHeight)); + for (let cidx in this._active_columns.children) { + let child = this._active_columns.children[cidx]; + if (child.offsetTop + child.offsetHeight > event.offsetY + this._active_columns.scrollTop) { + return cidx; + } + } + return this._active_columns.children.length; } } @@ -559,7 +563,12 @@ registerElement(template, { _plugin: { get: function () { - return RENDERERS[this._vis_selector.value || this.getAttribute('view') || Object.apply.keys(RENDERERS)[0] ]; + let view = this.getAttribute('view'); + if (!view) { + view = Object.keys(RENDERERS)[0]; + } + this.setAttribute('view', view); + return RENDERERS[ view ]; } }, @@ -568,7 +577,7 @@ registerElement(template, { if (this._show_config) { this._side_panel.style.display = 'none'; this._top_panel.style.display = 'none'; - this.removeAttribute('settings') + this.removeAttribute('settings'); } else { this._side_panel.style.display = 'flex'; this._top_panel.style.display = 'flex'; diff --git a/packages/perspective-viewer/src/less/row.less b/packages/perspective-viewer/src/less/row.less index 39a7fe8cd2..70f07813aa 100644 --- a/packages/perspective-viewer/src/less/row.less +++ b/packages/perspective-viewer/src/less/row.less @@ -164,9 +164,12 @@ perspective-row #sort_order { #side_panel { padding-top: 5px; + flex: 0 1 auto; } #side_panel perspective-row .column_name { + overflow: hidden; + max-width: 180px; white-space: nowrap; text-overflow: ellipsis; } @@ -228,6 +231,11 @@ perspective-row .row_draggable select:focus { } perspective-viewer[view=scatter], perspective-viewer[view=line] { + + svg image { + opacity: 0.8; + } + #active_columns { :nth-child(1) .is_visible::before { .selected_indicator(); diff --git a/packages/perspective-viewer/src/less/view.less b/packages/perspective-viewer/src/less/view.less index 9cc95b59b2..bae61235fb 100644 --- a/packages/perspective-viewer/src/less/view.less +++ b/packages/perspective-viewer/src/less/view.less @@ -103,7 +103,6 @@ perspective-viewer { ul { min-height: 20px; - position: static !important; } #active_columns, #inactive_columns { @@ -113,6 +112,7 @@ perspective-viewer { border: none !important; margin-top: 8px; overflow-y: auto; + position: relative; } #inactive_columns { @@ -135,7 +135,7 @@ perspective-viewer { display: flex; list-style: none; flex-direction: column; - flex-grow: 1; + flex: 1; } .rrow { display: flex; @@ -160,10 +160,6 @@ perspective-viewer { white-space: nowrap; } - .fixed { - max-width: 180px; - } - #top_panel perspective-row + perspective-row:before { content: ","; padding: 0px 10px 0px 0px; @@ -250,7 +246,7 @@ perspective-viewer { input::placeholder { color: #ccc; } - rect { + rect, svg image { opacity: 0.5; } path, circle { diff --git a/packages/perspective-viewer/test/js/utils.js b/packages/perspective-viewer/test/js/utils.js index 8c76f07d50..b29699fbc7 100644 --- a/packages/perspective-viewer/test/js/utils.js +++ b/packages/perspective-viewer/test/js/utils.js @@ -7,9 +7,15 @@ * */ -var http = require('http'); -var fs = require('fs'); -var path = require('path'); +const http = require('http'); +const fs = require('fs'); +const path = require('path'); +const crypto = require('crypto'); +const puppeteer = require('puppeteer'); + +const cons = require('console'); +const private_console = new cons.Console(process.stdout, process.stderr); +const cp = require('child_process'); let __PORT__; @@ -87,13 +93,29 @@ if (!fs.existsSync('screenshots')) { fs.mkdirSync('screenshots'); } -const puppeteer = require('puppeteer'); -let browser; - -var crypto = require('crypto'); +let browser, page, url, errors = []; beforeAll(async () => { browser = await puppeteer.launch({args: ['--no-sandbox', '--disable-setuid-sandbox']}); + page = await browser.newPage(); + + // CSS Animations break our screenshot tests, so set the + // animation playback rate to something extreme. + await page._client.send('Animation.setPlaybackRate', { playbackRate: 100.0 }); + page.on('console', msg => { + if (msg.type === 'error') { + errors.push(msg.text); + } + if (process.env.DEBUG) { + private_console.log(msg.text); + } + }); + page.on('pageerror', msg => { + errors.push(msg.message); + if (process.env.DEBUG) { + private_console.error(msg.message); + } + }); }); afterAll(() => { @@ -103,8 +125,6 @@ afterAll(() => { } }); -let url; - describe.page = (_url, body) => { if (!fs.existsSync('screenshots/' + _url.replace('.html', ''))) { fs.mkdirSync('screenshots/' + _url.replace('.html', '')); @@ -118,42 +138,23 @@ describe.page = (_url, body) => { }); } -const cons = require('console'); -const private_console = new cons.Console(process.stdout, process.stderr); -const cp = require('child_process'); - test.capture = function capture(name, body, timeout = 60000) { let _url = url; test(name, async () => { - let errors = []; + errors = []; if (process.env.DEBUG) private_console.log("---- " + name + " -----------------------------"); - const page = await browser.newPage(); - page.on('console', msg => { - if (msg.type === 'error') { - errors.push(msg.text); - } - if (process.env.DEBUG) { - private_console.log(msg.text); - } - }); - page.on('pageerror', msg => { - errors.push(msg.message); - if (process.env.DEBUG) { - private_console.error(msg.message); - } - }); - + + await new Promise(setTimeout); await page.goto(`http://127.0.0.1:${__PORT__}/${_url}`); - await page.waitForSelector('perspective-viewer[render_time]'); await page.waitForSelector('perspective-viewer:not([updating])'); + await body(page); // let animation run; await page.waitForSelector('perspective-viewer:not([updating])'); - await page.waitFor(500); const screenshot = await page.screenshot(); - await page.close(); + // await page.close(); const hash = crypto.createHash('md5').update(screenshot).digest("hex"); if (process.env.WRITE_TESTS) { results[_url + '/' + name] = hash; diff --git a/packages/perspective/CMakeLists.txt b/packages/perspective/CMakeLists.txt index 14d05977c6..3dec3f1f4d 100644 --- a/packages/perspective/CMakeLists.txt +++ b/packages/perspective/CMakeLists.txt @@ -28,7 +28,7 @@ set(EXTENDED_FLAGS " \ if(DEFINED ENV{PSP_DEBUG}) set(OPT_FLAGS " \ - -Os \ + -O1 \ -g4 \ -s SAFE_HEAP=1 \ -s DISABLE_EXCEPTION_CATCHING=0 \ diff --git a/packages/perspective/package.json b/packages/perspective/package.json index 45f579aae0..ee9c65fd6b 100644 --- a/packages/perspective/package.json +++ b/packages/perspective/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective", - "version": "0.1.4", + "version": "0.1.5", "description": "Perspective.js", "main": "build/perspective.js", "publishConfig": { @@ -34,7 +34,8 @@ "author": "", "license": "Apache", "dependencies": { - "@jpmorganchase/perspective-common": "^0.1.2", + "@apache-arrow/es5-esm": "^0.3.1", + "@jpmorganchase/perspective-common": "^0.1.5", "d3-array": "^1.2.1", "moment": "^2.19.1", "papaparse": "^4.3.6", @@ -42,7 +43,7 @@ }, "devDependencies": { "@apache-arrow/es5-esm": "^0.2.0", - "@jpmorganchase/perspective-common": "^0.1.2", + "@jpmorganchase/perspective-common": "^0.1.5", "arraybuffer-loader": "^1.0.2", "babel-core": "^6.26.0", "babel-loader": "^7.1.2", diff --git a/packages/perspective/src/cpp/main.cpp b/packages/perspective/src/cpp/main.cpp index e5a08da86f..e37135a772 100644 --- a/packages/perspective/src/cpp/main.cpp +++ b/packages/perspective/src/cpp/main.cpp @@ -177,14 +177,15 @@ namespace arrow { void fill_col_dict(t_uint32 nrows, t_uint32 dsize, val dcol, val vkeys, t_col_sptr col, const char* destType) { - val vdata = dcol["data"]; + // ptaylor: This assumes the dictionary is either a Binary or Utf8 Vector. Should it support other Vector types? + val vdata = dcol["values"]; t_int32 vsize = vdata["length"].as(); std::vector data; data.reserve(vsize); data.resize(vsize); vecFromTypedArray(vdata, data.data(), vsize); - val voffsets = dcol["offsets"]; + val voffsets = dcol["valueOffsets"]; t_int32 osize = voffsets["length"].as(); std::vector offsets; offsets.reserve(osize); @@ -215,7 +216,7 @@ _fill_col(val dcol, t_col_sptr col, t_bool is_arrow) t_uindex nrows = col->size(); if (is_arrow) { - val data = dcol["data"]; + val data = dcol["values"]; arrow::vecFromTypedArray(data, col->get_nth(0), nrows); } else { for (auto i = 0; i < nrows; ++i) @@ -233,7 +234,7 @@ _fill_col(val dcol, t_col_sptr col, t_bool is_arrow) t_uindex nrows = col->size(); if (is_arrow) { - val data = dcol["data"]; + val data = dcol["values"]; // arrow packs 64 bit into two 32 bit ints arrow::vecFromTypedArray(data, col->get_nth(0), nrows * 2); } else { @@ -248,17 +249,17 @@ _fill_col(val dcol, t_col_sptr col, t_bool is_arrow) t_uindex nrows = col->size(); if (is_arrow) { - val data = dcol["data"]; + val data = dcol["values"]; // arrow packs 64 bit into two 32 bit ints arrow::vecFromTypedArray(data, col->get_nth(0), nrows*2); - t_str unit = dcol["unit"].as(); - if (unit != "MILLISECOND") { + t_int8 unit = dcol["type"]["unit"].as(); + if (unit != /* Arrow.enum_.TimeUnit.MILLISECOND */ 1) { // Slow path - need to convert each value t_int64 factor = 1; - if (unit == "NANOSECOND") { + if (unit == /* Arrow.enum_.TimeUnit.NANOSECOND */ 3) { factor = 1e6; - } else if (unit == "MICROSECOND") { + } else if (unit == /* Arrow.enum_.TimeUnit.MICROSECOND */ 2) { factor = 1e3; } for (auto i = 0; i < nrows; ++i) @@ -283,7 +284,7 @@ _fill_col(val dcol, t_col_sptr col, t_bool is_arrow) if (is_arrow) { // arrow packs bools into a bitmap - val data = dcol["data"]; + val data = dcol["values"]; for (auto i = 0; i < nrows; ++i) { t_uint8 elem = data[i / 8].as(); @@ -309,17 +310,13 @@ _fill_col(val dcol, t_col_sptr col, t_bool is_arrow) if (is_arrow) { if (dcol["constructor"]["name"].as() == "DictionaryVector") { - val dictvec = dcol["data"]; + + val dictvec = dcol["dictionary"]; // Get number of dictionary entries t_uint32 dsize = dictvec["length"].as(); - // UTF-8 dictionaries have an extra level of js object! - if (dictvec["constructor"]["name"].as() == "Utf8Vector") { - dictvec = dictvec["values"]; - } - - val vkeys = dcol["keys"]["data"]; + val vkeys = dcol["indices"]["values"]; // Perspective stores string indices in a 32bit unsigned array // Javascript's typed arrays handle copying from various bitwidth arrays properly @@ -340,18 +337,15 @@ _fill_col(val dcol, t_col_sptr col, t_bool is_arrow) } } else if (dcol["constructor"]["name"].as() == "Utf8Vector" || dcol["constructor"]["name"].as() == "BinaryVector") { - if (dcol["constructor"]["name"].as() == "Utf8Vector") { - dcol = dcol["values"]; - } - val vdata = dcol["data"]; + val vdata = dcol["values"]; t_int32 vsize = vdata["length"].as(); std::vector data; data.reserve(vsize); data.resize(vsize); arrow::vecFromTypedArray(vdata, data.data(), vsize); - val voffsets = dcol["offsets"]; + val voffsets = dcol["valueOffsets"]; t_int32 osize = voffsets["length"].as(); std::vector offsets; offsets.reserve(osize); @@ -467,12 +461,7 @@ _fill_data(t_table_sptr tbl, if (null_count == 0) { col->valid_raw_fill(true); } else { - val validity = dcol; - if (dcol["constructor"]["name"].as() == "Utf8Vector") { - validity = dcol["values"]["validity"]["data"]; - } else { - validity = dcol["validity"]["data"]; - } + val validity = dcol["nullBitmap"]; arrow::fill_col_valid(validity, col); } } diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index 75cce8e65b..ef9310b9a9 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -79,7 +79,7 @@ const DATE_PARSE_CANDIDATES = [ * Do any necessary data transforms on columns. Currently it does the following * transforms * 1. Date objects are converted into float millis since epoch - * + * * @private * @param {string} type type of column * @param {array} data array of columnar data @@ -174,8 +174,15 @@ function parse_data(data, names, types) { const date_exclusions = []; for (let x = 0; x < data.length; x ++) { if (!(name in data[x]) || data[x][name] === undefined) continue; - if (inferredType.value === __MODULE__.t_dtype.DTYPE_FLOAT64.value || inferredType.value === __MODULE__.t_dtype.DTYPE_INT32.value) { + if (inferredType.value === __MODULE__.t_dtype.DTYPE_FLOAT64.value) { col.push(Number(data[x][name])); + } else if (inferredType.value === __MODULE__.t_dtype.DTYPE_INT32.value) { + const val = Number(data[x][name]); + if (val <= 2147483647 && val >= -2147483648) { + col.push(val); + } else { + types[n] = __MODULE__.t_dtype.DTYPE_FLOAT64; + } } else if (inferredType.value === __MODULE__.t_dtype.DTYPE_BOOL.value) { let cell = data[x][name]; if ((typeof cell) === "string") { @@ -192,7 +199,7 @@ function parse_data(data, names, types) { col.push(-1); } else { let val = data[x][name]; - if (typeof val === "string") { + if (typeof val === "string") { val = moment(data[x][name], date_types, true); if (!val.isValid() || date_types.length === 0) { let found = false; @@ -280,65 +287,100 @@ function parse_data(data, names, types) { * types - the column t_dtypes. */ function load_arrow_buffer(data, names, types) { - // TODO Need to validate that the names/types passed in match those in the buffer - - var arrow = Arrow.Table.from([new Uint8Array(data)]); - - names = []; - types = []; - let cdata = []; - for (let column of arrow.columns) { - switch (column.type) { - case 'Binary': - case 'Utf8': - types.push(__MODULE__.t_dtype.DTYPE_STR); - break; - case 'FloatingPoint': - if (column instanceof Arrow.Float64Vector) { - types.push(__MODULE__.t_dtype.DTYPE_FLOAT64); - } - else if (column instanceof Arrow.Float32Vector) { - types.push(__MODULE__.t_dtype.DTYPE_FLOAT32); - } - break; - case 'Int': - if (column instanceof Arrow.Int64Vector) { - types.push(__MODULE__.t_dtype.DTYPE_INT64); - } - else if (column instanceof Arrow.Int32Vector) { - types.push(__MODULE__.t_dtype.DTYPE_INT32); - } - else if (column instanceof Arrow.Int16Vector) { - types.push(__MODULE__.t_dtype.DTYPE_INT16); - } - else if (column instanceof Arrow.Int8Vector) { - types.push(__MODULE__.t_dtype.DTYPE_INT8); - } - break; - case 'Bool': - types.push(__MODULE__.t_dtype.DTYPE_BOOL); - break; - case 'Timestamp': - types.push(__MODULE__.t_dtype.DTYPE_TIME); - break; - default: - continue; - break; - } - cdata.push(column); - names.push(column.name); - } - + let arrow = Arrow.Table.from([new Uint8Array(data)]); + let loader = arrow.schema.fields.reduce((loader, field, colIdx) => { + return loader.loadColumn(field, arrow.getColumnAt(colIdx)); + }, new ArrowColumnLoader()); return { row_count: arrow.length, is_arrow: true, - names: names, - types: types, - cdata: cdata + names: loader.names, + types: loader.types, + cdata: loader.cdata }; } +/** + * + * @private + */ +class ArrowColumnLoader extends Arrow.visitor.TypeVisitor { + constructor(cdata, names, types) { + super(); + this.cdata = cdata || []; + this.names = names || []; + this.types = types || []; + } + loadColumn(field/*: Arrow.type.Field*/, column/*: Arrow.Vector*/) { + if (this.visit(field.type)) { + this.cdata.push(column); + this.names.push(field.name); + } + return this; + } + // visitNull(type/*: Arrow.type.Null*/) {} + visitBool(type/*: Arrow.type.Bool*/) { + this.types.push(__MODULE__.t_dtype.DTYPE_BOOL); + return true; + } + visitInt(type/*: Arrow.type.Int*/) { + const bitWidth = type.bitWidth; + if (bitWidth === 64) { + this.types.push(__MODULE__.t_dtype.DTYPE_INT64); + } + else if (bitWidth === 32) { + this.types.push(__MODULE__.t_dtype.DTYPE_INT32); + } + else if (bitWidth === 16) { + this.types.push(__MODULE__.t_dtype.DTYPE_INT16); + } + else if (bitWidth === 8) { + this.types.push(__MODULE__.t_dtype.DTYPE_INT8); + } + return true; + } + visitFloat(type/*: Arrow.type.Float*/) { + const precision = type.precision; + if (precision === Arrow.enum_.Precision.DOUBLE) { + this.types.push(__MODULE__.t_dtype.DTYPE_FLOAT64); + } + else if (precision === Arrow.enum_.Precision.SINGLE) { + this.types.push(__MODULE__.t_dtype.DTYPE_FLOAT32); + } + // todo? + // else if (type.precision === Arrow.enum_.Precision.HALF) { + // this.types.push(__MODULE__.t_dtype.DTYPE_FLOAT16); + // } + return true; + } + visitUtf8(type/*: Arrow.type.Utf8 */) { + this.types.push(__MODULE__.t_dtype.DTYPE_STR); + return true; + } + visitBinary(type/*: Arrow.type.Binary */) { + this.types.push(__MODULE__.t_dtype.DTYPE_STR); + return true; + } + // visitFixedSizeBinary(type/*: Arrow.type.FixedSizeBinary*/) {} + // visitDate(type/*: Arrow.type.Date_*/) {} + visitTimestamp(type/*: Arrow.type.Timestamp*/) { + this.types.push(__MODULE__.t_dtype.DTYPE_TIME); + return true; + } + // visitTime(type/*: Arrow.type.Time*/) {} + // visitDecimal(type/*: Arrow.type.Decimal*/) {} + // visitList(type/*: Arrow.type.List*/) {} + // visitStruct(type/*: Arrow.type.Struct*/) {} + // visitUnion(type/*: Arrow.type.Union*/) {} + visitDictionary(type/*: Arrow.type.Dictionary*/) { + return this.visit(type.dictionary); + } + // visitInterval(type/*: Arrow.type.Interval*/) {} + // visitFixedSizeList(type/*: Arrow.type.FixedSizeList*/) {} + // visitMap(type/*: Arrow.type.Map_*/) {} +} + /****************************************************************************** * * View @@ -346,16 +388,16 @@ function load_arrow_buffer(data, names, types) { */ /** - * A View object represents a specific transform (configuration or pivot, + * A View object represents a specific transform (configuration or pivot, * filter, sort, etc) configuration on an underlying {@link table}. A View * receives all updates from the {@link table} from which it is derived, and - * can be serialized to JSON or trigger a callback when it is updated. View - * objects are immutable, and will remain in memory and actively process + * can be serialized to JSON or trigger a callback when it is updated. View + * objects are immutable, and will remain in memory and actively process * updates until its {@link view#delete} method is called. * * Note This constructor is not public - Views are created - * by invoking the {@link table#view} method. - * + * by invoking the {@link table#view} method. + * * @example * // Returns a new View, pivoted in the row space by the "name" column. * table.view({row_pivots: ["name"]}); @@ -442,7 +484,7 @@ view.prototype._column_names = function() { * The schema of this {@link view}. A schema is an Object, the keys of which * are the columns of this {@link view}, and the values are their string type names. * If this {@link view} is aggregated, theses will be the aggregated types; - * otherwise these types will be the same as the columns in the underlying + * otherwise these types will be the same as the columns in the underlying * {@link table} * * @async @@ -502,12 +544,12 @@ view.prototype.schema = async function() { * @param {number} options.start_col The starting column index from which * to serialize. * @param {number} options.end_col The ending column index from which - * to serialize. - * - * @returns {Promise} A Promise resolving to An array of Objects + * to serialize. + * + * @returns {Promise} A Promise resolving to An array of Objects * representing the rows of this {@link view}. If this {@link view} had a * "row_pivots" config parameter supplied when constructed, each row Object - * will have a "__ROW_PATH__" key, whose value specifies this row's + * will have a "__ROW_PATH__" key, whose value specifies this row's * aggregated path. If this {@link view} had a "column_pivots" config * parameter supplied, the keys of this object will be comma-prepended with * their comma-separated column paths. @@ -610,8 +652,8 @@ view.prototype.num_columns = async function() { * underlying table emits an update, this callback will be invoked with the * aggregated row deltas. * - * @param {function} callback A callback function invoked on update. The - * parameter to this callback shares a structure with the return type of + * @param {function} callback A callback function invoked on update. The + * parameter to this callback shares a structure with the return type of * {@link view#to_json}. */ view.prototype.on_update = function (callback) { @@ -662,11 +704,11 @@ view.prototype.on_delete = function (callback) { /** * A Table object is the basic data container in Perspective. Tables are * typed - they have an immutable set of column names, and a known type for - * each. - * + * each. + * * Note This constructor is not public - Tables are created - * by invoking the {@link table} factory method, either on the perspective - * module object, or an a {@link worker} instance. + * by invoking the {@link table} factory method, either on the perspective + * module object, or an a {@link worker} instance. * * @class * @hideconstructor @@ -767,16 +809,16 @@ table.prototype.schema = async function() { * configuration. * * @param {Object} [config] The configuration object for this {@link view}. - * @param {Array} [config.row_pivot] An array of column names + * @param {Array} [config.row_pivot] An array of column names * to use as {@link https://en.wikipedia.org/wiki/Pivot_table#Row_labels Row Pivots}. - * @param {Array} [config.column_pivot] An array of column names + * @param {Array} [config.column_pivot] An array of column names * to use as {@link https://en.wikipedia.org/wiki/Pivot_table#Column_labels Column Pivots}. * @param {Array} [config.aggregate] An Array of Aggregate configuration objects, - * each of which should provide an "name" and "op" property, repsresnting the string + * each of which should provide an "name" and "op" property, repsresnting the string * aggregation type and associated column name, respectively. Aggregates not provided * will use their type defaults * @param {Array>} [config.filter] An Array of Filter configurations to - * apply. A filter configuration is an array of 3 elements: A column name, + * apply. A filter configuration is an array of 3 elements: A column name, * a supported filter comparison string (e.g. '===', '>'), and a value to compare. * @param {Array} [config.sort] An Array of column names by which to sort. * @@ -851,7 +893,7 @@ table.prototype.view = function(config) { config.row_pivot = config.row_pivot || []; config.column_pivot = config.column_pivot || []; - // Column only mode + // Column only mode if (config.row_pivot.length === 0 && config.column_pivot.length > 0) { config.row_pivot = ['psp_okey']; config.column_only = true; @@ -875,7 +917,7 @@ table.prototype.view = function(config) { if (config.sort) { if (config.column_pivot.length > 0 && config.row_pivot.length > 0) { config.sort = config.sort.filter(x => config.row_pivot.indexOf(x) === -1); - } + } sort = config.sort.map(x => [config.aggregate.map(agg => agg.column).indexOf(x), 1]); } @@ -996,19 +1038,19 @@ table.prototype.view = function(config) { * Updates the rows of a {@link table}. Updated rows are pushed down to any * derived {@link view} objects. * - * @param {Object|Array|string} data The input data + * @param {Object|Array|string} data The input data * for this table. The supported input types mirror the constructor options, minus - * the ability to pass a schema (Object) as this table has. + * the ability to pass a schema (Object) as this table has. * already been constructed, thus its types are set in stone. - * + * * @see {@link table} - */ + */ table.prototype.update = function (data) { let pdata; let cols = this._columns(); let schema = this.gnode.get_tblschema(); let types = schema.types(); - + if (data instanceof ArrayBuffer) { pdata = load_arrow_buffer(data, cols, types); } @@ -1222,8 +1264,8 @@ const perspective = { worker: function () {}, /** - * A factory method for constructing {@link table}s. - * + * A factory method for constructing {@link table}s. + * * @example * // Creating a table directly from node * var table = perspective.table([{x: 1}, {x: 2}]); @@ -1232,12 +1274,12 @@ const perspective = { * // Creating a table from a Web Worker (instantiated via the worker() method). * var table = worker.table([{x: 1}, {x: 2}]); * - * @param {Object|Object|Array|string} data The input data - * for this table. When supplied an Object with string values, an empty - * table is returned using this Object as a schema. When an Object with - * Array values is supplied, a table is returned using this object's + * @param {Object|Object|Array|string} data The input data + * for this table. When supplied an Object with string values, an empty + * table is returned using this Object as a schema. When an Object with + * Array values is supplied, a table is returned using this object's * key/value pairs as name/columns respectively. When an Array is supplied, - * a table is constructed using this Array's objects as rows. When + * a table is constructed using this Array's objects as rows. When * a string is supplied, the parameter as parsed as a CSV. * @param {Object} [options] An optional options dictionary. * @param {string} options.index The name of the column in the resulting