diff --git a/.travis.yml b/.travis.yml index 1f87f5b52c..93183e20c0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,6 @@ node_js: cache: directories: - node_modules - - obj - packages/perspective/node_modules - packages/perspective-examples/node_modules - packages/perspective-jupyterlab/node_modules diff --git a/CHANGELOG.md b/CHANGELOG.md index cff3b87ad5..cd7fd52559 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [0.2.8] - 2018-11-21 +### Added +* [#317](https://github.com/jpmorganchase/perspective/pull/317) Applying 'column-pivots' now preserves the sort order. +* [#319](https://github.com/jpmorganchase/perspective/pull/319) Sorting by a column in 'column-pivots' will apply the sort to column order. + +### Fixes +* [#306](https://github.com/jpmorganchase/perspective/pull/306) Fixed Jupyterlab plugin, updating it to work with the newest [perspective-python 0.1.1](https://github.com/timkpaine/perspective-python/tree/v0.1.1). + ## [0.2.7] - 2018-11-12 ### Fixes * [#304](https://github.com/jpmorganchase/perspective/pull/304) Fixed missing file in NPM package. diff --git a/docker/emsdk/Dockerfile b/docker/emsdk/Dockerfile index fbfe9bb5cf..9de8e346bf 100644 --- a/docker/emsdk/Dockerfile +++ b/docker/emsdk/Dockerfile @@ -1,90 +1,83 @@ # Liberally copied from trzeci/emscripten -ARG EMSCRIPTEN_SDK=sdk-tag-1.38.11-64bit -FROM trzeci/emscripten-slim:${EMSCRIPTEN_SDK} - -# ------------------------------------------------------------------------------ - -ARG VCS_REF -ARG BUILD_DATE -ARG EMSCRIPTEN_SDK - -# ------------------------------------------------------------------------------ - -RUN echo "\n## Start building" \ - \ -&& echo "\n## Update and install packages" \ -&& apt-get -qq -y update && apt-get -qq install -y --no-install-recommends \ - wget \ - curl \ - zip \ - unzip \ - git \ - ssh-client \ - ca-certificates \ - build-essential \ - libboost-all-dev \ - make \ - ant \ - libidn11 \ - \ -&& echo "\n## Installing JRE 8" \ -&& echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list \ -&& apt-get -qq -y update && apt-get -qq install -y --no-install-recommends -t jessie-backports openjdk-8-jre-headless \ - \ -&& echo "\n## Installing CMake" \ -&& wget https://cmake.org/files/v3.7/cmake-3.7.2-Linux-x86_64.sh -q \ -&& mkdir /opt/cmake \ -&& printf "y\nn\n" | sh cmake-3.7.2-Linux-x86_64.sh --prefix=/opt/cmake > /dev/null \ -&& rm -fr cmake*.sh /opt/cmake/doc \ -&& rm -fr /opt/cmake/bin/cmake-gui \ -&& rm -fr /opt/cmake/bin/ccmake \ -&& rm -fr /opt/cmake/bin/cpack \ -&& ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake \ -&& ln -s /opt/cmake/bin/ctest /usr/local/bin/ctest \ - \ -&& printf "JAVA='$(which java)'\n" >> $EM_CONFIG \ - \ -&& sleep 2 \ -&& touch ${EM_CONFIG}_sanity \ - \ -&& emcc --version \ - \ -&& emcc --clear-cache --clear-ports \ - \ -&& echo "\n## Compile sample code" \ -&& mkdir -p /tmp/emscripten_test && cd /tmp/emscripten_test \ -&& printf '#include \nint main(){std::cout<<"HELLO FROM DOCKER C++"< test.cpp \ -&& em++ -O2 test.cpp -o test.js && nodejs test.js \ -&& em++ test.cpp -o test.js && nodejs test.js \ -&& em++ test.cpp -o test.js --closure 1 && nodejs test.js \ - \ -&& cd / \ -&& rm -fr /tmp/emscripten_test \ - \ -&& echo "\n## Moving Boost locally" \ - \ -&& mkdir /boost_includes \ -&& cp -r /usr/include/boost /boost_includes/ \ - \ -&& echo "\n## Cleaning up" \ -&& apt-mark manual make openjdk-8-jre-headless wget gcc git \ -&& apt-get -y remove openjdk-7-jre-headless \ -&& apt-get -y clean \ -&& apt-get -y autoclean \ -&& apt-get -y autoremove \ - \ -&& rm -rf /var/lib/apt/lists/* \ -&& rm -rf /var/cache/debconf/*-old \ -&& rm -rf /usr/share/doc/* \ -&& rm -rf /usr/share/man/?? \ -&& rm -rf /usr/share/man/??_* \ -&& cp -R /usr/share/locale/en\@* /tmp/ && rm -rf /usr/share/locale/* && mv /tmp/en\@* /usr/share/locale/ \ - \ -&& echo "\n## Installed packages:" \ -&& apt-mark showmanual | xargs -I % sh -c 'echo " * \`%\` : **"$(xargs dpkg -s % | grep Version | sed "s/Version: //")"**"' \ - \ -&& chmod -R 777 ${EM_DATA} \ -&& echo "\n## Done" +FROM trzeci/emscripten-slim:sdk-tag-1.38.20-64bit + +RUN apt-get -qq -y update +RUN apt-get -qq install -y --no-install-recommends \ +wget \ +curl \ +zip \ +unzip \ +git \ +ssh-client \ +ca-certificates \ +build-essential \ +libboost-all-dev \ +make \ +ant \ +libidn11 + + +RUN echo "deb http://http.debian.net/debian jessie-backports main" >> /etc/apt/sources.list +RUN apt-get -qq -y update && apt-get -qq install -y --no-install-recommends -t jessie-backports openjdk-8-jre-headless + +RUN wget https://cmake.org/files/v3.7/cmake-3.7.2-Linux-x86_64.sh -q +RUN mkdir /opt/cmake +RUN printf "y\nn\n" | sh cmake-3.7.2-Linux-x86_64.sh --prefix=/opt/cmake > /dev/null +RUN rm -fr cmake*.sh /opt/cmake/doc +RUN rm -fr /opt/cmake/bin/cmake-gui +RUN rm -fr /opt/cmake/bin/ccmake +RUN rm -fr /opt/cmake/bin/cpack +RUN ln -s /opt/cmake/bin/cmake /usr/local/bin/cmake +RUN ln -s /opt/cmake/bin/ctest /usr/local/bin/ctest + +RUN printf "JAVA='$(which java)'\n" >> $EM_CONFIG + +RUN sleep 2 +RUN touch ${EM_CONFIG}_sanity + +RUN emcc --version + +RUN emcc --clear-cache --clear-ports + +RUN mkdir -p /tmp/emscripten_test +RUN cd /tmp/emscripten_test +RUN printf '#include \nint main(){std::cout<<"HELLO FROM DOCKER C++"< test.cpp + +RUN em++ -O2 test.cpp -o test.js +RUN node test.js + +RUN em++ test.cpp -o test.js +RUN node test.js + +RUN em++ test.cpp -o test.js --closure 1 +RUN node test.js + +RUN cd / +RUN rm -fr /tmp/emscripten_test + +RUN mkdir /boost_includes +RUN cp -r /usr/include/boost /boost_includes/ + +RUN apt-mark manual make openjdk-8-jre-headless wget gcc git +RUN apt-get -y remove openjdk-7-jre-headless +RUN apt-get -y clean +RUN apt-get -y autoclean +RUN apt-get -y autoremove + +RUN rm -rf /var/lib/apt/lists/* +RUN rm -rf /var/cache/debconf/*-old +RUN rm -rf /usr/share/doc/* +RUN rm -rf /usr/share/man/?? +RUN rm -rf /usr/share/man/??_* +RUN cp -R /usr/share/locale/en\@* /tmp/ +RUN rm -rf /usr/share/locale/* +RUN mv /tmp/en\@* /usr/share/locale/ + +RUN chmod -R 777 ${EM_DATA} + +ENV PATH="/emsdk_portable/node/bin:${PATH}" +RUN npm install --global yarn +RUN yarn --version ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64/jre \ No newline at end of file diff --git a/docs/installation.md b/docs/installation.md index 3ff34b353d..519539cf45 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -47,6 +47,7 @@ library can be used directly from CDN by simply adding these scripts to your `.html`'s `` section: ```html + @@ -75,3 +76,26 @@ Doing so is quite a bit more complex than a standard pure Javascript NPM package, so if you're not looking to hack on Perspective itself, you are likely better off choosing the CDN or NPM methods above. See the [developer docs](development.html) for details. + +## Jupyterlab + +Perspective comes bundled with a complete Jupyterlab plugin which can be +accessed from Python via the complementary +[`perspective-python`](https://github.com/timkpaine/perspective-python) +package. `perspective-python` implements mostly the same API as +``, and works with static `pandas.DataFrame` objects as well +as streaming incremental updates via the `update()` method (as in Javascript). + + + +You'll need to install both to utilize Perspective from Python in Jupyterlab. +Assuming you've already installed the latter 3, you can install the Perspective +plugin as below, or follow the install from source instructions from the +`perspective-python` +[documentation](https://perspective-python.readthedocs.io/en/latest/index.html). + +```bash +pip install perspective-python +jupyter labextension install @jpmorganchase/perspective-jupyterlab +``` + diff --git a/examples/cli/package.json b/examples/cli/package.json index a239f15f45..c0b4fe27d3 100644 --- a/examples/cli/package.json +++ b/examples/cli/package.json @@ -1,7 +1,7 @@ { "name": "cli", "private": true, - "version": "0.2.7", + "version": "0.2.8", "description": "An example of Remote Perspective as a CLI tool.", "scripts": { "start": "node server.js" @@ -12,9 +12,9 @@ "perspective-cli": "server.js" }, "dependencies": { - "@jpmorganchase/perspective": "^0.2.7", - "@jpmorganchase/perspective-viewer": "^0.2.7", - "@jpmorganchase/perspective-viewer-highcharts": "^0.2.7", - "@jpmorganchase/perspective-viewer-hypergrid": "^0.2.7" + "@jpmorganchase/perspective": "^0.2.8", + "@jpmorganchase/perspective-viewer": "^0.2.8", + "@jpmorganchase/perspective-viewer-highcharts": "^0.2.8", + "@jpmorganchase/perspective-viewer-hypergrid": "^0.2.8" } } diff --git a/examples/git_history/package.json b/examples/git_history/package.json index 57761f3d80..162657ccf5 100644 --- a/examples/git_history/package.json +++ b/examples/git_history/package.json @@ -1,7 +1,7 @@ { "name": "git-history", "private": true, - "version": "0.2.7", + "version": "0.2.8", "description": "An example of Perspective's own GIT history rendered in Perspective.", "scripts": { "start": "node server.js" @@ -9,9 +9,9 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@jpmorganchase/perspective": "^0.2.7", - "@jpmorganchase/perspective-viewer": "^0.2.7", - "@jpmorganchase/perspective-viewer-highcharts": "^0.2.7", - "@jpmorganchase/perspective-viewer-hypergrid": "^0.2.7" + "@jpmorganchase/perspective": "^0.2.8", + "@jpmorganchase/perspective-viewer": "^0.2.8", + "@jpmorganchase/perspective-viewer-highcharts": "^0.2.8", + "@jpmorganchase/perspective-viewer-hypergrid": "^0.2.8" } } diff --git a/examples/simple/package.json b/examples/simple/package.json index b5cc53ea1b..bcca4f5f4f 100644 --- a/examples/simple/package.json +++ b/examples/simple/package.json @@ -1,7 +1,7 @@ { "name": "simple", "private": true, - "version": "0.2.7", + "version": "0.2.8", "description": "A collection of simple client-side Perspective examples.", "scripts": { "start": "node server.js" @@ -9,9 +9,9 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@jpmorganchase/perspective": "^0.2.7", - "@jpmorganchase/perspective-viewer": "^0.2.7", - "@jpmorganchase/perspective-viewer-highcharts": "^0.2.7", - "@jpmorganchase/perspective-viewer-hypergrid": "^0.2.7" + "@jpmorganchase/perspective": "^0.2.8", + "@jpmorganchase/perspective-viewer": "^0.2.8", + "@jpmorganchase/perspective-viewer-highcharts": "^0.2.8", + "@jpmorganchase/perspective-viewer-hypergrid": "^0.2.8" } } diff --git a/examples/webpack/package.json b/examples/webpack/package.json index 2f1ce65362..637fb9478e 100644 --- a/examples/webpack/package.json +++ b/examples/webpack/package.json @@ -1,7 +1,7 @@ { "name": "webpack", "private": true, - "version": "0.2.7", + "version": "0.2.8", "description": "An example of using the Perspective Webpack plugin to build a JS file with Webpack.", "scripts": { "start": "node server.js", @@ -10,7 +10,7 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@jpmorganchase/perspective": "^0.2.7", - "@jpmorganchase/perspective-viewer": "^0.2.7" + "@jpmorganchase/perspective": "^0.2.8", + "@jpmorganchase/perspective-viewer": "^0.2.8" } } diff --git a/lerna.json b/lerna.json index 93eb053c71..28adde77c5 100644 --- a/lerna.json +++ b/lerna.json @@ -4,5 +4,5 @@ "packages/*", "examples/*" ], - "version": "0.2.7" + "version": "0.2.8" } diff --git a/package.json b/package.json index b3dc130fcf..4bce1b4814 100644 --- a/package.json +++ b/package.json @@ -79,7 +79,6 @@ "fix:less": "prettier --tab-width 4 --write packages/**/src/less/*.less", "fix:html": "html-beautify packages/**/src/html/*.html -r", "fix:json": "prettier --tab-width 4 --write **/package.json", - "fix:markdown": "prettier --tab-width 4 --write docs/usage.md docs/installation.md", "fix": "npm-run-all --silent fix:*", "_build": "lerna run build ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}} --stream", "_build_test": "lerna run test:build ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}} --stream", diff --git a/packages/perspective-jupyterlab/README.md b/packages/perspective-jupyterlab/README.md index 0f1cf89b9e..083e1d2a7f 100644 --- a/packages/perspective-jupyterlab/README.md +++ b/packages/perspective-jupyterlab/README.md @@ -12,17 +12,8 @@ This extension allows in-lining perspective based charts in jupyterlab notebooks jupyter labextension install @jpmorganchase/perspective-jupyterlab ``` -### From source - -First build perspective ala the perspective documentation, then from inside the -jlab directory: - -```bash -jupyter labextension install . -``` - ### PIP ```bash -pip instal perspective-python -``` \ No newline at end of file +pip install perspective-python +``` diff --git a/packages/perspective-jupyterlab/package.json b/packages/perspective-jupyterlab/package.json index cb9cb13fa7..d17c1671d7 100644 --- a/packages/perspective-jupyterlab/package.json +++ b/packages/perspective-jupyterlab/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-jupyterlab", - "version": "0.2.7", + "version": "0.2.8", "description": "Perspective.js", "files": [ "build/*.d.ts", @@ -29,20 +29,13 @@ "clean": "rimraf build" }, "dependencies": { - "@jpmorganchase/perspective": "^0.2.7", - "@jpmorganchase/perspective-viewer": "^0.2.7", - "@jpmorganchase/perspective-viewer-highcharts": "^0.2.7", - "@jpmorganchase/perspective-viewer-hypergrid": "^0.2.7", + "@jpmorganchase/perspective": "^0.2.8", + "@jpmorganchase/perspective-viewer": "^0.2.8", + "@jpmorganchase/perspective-viewer-highcharts": "^0.2.8", + "@jpmorganchase/perspective-viewer-hypergrid": "^0.2.8", "@jupyter-widgets/base": "^1.1.10", - "@jupyterlab/application": "^0.19.0", - "@jupyterlab/apputils": "^0.19.0", - "@jupyterlab/rendermime-interfaces": "^1.2.0", - "@jupyterlab/services": "^3.2.0", "@phosphor/application": "^1.5.0", - "@phosphor/messaging": "^1.2.2", - "@phosphor/widgets": "^1.5.0", - "@types/socket.io-client": "^1.4.0", - "socket.io": "2.0.3" + "@phosphor/widgets": "^1.5.0" }, "jupyterlab": { "extension": "build/index.js" diff --git a/packages/perspective-jupyterlab/src/config/mimerenderer.config.js b/packages/perspective-jupyterlab/src/config/mimerenderer.config.js index 5e33bc3d4d..6f926b071a 100644 --- a/packages/perspective-jupyterlab/src/config/mimerenderer.config.js +++ b/packages/perspective-jupyterlab/src/config/mimerenderer.config.js @@ -1,6 +1,6 @@ /****************************************************************************** * - * Copyright (c) 2017, the Perspective Authors. + * Copyright (c) 2018, the Perspective Authors. * * This file is part of the Perspective library, distributed under the terms of * the Apache License 2.0. The full license can be found in the LICENSE file. diff --git a/packages/perspective-jupyterlab/src/config/plugin.config.js b/packages/perspective-jupyterlab/src/config/plugin.config.js index 37bf0ed23a..52c44b2a0a 100644 --- a/packages/perspective-jupyterlab/src/config/plugin.config.js +++ b/packages/perspective-jupyterlab/src/config/plugin.config.js @@ -1,6 +1,6 @@ /****************************************************************************** * - * Copyright (c) 2017, the Perspective Authors. + * Copyright (c) 2018, the Perspective Authors. * * This file is part of the Perspective library, distributed under the terms of * the Apache License 2.0. The full license can be found in the LICENSE file. @@ -13,7 +13,12 @@ const webpack = require("webpack"); module.exports = { entry: "./src/ts/index.ts", - externals: ["@jupyter-widgets/base"], + resolveLoader: { + alias: { + "file-worker-loader": "@jpmorganchase/perspective/src/loader/file_worker_loader.js" + } + }, + externals: /\@jupyter|\@phosphor/, plugins: [new webpack.ContextReplacementPlugin(/moment[\/\\]locale$/, /(en|es|fr)$/), new PerspectivePlugin()], module: { rules: [ diff --git a/packages/perspective-jupyterlab/src/less/material.less b/packages/perspective-jupyterlab/src/less/material.less index 9eee39d6b6..ddbe61f3a7 100644 --- a/packages/perspective-jupyterlab/src/less/material.less +++ b/packages/perspective-jupyterlab/src/less/material.less @@ -1,6 +1,6 @@ /****************************************************************************** * - * Copyright (c) 2017, the Perspective Authors. + * Copyright (c) 2018, the Perspective Authors. * * This file is part of the Perspective library, distributed under the terms of * the Apache License 2.0. The full license can be found in the LICENSE file. @@ -16,6 +16,7 @@ div.jp-PSPContainer { overflow: auto; resize: both; padding-right: 20px; + padding-bottom: 20px; height: 450px; } diff --git a/packages/perspective-jupyterlab/src/ts/arraybuffer.d.ts b/packages/perspective-jupyterlab/src/ts/arraybuffer.d.ts new file mode 100644 index 0000000000..761f201b09 --- /dev/null +++ b/packages/perspective-jupyterlab/src/ts/arraybuffer.d.ts @@ -0,0 +1,11 @@ +/****************************************************************************** + * + * Copyright (c) 2018, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ + +declare module 'arraybuffer-loader!*'; +declare module 'file-worker-loader?inline=true!*'; diff --git a/packages/perspective-jupyterlab/src/ts/index.ts b/packages/perspective-jupyterlab/src/ts/index.ts index e8bef82194..eea847a754 100644 --- a/packages/perspective-jupyterlab/src/ts/index.ts +++ b/packages/perspective-jupyterlab/src/ts/index.ts @@ -1,6 +1,6 @@ /****************************************************************************** * - * Copyright (c) 2017, the Perspective Authors. + * Copyright (c) 2018, the Perspective Authors. * * This file is part of the Perspective library, distributed under the terms of * the Apache License 2.0. The full license can be found in the LICENSE file. diff --git a/packages/perspective-jupyterlab/src/ts/mimerenderer.ts b/packages/perspective-jupyterlab/src/ts/mimerenderer.ts index 4f8ac636e1..956f4f015a 100644 --- a/packages/perspective-jupyterlab/src/ts/mimerenderer.ts +++ b/packages/perspective-jupyterlab/src/ts/mimerenderer.ts @@ -1,6 +1,6 @@ /****************************************************************************** * - * Copyright (c) 2017, the Perspective Authors. + * Copyright (c) 2018, the Perspective Authors. * * This file is part of the Perspective library, distributed under the terms of * the Apache License 2.0. The full license can be found in the LICENSE file. diff --git a/packages/perspective-jupyterlab/src/ts/plugin.ts b/packages/perspective-jupyterlab/src/ts/plugin.ts index b69dfaf634..1f5c169699 100644 --- a/packages/perspective-jupyterlab/src/ts/plugin.ts +++ b/packages/perspective-jupyterlab/src/ts/plugin.ts @@ -1,6 +1,6 @@ /****************************************************************************** * - * Copyright (c) 2017, the Perspective Authors. + * Copyright (c) 2018, the Perspective Authors. * * This file is part of the Perspective library, distributed under the terms of * the Apache License 2.0. The full license can be found in the LICENSE file. diff --git a/packages/perspective-jupyterlab/src/ts/utils.ts b/packages/perspective-jupyterlab/src/ts/utils.ts index 5fffeb2aef..e6994cf5de 100644 --- a/packages/perspective-jupyterlab/src/ts/utils.ts +++ b/packages/perspective-jupyterlab/src/ts/utils.ts @@ -1,14 +1,12 @@ /****************************************************************************** * - * Copyright (c) 2017, the Perspective Authors. + * Copyright (c) 2018, the Perspective Authors. * * This file is part of the Perspective library, distributed under the terms of * the Apache License 2.0. The full license can be found in the LICENSE file. * */ -import {Clipboard} from '@jupyterlab/apputils'; - /* defines */ export const MIME_TYPE = 'application/psp+json'; @@ -21,163 +19,3 @@ const PSP_CONTAINER_CLASS = 'jp-PSPContainer'; export const PSP_CONTAINER_CLASS_DARK = 'jp-PSPContainer-dark'; - -export function datasourceToSource(source: string){ - if(source.indexOf('comm://') !== -1){ - return 'comm'; - } else{ - return 'static'; - } -} - -export function convertToCSV(objArray: Array): string { - //https://medium.com/@danny.pule/export-json-to-csv-file-using-javascript-a0b7bc5b00d2 - var array = typeof objArray != 'object' ? JSON.parse(objArray) : objArray; - var str = ''; - - for (var i = 0; i < array.length; i++) { - var header = ''; - var line = ''; - for (var index in array[i]) { - if (i === 0){ - if (header != ''){ - header += ', '; - } - header += index; - } - if (line != ''){ - line += ', '; - } - line += array[i][index]; - } - - if (i === 0){ - str += header + '\r\n'; - } - str += line + '\r\n'; - } - - return str; -} - -export -function createCopyDl(psp: any) : {[key:string]:HTMLElement;} { - let copy_button = document.createElement('button'); - copy_button.textContent = 'copy'; - let dl_button = document.createElement('button'); - dl_button.textContent = 'download'; - - let viewtype: string; - let data: any; - let height: number; - let width: number; - let png: any; - let canvas: HTMLCanvasElement; - let img: any; - - copy_button.onmousedown = () => { - viewtype = psp.getAttribute('view'); - if (viewtype === 'hypergrid') { - psp._view.to_json().then((dat: Array) => { - data = dat; - }); - } else { - let svg = psp.querySelector('svg') as SVGSVGElement; - height = svg.height.baseVal.value; - width = svg.width.baseVal.value; - data = new XMLSerializer().serializeToString(svg); - canvas = document.createElement('canvas') as HTMLCanvasElement; - canvas.height = height; - canvas.width = width; - - let ctx = canvas.getContext("2d"); - let DOMURL = window.URL || (window as any).webkitURL || window; - img = new Image(); - let svg2 = new Blob([data], {type: "image/svg+xml;charset=utf-8"}); - - let url = DOMURL.createObjectURL(svg2); - img.onload = function() { - ctx.drawImage(img, 0, 0, width, height); - png = canvas.toDataURL("image/png"); - }; - img.src = url; - } - } - - dl_button.onmousedown = copy_button.onmousedown; - - copy_button.onclick = () => { - let copied = false; - let timeout = 100; - while (timeout<=16000){ - setTimeout(() => { - if(copied){ - return; - } - if (data) { - if(viewtype === 'hypergrid'){ - Clipboard.copyToSystem(convertToCSV(data)); - } else { - if (!png){ - //not ready - return - } - canvas.focus(); - document.execCommand("copy"); - } - copied = true; - } else if (timeout == 16000){ - console.error('Timeout waiting for perspective!'); - } - }, timeout); - timeout *= 2; - } - }; - - dl_button.onclick = () => { - let dl = false; - let timeout = 100; - while (timeout<=16000){ - setTimeout(() => { - if(dl){ - return; - } - if (data) { - if(viewtype === 'hypergrid'){ - let csv = convertToCSV(data); - let csvContent = "data:text/csv;charset=utf-8," + csv; - let DOMURL = window.URL || (window as any).webkitURL || window; - var a = document.createElement('a'); - a.download = 'psp.csv'; - a.target = "_blank"; - a.href = csvContent; - document.body.appendChild(a); - a.click(); - a.remove(); - DOMURL.revokeObjectURL(png); - } else { - if (!png){ - //not ready - return - } - let DOMURL = window.URL || (window as any).webkitURL || window; - var a = document.createElement('a'); - a.download = 'psp.png'; - a.target = "_blank"; - a.href = png; - document.body.appendChild(a); - a.click(); - a.remove(); - DOMURL.revokeObjectURL(png); - } - dl = true; - } else if (timeout == 16000){ - console.error('Timeout waiting for perspective!'); - } - }, timeout); - timeout *= 2; - } - }; - - return {copy: copy_button, dl:dl_button}; -} diff --git a/packages/perspective-jupyterlab/src/ts/version.ts b/packages/perspective-jupyterlab/src/ts/version.ts index 440d6eea03..de59b14954 100644 --- a/packages/perspective-jupyterlab/src/ts/version.ts +++ b/packages/perspective-jupyterlab/src/ts/version.ts @@ -1,6 +1,6 @@ /****************************************************************************** * - * Copyright (c) 2017, the Perspective Authors. + * Copyright (c) 2018, the Perspective Authors. * * This file is part of the Perspective library, distributed under the terms of * the Apache License 2.0. The full license can be found in the LICENSE file. @@ -8,4 +8,4 @@ */ export -const PERSPECTIVE_VERSION = '0.1.18'; \ No newline at end of file +const PERSPECTIVE_VERSION = '0.2.8'; \ No newline at end of file diff --git a/packages/perspective-jupyterlab/src/ts/widget.ts b/packages/perspective-jupyterlab/src/ts/widget.ts index d520b00c56..f1fe796e1b 100644 --- a/packages/perspective-jupyterlab/src/ts/widget.ts +++ b/packages/perspective-jupyterlab/src/ts/widget.ts @@ -1,6 +1,6 @@ /****************************************************************************** * - * Copyright (c) 2017, the Perspective Authors. + * Copyright (c) 2018, the Perspective Authors. * * This file is part of the Perspective library, distributed under the terms of * the Apache License 2.0. The full license can be found in the LICENSE file. @@ -11,24 +11,25 @@ import { DOMWidgetModel, DOMWidgetView, ISerializers } from '@jupyter-widgets/base'; -import {Session} from '@jupyterlab/services'; - /* defines */ import {MIME_TYPE, PSP_CLASS, PSP_CONTAINER_CLASS, PSP_CONTAINER_CLASS_DARK} from './utils.ts'; import {PERSPECTIVE_VERSION} from './version.ts'; -/* Helper methods */ -import { - datasourceToSource, createCopyDl -} from './utils.ts'; - /* perspective components */ import "@jpmorganchase/perspective-viewer"; import "@jpmorganchase/perspective-viewer-hypergrid"; import "@jpmorganchase/perspective-viewer-highcharts"; + +import perspective from "@jpmorganchase/perspective"; +import * as wasm from "arraybuffer-loader!@jpmorganchase/perspective/build/psp.async.wasm"; +import * as worker from "file-worker-loader?inline=true!@jpmorganchase/perspective/build/perspective.wasm.worker.js"; + +perspective.override({wasm, worker}); + + /* css */ -import '../less/material.less'; +import '!!style-loader!css-loader!less-loader!../less/material.less'; export class PerspectiveModel extends DOMWidgetModel { @@ -50,7 +51,11 @@ class PerspectiveModel extends DOMWidgetModel { columnpivots: [], aggregates: [], sort: [], + index: '', + limit: -1, + computedcolumns: [], settings: false, + embed: false, dark: false }; } @@ -72,6 +77,7 @@ class PerspectiveModel extends DOMWidgetModel { export class PerspectiveView extends DOMWidgetView { private psp: any; + private embed = false; render() { this.psp = Private.createNode(this.el); @@ -86,18 +92,14 @@ class PerspectiveView extends DOMWidgetView { this.model.on('change:columnpivots', this.columnpivots_changed, this); this.model.on('change:aggregates', this.aggregates_changed, this); this.model.on('change:sort', this.sort_changed, this); + this.model.on('change:computedcolumns', this.computedcolumns_changed, this); this.model.on('change:settings', this.settings_changed, this); + this.model.on('change:embed', this.embed_changed, this); this.model.on('change:dark', this.dark_changed, this); this.model.on('msg:custom', this._update, this); this.displayed.then(()=> { - let data = this.model.get('_data'); - if (data.length > 0){ - this.psp.update(this.model.get('_data')); - } else { - this.datasrc_changed(); - } this.settings_changed(); this.dark_changed(); this.view_changed(); @@ -112,6 +114,7 @@ class PerspectiveView extends DOMWidgetView { // do aggregates after columns this.aggregates_changed(); + this.datasrc_changed(); }); } @@ -133,7 +136,18 @@ class PerspectiveView extends DOMWidgetView { let data = this.model.get('_data'); if (Object.keys(schema).length > 0 ){ - this.psp.load(schema); + let limit = this.model.get('limit'); + let index = this.model.get('index'); + let options = {} as {[key: string]: any}; + + if (limit > 0){ + options['limit'] = limit; + } + if (index){ + options['index'] = index; + } + + this.psp.load(schema, options); } if (data.length > 0){ this.psp.update(this.model.get('_data')); @@ -142,36 +156,7 @@ class PerspectiveView extends DOMWidgetView { datasrc_changed(){ this.psp.delete(); - let type = datasourceToSource(this.model.get('datasrc')); - - if (type === 'static') { - this.data_changed(); - } else if (type === 'comm') { - //grab session id - let els = this.model.get('datasrc').replace('comm://', '').split('/'); - let kernelId = els[0]; - let name = els[1]; - let channel = els[2]; - - Session.listRunning().then(sessionModels => { - for (let i=0; i { - let dat = msg['content']['data']; - let tmp = JSON.parse(dat); - this.psp.update(tmp); - }; - comm.onClose = (msg: any) => {}; - } - } - }); - } else { - throw new Error('Source not recognized!'); - } + this.data_changed(); } schema_changed(){ @@ -193,7 +178,7 @@ class PerspectiveView extends DOMWidgetView { columns_changed(){ let columns = this.model.get('columns'); if(columns.length > 0){ - this.psp.setAttribute('columns', JSON.stringify(this.model.get('columns'))); + this.psp.setAttribute('columns', JSON.stringify(columns)); } else { this.psp.removeAttribute('columns'); } @@ -215,10 +200,35 @@ class PerspectiveView extends DOMWidgetView { this.psp.setAttribute('sort', JSON.stringify(this.model.get('sort'))); } + computedcolumns_changed(){ + let computedcolumns = this.model.get('computedcolumns'); + if(computedcolumns.length > 0){ + this.psp.setAttribute('computed-columns', JSON.stringify(computedcolumns)); + } else { + this.psp.removeAttribute('computed-columns'); + } + } + + limit_changed(){ + let limit = this.model.get('limit'); + if(limit > 0){ + this.psp.setAttribute('limit', limit); + } else { + this.psp.removeAttribute('limit'); + } + } + settings_changed(){ this.psp.setAttribute('settings', this.model.get('settings')); } + embed_changed(){ + this.embed = this.model.get('embed'); + if(this.embed){ + console.log('Warning: embed not implemented'); + } + } + dark_changed(){ let dark = this.model.get('dark'); if(dark){ @@ -243,19 +253,24 @@ namespace Private { psp.className = PSP_CLASS; psp.setAttribute('type', MIME_TYPE); - let btns = createCopyDl(psp); - while(node.lastChild){ node.removeChild(node.lastChild); } node.appendChild(psp); + // allow perspective's event handlers to do their work + psp.addEventListener( 'contextmenu', stop, false ); + psp.addEventListener( 'mousedown', stop, false ); + psp.addEventListener( 'mousedown', stop, false ); + + function stop( event: MouseEvent ) { + event.stopPropagation(); + } + let div = document.createElement('div'); div.style.setProperty('display', 'flex'); div.style.setProperty('flex-direction', 'row'); - div.appendChild(btns['copy']); - div.appendChild(btns['dl']); node.appendChild(div); return psp; } diff --git a/packages/perspective-viewer-highcharts/package.json b/packages/perspective-viewer-highcharts/package.json index 2075445378..4a82cb58f6 100644 --- a/packages/perspective-viewer-highcharts/package.json +++ b/packages/perspective-viewer-highcharts/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-viewer-highcharts", - "version": "0.2.7", + "version": "0.2.8", "description": "Perspective.js", "main": "src/js/highcharts.js", "files": [ @@ -37,8 +37,8 @@ "author": "", "license": "Apache", "dependencies": { - "@jpmorganchase/perspective": "^0.2.7", - "@jpmorganchase/perspective-viewer": "^0.2.7", + "@jpmorganchase/perspective": "^0.2.8", + "@jpmorganchase/perspective-viewer": "^0.2.8", "babel-runtime": "^6.26.0", "chroma-js": "^1.3.4", "gradient-parser": "0.1.5", diff --git a/packages/perspective-viewer-highcharts/src/js/externals.js b/packages/perspective-viewer-highcharts/src/js/externals.js index 48762866ea..6b491b1c5f 100644 --- a/packages/perspective-viewer-highcharts/src/js/externals.js +++ b/packages/perspective-viewer-highcharts/src/js/externals.js @@ -122,8 +122,8 @@ Highcharts.setOptions({ (!point.value // LINE CHANGED ? nullColor : colorAxis && value !== undefined - ? colorAxis.toColor(value, point) - : point.color || series.color); + ? colorAxis.toColor(value, point) + : point.color || series.color); if (color) { point.color = color; diff --git a/packages/perspective-viewer-hypergrid/package.json b/packages/perspective-viewer-hypergrid/package.json index 4c87968c4d..544c3c9dd9 100644 --- a/packages/perspective-viewer-hypergrid/package.json +++ b/packages/perspective-viewer-hypergrid/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective-viewer-hypergrid", - "version": "0.2.7", + "version": "0.2.8", "description": "Perspective.js", "main": "src/js/hypergrid.js", "files": [ @@ -36,8 +36,8 @@ "author": "", "license": "Apache", "dependencies": { - "@jpmorganchase/perspective": "^0.2.7", - "@jpmorganchase/perspective-viewer": "^0.2.7", + "@jpmorganchase/perspective": "^0.2.8", + "@jpmorganchase/perspective-viewer": "^0.2.8", "babel-polyfill": "^6.26.0", "babel-runtime": "^6.26.0", "datasaur-local": "3.0.0", diff --git a/packages/perspective-viewer/package.json b/packages/perspective-viewer/package.json index 213bf8ce83..95e092f3f8 100644 --- a/packages/perspective-viewer/package.json +++ b/packages/perspective-viewer/package.json @@ -1,8 +1,8 @@ { "name": "@jpmorganchase/perspective-viewer", - "version": "0.2.7", + "version": "0.2.8", "description": "Perspective.js", - "main": "src/js/view.js", + "main": "src/js/viewer.js", "files": [ "build/**/*", "src/**/*", @@ -41,7 +41,7 @@ "author": "", "license": "Apache", "dependencies": { - "@jpmorganchase/perspective": "^0.2.7", + "@jpmorganchase/perspective": "^0.2.8", "@webcomponents/shadycss": "^1.5.2", "@webcomponents/webcomponentsjs": "~2.0.4", "awesomplete": "^1.1.2", diff --git a/packages/perspective-viewer/src/config/view.config.js b/packages/perspective-viewer/src/config/view.config.js index 884a55e57c..1e7e7e1b80 100644 --- a/packages/perspective-viewer/src/config/view.config.js +++ b/packages/perspective-viewer/src/config/view.config.js @@ -2,10 +2,9 @@ const path = require("path"); const common = require("@jpmorganchase/perspective/src/config/common.config.js"); module.exports = Object.assign({}, common(), { - entry: "./src/js/view.js", + entry: "./src/js/viewer.js", output: { filename: "perspective.view.js", - library: "perspective-view", libraryTarget: "umd", path: path.resolve(__dirname, "../../build") } diff --git a/packages/perspective-viewer/src/html/view.html b/packages/perspective-viewer/src/html/viewer.html similarity index 100% rename from packages/perspective-viewer/src/html/view.html rename to packages/perspective-viewer/src/html/viewer.html diff --git a/packages/perspective-viewer/src/js/computed_column.js b/packages/perspective-viewer/src/js/computed_column.js index fe07496418..af6389a618 100644 --- a/packages/perspective-viewer/src/js/computed_column.js +++ b/packages/perspective-viewer/src/js/computed_column.js @@ -7,19 +7,15 @@ * */ -import {polyfill} from "mobile-drag-drop"; - import {bindTemplate} from "./utils.js"; -import State from "./computed_column/State.js"; -import Computation from "./computed_column/Computation.js"; +import State from "./computed_column/state.js"; +import Computation from "./computed_column/computation.js"; import template from "../html/computed_column.html"; import style from "../less/computed_column.less"; -import {disallow_drop} from "./dragdrop.js"; - -polyfill({}); +import {disallow_drop} from "./viewer/dragdrop.js"; // Computations const hour_of_day = function(val) { diff --git a/packages/perspective-viewer/src/js/computed_column/Computation.js b/packages/perspective-viewer/src/js/computed_column/computation.js similarity index 100% rename from packages/perspective-viewer/src/js/computed_column/Computation.js rename to packages/perspective-viewer/src/js/computed_column/computation.js diff --git a/packages/perspective-viewer/src/js/computed_column/State.js b/packages/perspective-viewer/src/js/computed_column/state.js similarity index 100% rename from packages/perspective-viewer/src/js/computed_column/State.js rename to packages/perspective-viewer/src/js/computed_column/state.js diff --git a/packages/perspective-viewer/src/js/row.js b/packages/perspective-viewer/src/js/row.js index ed67db2ad7..3bba9cf551 100644 --- a/packages/perspective-viewer/src/js/row.js +++ b/packages/perspective-viewer/src/js/row.js @@ -226,7 +226,8 @@ class Row extends HTMLElement { }); this._sort_order.addEventListener("click", event => { const current = this.getAttribute("sort-order"); - const order = (perspective.SORT_ORDERS.indexOf(current) + 1) % 5; + const sort_size = this.getAttribute("type") === "string" ? 3 : 5; + const order = (perspective.SORT_ORDERS.indexOf(current) + 1) % sort_size; this.setAttribute("sort-order", perspective.SORT_ORDERS[order]); this.dispatchEvent(new CustomEvent("sort-order", {detail: event})); }); diff --git a/packages/perspective-viewer/src/js/utils.js b/packages/perspective-viewer/src/js/utils.js index 79a62b8795..f8ce3ad1ab 100644 --- a/packages/perspective-viewer/src/js/utils.js +++ b/packages/perspective-viewer/src/js/utils.js @@ -125,9 +125,10 @@ export function registerElement(templateString, styleString, proto) { window.customElements.define(name, _perspective_element); } -export function bindTemplate(template, styleString) { +export function bindTemplate(template, ...styleStrings) { + const style = styleStrings.map(x => x.toString()).join("\n"); return function(cls) { - return registerElement(template, styleString, cls); + return registerElement(template, {toString: () => style}, cls); }; } diff --git a/packages/perspective-viewer/src/js/view/ViewPrivate.js b/packages/perspective-viewer/src/js/view/ViewPrivate.js deleted file mode 100644 index dc42b83f5b..0000000000 --- a/packages/perspective-viewer/src/js/view/ViewPrivate.js +++ /dev/null @@ -1,770 +0,0 @@ -/****************************************************************************** - * - * Copyright (c) 2018, the Perspective Authors. - * - * This file is part of the Perspective library, distributed under the terms of - * the Apache License 2.0. The full license can be found in the LICENSE file. - * - */ - -import "@webcomponents/webcomponentsjs"; -import "@webcomponents/shadycss/custom-style-interface.min.js"; - -import _ from "underscore"; - -import perspective from "@jpmorganchase/perspective"; -import {undrag, column_undrag, column_dragleave, column_dragover, column_drop, drop, drag_enter, allow_drop, disallow_drop} from "../dragdrop.js"; -import {column_visibility_clicked, column_aggregate_clicked, column_filter_clicked, sort_order_clicked} from "./actions.js"; -import {_show_context_menu, _toggle_config} from "./dom.js"; -import {CancelTask} from "./CancelTask.js"; -import {renderers} from "./renderers.js"; -import {COMPUTATIONS} from "../computed_column.js"; - -export class ViewPrivate extends HTMLElement { - get _plugin() { - let current_renderers = renderers.getInstance(); - let view = this.getAttribute("view"); - if (!view) { - view = Object.keys(current_renderers)[0]; - } - this.setAttribute("view", view); - return current_renderers[view] || current_renderers[Object.keys(current_renderers)[0]]; - } - - // get viewer state - _get_view_dom_columns(selector, callback) { - selector = selector || "#active_columns perspective-row"; - let columns = Array.prototype.slice.call(this.shadowRoot.querySelectorAll(selector)); - if (!callback) { - return columns; - } - return columns.map(callback); - } - - _get_view_columns({active = true} = {}) { - let selector; - if (active) { - selector = "#active_columns perspective-row"; - } else { - selector = "#inactive_columns perspective-row"; - } - return this._get_view_dom_columns(selector, col => { - return col.getAttribute("name"); - }); - } - - _get_view_aggregates(selector) { - selector = selector || "#active_columns perspective-row"; - return this._get_view_dom_columns(selector, s => { - return { - op: s.getAttribute("aggregate"), - column: s.getAttribute("name") - }; - }); - } - - _get_view_row_pivots() { - return this._get_view_dom_columns("#row_pivots perspective-row", col => { - return col.getAttribute("name"); - }); - } - - _get_view_column_pivots() { - return this._get_view_dom_columns("#column_pivots perspective-row", col => { - return col.getAttribute("name"); - }); - } - - _get_view_filters() { - return this._get_view_dom_columns("#filters perspective-row", col => { - let {operator, operand} = JSON.parse(col.getAttribute("filter")); - return [col.getAttribute("name"), operator, operand]; - }); - } - - _get_view_sorts() { - return this._get_view_dom_columns("#sort perspective-row", col => { - let order = col.getAttribute("sort-order") || "asc"; - return [col.getAttribute("name"), order]; - }); - } - - _get_view_hidden(aggregates, sort) { - aggregates = aggregates || this._get_view_aggregates(); - let hidden = []; - sort = sort || this._get_view_sorts(); - for (let s of sort) { - if (aggregates.map(agg => agg.column).indexOf(s[0]) === -1) { - hidden.push(s[0]); - } - } - return hidden; - } - - _get_visible_column_count() { - return this._get_view_dom_columns().length; - } - - // TODO: move to state_read - get_aggregate_attribute() { - const aggs = JSON.parse(this.getAttribute("aggregates")) || {}; - return Object.keys(aggs).map(col => ({column: col, op: aggs[col]})); - } - - // TODO: move to state_apply - set_aggregate_attribute(aggs) { - this.setAttribute( - "aggregates", - JSON.stringify( - aggs.reduce((obj, agg) => { - obj[agg.column] = agg.op; - return obj; - }, {}) - ) - ); - } - - // Loads a new table into perspective-viewer - async load_table(table, computed = false) { - this.shadowRoot.querySelector("#app").classList.add("hide_message"); - this.setAttribute("updating", true); - - if (this._table && !computed) { - this.removeAttribute("computed-columns"); - } - this._clear_state(); - - this._table = table; - - if (this.hasAttribute("computed-columns") && !computed) { - const computed_columns = JSON.parse(this.getAttribute("computed-columns")); - if (computed_columns.length > 0) { - for (let col of computed_columns) { - await this._create_computed_column({ - detail: { - column_name: col.name, - input_columns: col.inputs.map(x => ({name: x})), - computation: COMPUTATIONS[col.func] - } - }); - } - this._debounce_update({ignore_size_check: false}); - return; - } - } - - let [cols, schema, computed_schema] = await Promise.all([table.columns(), table.schema(), table.computed_schema()]); - - // TODO: separate DOM into helper methods? - this._inactive_columns.innerHTML = ""; - this._active_columns.innerHTML = ""; - - this._initial_col_order = cols.slice(); - if (!this.hasAttribute("columns")) { - this.setAttribute("columns", JSON.stringify(this._initial_col_order)); - } - - let type_order = {integer: 2, string: 0, float: 3, boolean: 4, datetime: 1}; - - // Sort columns by type and then name - cols.sort((a, b) => { - let s1 = type_order[schema[a]], - s2 = type_order[schema[b]]; - let r = 0; - if (s1 == s2) { - let a1 = a.toLowerCase(), - b1 = b.toLowerCase(); - r = a1 < b1 ? -1 : 1; - } else { - r = s1 < s2 ? -1 : 1; - } - return r; - }); - - // Update Aggregates. - let aggregates = []; - const found = {}; - - if (this.hasAttribute("aggregates")) { - // Double check that the persisted aggregates actually match the - // expected types. - aggregates = this.get_aggregate_attribute() - .map(col => { - let _type = schema[col.column]; - found[col.column] = true; - if (_type) { - if (col.op === "" || perspective.TYPE_AGGREGATES[_type].indexOf(col.op) === -1) { - col.op = perspective.AGGREGATE_DEFAULTS[_type]; - } - return col; - } else { - console.warn(`No column "${col.column}" found (specified in aggregates attribute).`); - } - }) - .filter(x => x); - } - - // Add columns detected from dataset. - for (let col of cols) { - if (!found[col]) { - aggregates.push({ - column: col, - op: perspective.AGGREGATE_DEFAULTS[schema[col]] - }); - } - } - - this.set_aggregate_attribute(aggregates); - - // Update column rows. - let shown = JSON.parse(this.getAttribute("columns") || "[]").filter(x => cols.indexOf(x) > -1); - - // strip computed columns from sorted columns & schema - place at end - if (!_.isEmpty(computed_schema)) { - const computed_columns = _.keys(computed_schema); - for (let i = 0; i < computed_columns.length; i++) { - const cc = computed_columns[i]; - if (cols.includes(cc)) { - cols.splice(cols.indexOf(cc), 1); - } - if (_.has(schema, cc)) { - delete schema[cc]; - } - } - } - - const computed_cols = _.pairs(computed_schema); - - if (!this.hasAttribute("columns") || shown.length === 0) { - for (let x of cols) { - let aggregate = aggregates.filter(a => a.column === x).map(a => a.op)[0]; - let row = this.new_row(x, schema[x], aggregate); - this._inactive_columns.appendChild(row); - } - - // fixme better approach please - for (let cc of computed_cols) { - let cc_data = this._format_computed_data(cc); - let aggregate = aggregates.filter(a => a.column === cc_data.column_name).map(a => a.op)[0]; - let row = this.new_row(cc_data.column_name, cc_data.type, aggregate, null, null, cc_data); - this._inactive_columns.appendChild(row); - } - - this._set_column_defaults(); - shown = JSON.parse(this.getAttribute("columns") || "[]").filter(x => cols.indexOf(x) > -1); - for (let x in cols) { - if (shown.indexOf(x) !== -1) { - this._inactive_columns.children[x].classList.add("active"); - } - } - } else { - for (let x of cols) { - let aggregate = aggregates.filter(a => a.column === x).map(a => a.op)[0]; - let row = this.new_row(x, schema[x], aggregate); - this._inactive_columns.appendChild(row); - if (shown.includes(x)) { - row.classList.add("active"); - } - } - - // fixme better approach please - for (let cc of computed_cols) { - let cc_data = this._format_computed_data(cc); - let aggregate = aggregates.filter(a => a.column === cc_data.column_name).map(a => a.op)[0]; - let row = this.new_row(cc_data.column_name, cc_data.type, aggregate, null, null, cc_data); - this._inactive_columns.appendChild(row); - if (shown.includes(cc)) { - row.classList.add("active"); - } - } - - for (let x of shown) { - let active_row = this.new_row(x, schema[x]); - this._active_columns.appendChild(active_row); - } - } - - if (cols.length === shown.length) { - this._inactive_columns.parentElement.classList.add("collapse"); - } else { - this._inactive_columns.parentElement.classList.remove("collapse"); - } - - this.shadowRoot.querySelector("#columns_container").style.visibility = "visible"; - this.shadowRoot.querySelector("#side_panel__actions").style.visibility = "visible"; - - this.filters = this.getAttribute("filters"); - await this._debounce_update(); - } - - // Generates a new row in state + DOM - new_row(name, type, aggregate, filter, sort, computed) { - let row = document.createElement("perspective-row"); - - if (!type) { - let all = this._get_view_dom_columns("#inactive_columns perspective-row"); - if (all.length > 0) { - type = all.find(x => x.getAttribute("name") === name); - if (type) { - type = type.getAttribute("type"); - } else { - type = "integer"; - } - } else { - type = ""; - } - } - - if (!aggregate) { - let aggregates = this.get_aggregate_attribute(); - if (aggregates) { - aggregate = aggregates.find(x => x.column === name); - if (aggregate) { - aggregate = aggregate.op; - } else { - aggregate = perspective.AGGREGATE_DEFAULTS[type]; - } - } else { - aggregate = perspective.AGGREGATE_DEFAULTS[type]; - } - } - - if (filter) { - row.setAttribute("filter", filter); - if (type === "string") { - const v = this._table.view({row_pivot: [name], aggregate: []}); - v.to_json().then(json => { - row.choices(json.slice(1, json.length).map(x => x.__ROW_PATH__)); - v.delete(); - }); - } - } - - if (sort) { - row.setAttribute("sort-order", sort); - } else { - row.setAttribute("sort-order", "asc"); - } - - row.setAttribute("type", type); - row.setAttribute("name", name); - row.setAttribute("aggregate", aggregate); - - row.addEventListener("visibility-clicked", column_visibility_clicked.bind(this)); - row.addEventListener("aggregate-selected", column_aggregate_clicked.bind(this)); - row.addEventListener("filter-selected", column_filter_clicked.bind(this)); - row.addEventListener("close-clicked", event => undrag.call(this, event.detail)); - row.addEventListener("row-drag", () => { - this.classList.add("dragging"); - this._original_index = Array.prototype.slice.call(this._active_columns.children).findIndex(x => x.getAttribute("name") === name); - if (this._original_index !== -1) { - this._drop_target_hover = this._active_columns.children[this._original_index]; - setTimeout(() => row.setAttribute("drop-target", true)); - } else { - this._drop_target_hover = this.new_row(name, type, aggregate); - } - }); - row.addEventListener("sort-order", sort_order_clicked.bind(this)); - row.addEventListener("row-dragend", () => this.classList.remove("dragging")); - - if (computed) { - row.setAttribute("computed_column", JSON.stringify(computed)); - row.classList.add("computed"); - } - - return row; - } - - // Update viewer with new data. - async _viewer_update(ignore_size_check = false) { - if (!this._table) return; - let row_pivots = this._get_view_row_pivots(); - let column_pivots = this._get_view_column_pivots(); - let filters = this._get_view_filters(); - let aggregates = this._get_view_aggregates(); - if (aggregates.length === 0) return; - let sort = this._get_view_sorts(); - let hidden = this._get_view_hidden(aggregates, sort); - for (let s of hidden) { - let all = this._get_view_aggregates("#inactive_columns perspective-row"); - aggregates.push(all.reduce((obj, y) => (y.column === s ? y : obj))); - } - - if (this._view) { - this._view.delete(); - this._view = undefined; - } - this._view = this._table.view({ - filter: filters, - row_pivot: row_pivots, - column_pivot: column_pivots, - aggregate: aggregates, - sort: sort - }); - - if (ignore_size_check === false && this._show_warnings === true && this._plugin.max_size !== undefined) { - // validate that the render does not slow down the browser - const num_columns = await this._view.num_columns(); - const num_rows = await this._view.num_rows(); - const count = num_columns * num_rows; - if (count >= this._plugin.max_size) { - this._plugin_information.classList.remove("hidden"); - this.removeAttribute("updating"); - return; - } - } - - this._view.on_update(() => { - if (!this._debounced) { - let view_count = document.getElementsByTagName("perspective-viewer").length; - let timeout = this.getAttribute("render_time") * view_count * 2; - timeout = Math.min(10000, Math.max(0, timeout)); - this._debounced = setTimeout(() => { - this._debounced = undefined; - const timer = this._render_time(); - if (this._task && !this._task.initial) { - this._task.cancel(); - } - const task = (this._task = new CancelTask()); - let updater = this._plugin.update; - if (!updater) { - updater = this._plugin.create; - } - updater - .call(this, this._datavis, this._view, task) - .then(() => { - timer(); - task.cancel(); - }) - .catch(err => { - console.error("Error rendering plugin.", err); - }) - .finally(() => this.dispatchEvent(new Event("perspective-view-update"))); - }, timeout || 0); - } - }); - - const timer = this._render_time(); - this._render_count = (this._render_count || 0) + 1; - if (this._task) { - this._task.cancel(); - } - const task = (this._task = new CancelTask(() => { - this._render_count--; - })); - task.initial = true; - - await this._plugin.create - .call(this, this._datavis, this._view, task) - .catch(err => { - console.warn(err); - }) - .finally(() => { - if (!this.hasAttribute("render_time")) { - this.dispatchEvent(new Event("perspective-view-update")); - } - timer(); - task.cancel(); - if (this._render_count === 0) { - this.removeAttribute("updating"); - } - }); - } - - _update_column_view(columns, reset = false) { - if (!columns) { - columns = this._get_view_columns(); - } - this.setAttribute("columns", JSON.stringify(columns)); - const lis = this._get_view_dom_columns("#inactive_columns perspective-row"); - if (columns.length === lis.length) { - this._inactive_columns.parentElement.classList.add("collapse"); - } else { - this._inactive_columns.parentElement.classList.remove("collapse"); - } - lis.forEach(x => { - const index = columns.indexOf(x.getAttribute("name")); - if (index === -1) { - x.classList.remove("active"); - } else { - x.classList.add("active"); - } - }); - if (reset) { - this._active_columns.innerHTML = ""; - columns.map(y => { - let ref = lis.find(x => x.getAttribute("name") === y); - if (ref) { - this._active_columns.appendChild(this.new_row(ref.getAttribute("name"), ref.getAttribute("type"))); - } - }); - } - } - - _render_time() { - const t = performance.now(); - return () => this.setAttribute("render_time", performance.now() - t); - } - - // set viewer state - _set_column_defaults() { - let cols = this._get_view_dom_columns("#inactive_columns perspective-row"); - let active_cols = this._get_view_dom_columns(); - if (cols.length > 0) { - if (this._plugin.initial) { - let pref = []; - let count = this._plugin.initial.count || 2; - if (active_cols.length === count) { - pref = active_cols.map(x => x.getAttribute("name")); - } else if (active_cols.length < count) { - pref = active_cols.map(x => x.getAttribute("name")); - this._fill_numeric(cols, pref); - if (pref.length < count) { - this._fill_numeric(cols, pref, true); - } - } else { - if (this._plugin.initial.type === "number") { - this._fill_numeric(active_cols, pref); - if (pref.length < count) { - this._fill_numeric(cols, pref); - } - if (pref.length < count) { - this._fill_numeric(cols, pref, true); - } - } - } - this.setAttribute("columns", JSON.stringify(pref.slice(0, count))); - } else if (this._plugin.selectMode === "select") { - this.setAttribute("columns", JSON.stringify([cols[0].getAttribute("name")])); - } - } - } - - _clear_state() { - if (this._task) { - this._task.cancel(); - } - let all = []; - if (this._view) { - let view = this._view; - this._view = undefined; - all.push(view.delete()); - } - if (this._table) { - let table = this._table; - this._table = undefined; - if (table._owner_viewer && table._owner_viewer === this) { - all.push(table.delete()); - } - } - return Promise.all(all); - } - - _fill_numeric(cols, pref, bypass = false) { - for (let col of cols) { - let type = col.getAttribute("type"); - let name = col.getAttribute("name"); - if (bypass || (["float", "integer"].indexOf(type) > -1 && pref.indexOf(name) === -1)) { - pref.push(name); - } - } - } - - // Computed Columns - _format_computed_data(cc) { - return { - column_name: cc[0], - input_columns: cc[1].input_columns, - input_type: cc[1].input_type, - computation: cc[1].computation, - type: cc[1].type - }; - } - - // UI action - _open_computed_column(event) { - //const data = event.detail; - event.stopImmediatePropagation(); - /*if (event.type === 'perspective-computed-column-edit') { - this._computed_column._edit_computed_column(data); - }*/ - this._computed_column.style.display = "flex"; - this._side_panel_actions.style.display = "none"; - } - - // edits state - _set_computed_column_input(event) { - event.detail.target.appendChild(this.new_row(event.detail.column.name, event.detail.column.type)); - this._update_column_view(); - } - - // edits state - _validate_computed_column(event) { - const new_column = event.detail; - let computed_columns = JSON.parse(this.getAttribute("computed-columns")); - if (computed_columns === null) { - computed_columns = []; - } - // names cannot be duplicates - for (let col of computed_columns) { - if (new_column.name === col.name) { - return; - } - } - computed_columns.push(new_column); - this.setAttribute("computed-columns", JSON.stringify(computed_columns)); - } - - // edits state, calls reload - async _create_computed_column(event) { - const data = event.detail; - let computed_column_name = data.column_name; - - const cols = await this._table.columns(); - // edit overwrites last column, otherwise avoid name collision - if (cols.includes(computed_column_name)) { - console.log(computed_column_name); - computed_column_name += ` ${Math.round(Math.random() * 100)}`; - } - - const params = [ - { - computation: data.computation, - column: computed_column_name, - func: data.computation.func, - inputs: data.input_columns.map(col => col.name), - input_type: data.computation.input_type, - type: data.computation.return_type - } - ]; - - const table = this._table.add_computed(params); - await this.load_table(table, true); - this._update_column_view(); - } - - // edits state - _transpose() { - let row_pivots = this.getAttribute("row-pivots"); - this.setAttribute("row-pivots", this.getAttribute("column-pivots")); - this.setAttribute("column-pivots", row_pivots); - } - - // setup functions - _register_ids() { - this._aggregate_selector = this.shadowRoot.querySelector("#aggregate_selector"); - this._vis_selector = this.shadowRoot.querySelector("#vis_selector"); - this._filters = this.shadowRoot.querySelector("#filters"); - this._row_pivots = this.shadowRoot.querySelector("#row_pivots"); - this._column_pivots = this.shadowRoot.querySelector("#column_pivots"); - this._datavis = this.shadowRoot.querySelector("#pivot_chart"); - this._active_columns = this.shadowRoot.querySelector("#active_columns"); - this._inactive_columns = this.shadowRoot.querySelector("#inactive_columns"); - this._side_panel_actions = this.shadowRoot.querySelector("#side_panel__actions"); - this._add_computed_column = this.shadowRoot.querySelector("#add-computed-column"); - this._computed_column = this.shadowRoot.querySelector("perspective-computed-column"); - this._computed_column_inputs = this._computed_column.querySelector("#psp-cc-computation-inputs"); - this._inner_drop_target = this.shadowRoot.querySelector("#drop_target_inner"); - this._drop_target = this.shadowRoot.querySelector("#drop_target"); - this._config_button = this.shadowRoot.querySelector("#config_button"); - this._reset_button = this.shadowRoot.querySelector("#reset_button"); - this._download_button = this.shadowRoot.querySelector("#download_button"); - this._copy_button = this.shadowRoot.querySelector("#copy_button"); - this._side_panel = this.shadowRoot.querySelector("#side_panel"); - this._top_panel = this.shadowRoot.querySelector("#top_panel"); - this._sort = this.shadowRoot.querySelector("#sort"); - this._transpose_button = this.shadowRoot.querySelector("#transpose_button"); - this._plugin_information = this.shadowRoot.querySelector(".plugin_information"); - this._plugin_information_action = this.shadowRoot.querySelector(".plugin_information__action"); - this._plugin_information_dismiss = this.shadowRoot.querySelector(".plugin_information__action--dismiss"); - } - - // most of these are drag and drop handlers - how to clean up? - _register_callbacks() { - this._toggle_config = _toggle_config.bind(this); - this._sort.addEventListener("drop", drop.bind(this)); - this._sort.addEventListener("dragend", undrag.bind(this)); - this._sort.addEventListener("dragenter", drag_enter.bind(this)); - this._sort.addEventListener("dragover", allow_drop.bind(this)); - this._sort.addEventListener("dragleave", disallow_drop.bind(this)); - this._row_pivots.addEventListener("drop", drop.bind(this)); - this._row_pivots.addEventListener("dragend", undrag.bind(this)); - this._row_pivots.addEventListener("dragenter", drag_enter.bind(this)); - this._row_pivots.addEventListener("dragover", allow_drop.bind(this)); - this._row_pivots.addEventListener("dragleave", disallow_drop.bind(this)); - this._column_pivots.addEventListener("drop", drop.bind(this)); - this._column_pivots.addEventListener("dragend", undrag.bind(this)); - this._column_pivots.addEventListener("dragenter", drag_enter.bind(this)); - this._column_pivots.addEventListener("dragover", allow_drop.bind(this)); - this._column_pivots.addEventListener("dragleave", disallow_drop.bind(this)); - this._filters.addEventListener("drop", drop.bind(this)); - this._filters.addEventListener("dragend", undrag.bind(this)); - this._filters.addEventListener("dragenter", drag_enter.bind(this)); - this._filters.addEventListener("dragover", allow_drop.bind(this)); - this._filters.addEventListener("dragleave", disallow_drop.bind(this)); - this._active_columns.addEventListener("drop", column_drop.bind(this)); - this._active_columns.addEventListener("dragenter", drag_enter.bind(this)); - this._active_columns.addEventListener("dragend", column_undrag.bind(this)); - this._active_columns.addEventListener("dragover", column_dragover.bind(this)); - this._active_columns.addEventListener("dragleave", column_dragleave.bind(this)); - this._add_computed_column.addEventListener("click", this._open_computed_column.bind(this)); - this._computed_column.addEventListener("perspective-computed-column-save", this._validate_computed_column.bind(this)); - this._computed_column.addEventListener("perspective-computed-column-update", this._set_computed_column_input.bind(this)); - //this._side_panel.addEventListener('perspective-computed-column-edit', this._open_computed_column.bind(this)); - this._config_button.addEventListener("click", this._toggle_config); - this._config_button.addEventListener("contextmenu", _show_context_menu.bind(this)); - this._reset_button.addEventListener("click", this.reset.bind(this)); - this._copy_button.addEventListener("click", event => this.copy(event.shiftKey)); - this._download_button.addEventListener("click", event => this.download(event.shiftKey)); - this._transpose_button.addEventListener("click", this._transpose.bind(this)); - this._drop_target.addEventListener("dragover", allow_drop.bind(this)); - - this._vis_selector.addEventListener("change", () => { - this.setAttribute("view", this._vis_selector.value); - this._debounce_update(); - }); - - this._plugin_information_action.addEventListener("click", () => { - this._debounce_update({ignore_size_check: true}); - this._plugin_information.classList.add("hidden"); - }); - this._plugin_information_dismiss.addEventListener("click", () => { - this._debounce_update({ignore_size_check: true}); - this._plugin_information.classList.add("hidden"); - this._show_warnings = false; - }); - } - - // sets state, manipulates DOM - _register_view_options() { - let current_renderers = renderers.getInstance(); - for (let name in current_renderers) { - const display_name = current_renderers[name].name || name; - const opt = ``; - this._vis_selector.innerHTML += opt; - } - } - - // sets state - _register_data_attribute() { - // TODO this feature needs to become a real attribute. - if (this.getAttribute("data")) { - let data = this.getAttribute("data"); - try { - data = JSON.parse(data); - } catch (e) {} - this.load(data); - } - } - - // setup for update - _register_debounce_instance() { - const _update = _.debounce((resolve, ignore_size_check) => { - this._viewer_update(ignore_size_check).then(resolve); - }, 10); - this._debounce_update = async ({ignore_size_check = false} = {}) => { - this.setAttribute("updating", true); - await new Promise(resolve => _update(resolve, ignore_size_check)); - }; - } -} diff --git a/packages/perspective-viewer/src/js/view/actions.js b/packages/perspective-viewer/src/js/view/actions.js deleted file mode 100644 index bdca7128e9..0000000000 --- a/packages/perspective-viewer/src/js/view/actions.js +++ /dev/null @@ -1,78 +0,0 @@ -/****************************************************************************** - * - * Copyright (c) 2018, the Perspective Authors. - * - * This file is part of the Perspective library, distributed under the terms of - * the Apache License 2.0. The full license can be found in the LICENSE file. - * - */ - -export function column_visibility_clicked(ev) { - let parent = ev.currentTarget; - let is_active = parent.parentElement.getAttribute("id") === "active_columns"; - if (is_active) { - if (this._get_visible_column_count() === 1) { - return; - } - if (ev.detail.shiftKey) { - for (let child of Array.prototype.slice.call(this._active_columns.children)) { - if (child !== parent) { - this._active_columns.removeChild(child); - } - } - } else { - this._active_columns.removeChild(parent); - } - } else { - // check if we're manipulating computed column input - if (ev.path && ev.path[1].classList.contains("psp-cc-computation__input-column")) { - // this._computed_column._register_inputs(); - this._computed_column.deselect_column(ev.currentTarget.getAttribute("name")); - this._update_column_view(); - return; - } - if ((ev.detail.shiftKey && this._plugin.selectMode === "toggle") || (!ev.detail.shiftKey && this._plugin.selectMode === "select")) { - for (let child of Array.prototype.slice.call(this._active_columns.children)) { - this._active_columns.removeChild(child); - } - } - let row = this.new_row(parent.getAttribute("name"), parent.getAttribute("type")); - this._active_columns.appendChild(row); - } - let cols = this._get_view_columns(); - this._update_column_view(cols); -} - -export function column_aggregate_clicked() { - let aggregates = this.get_aggregate_attribute(); - let new_aggregates = this._get_view_aggregates(); - for (let aggregate of aggregates) { - let updated_agg = new_aggregates.find(x => x.column === aggregate.column); - if (updated_agg) { - aggregate.op = updated_agg.op; - } - } - this.set_aggregate_attribute(aggregates); - this._update_column_view(); - this._debounce_update(); -} - -export function column_filter_clicked() { - let new_filters = this._get_view_filters(); - this._updating_filter = true; - this.setAttribute("filters", JSON.stringify(new_filters)); - this._updating_filter = false; - this._debounce_update(); -} - -export function sort_order_clicked() { - let sort = JSON.parse(this.getAttribute("sort")); - let new_sort = this._get_view_sorts(); - for (let s of sort) { - let updated_sort = new_sort.find(x => x[0] === s[0]); - if (updated_sort) { - s[1] = updated_sort[1]; - } - } - this.setAttribute("sort", JSON.stringify(sort)); -} \ No newline at end of file diff --git a/packages/perspective-viewer/src/js/view/dom.js b/packages/perspective-viewer/src/js/view/dom.js deleted file mode 100644 index a153332ab6..0000000000 --- a/packages/perspective-viewer/src/js/view/dom.js +++ /dev/null @@ -1,36 +0,0 @@ -/****************************************************************************** - * - * Copyright (c) 2018, the Perspective Authors. - * - * This file is part of the Perspective library, distributed under the terms of - * the Apache License 2.0. The full license can be found in the LICENSE file. - * - */ - -export function _show_context_menu(event) { - this.shadowRoot.querySelector("#app").classList.toggle("show_menu"); - event.stopPropagation(); - event.preventDefault(); - return false; -} - -export function _hide_context_menu() { - this.shadowRoot.querySelector("#app").classList.remove("show_menu"); -} - -export function _toggle_config() { - if (this._show_config) { - this._side_panel.style.display = "none"; - this._top_panel.style.display = "none"; - this.removeAttribute("settings"); - } else { - this._side_panel.style.display = "flex"; - this._top_panel.style.display = "flex"; - this.setAttribute("settings", true); - } - this._show_config = !this._show_config; - this._plugin.resize.call(this, true); - console.log(this._show_config); - _hide_context_menu.call(this); - this.dispatchEvent(new CustomEvent("perspective-toggle-settings", {detail: this._show_config})); -} \ No newline at end of file diff --git a/packages/perspective-viewer/src/js/view/renderers.js b/packages/perspective-viewer/src/js/view/renderers.js deleted file mode 100644 index 04696631f3..0000000000 --- a/packages/perspective-viewer/src/js/view/renderers.js +++ /dev/null @@ -1,34 +0,0 @@ -/****************************************************************************** - * - * Copyright (c) 2018, the Perspective Authors. - * - * This file is part of the Perspective library, distributed under the terms of - * the Apache License 2.0. The full license can be found in the LICENSE file. - * - */ - -export const renderers = (function() { - let RENDERERS = {}; - return { - getInstance: function() { - return RENDERERS; - }, - /** - * Register a plugin with the component. - * - * @param {string} name The logical unique name of the plugin. This will be - * used to set the component's `view` attribute. - * @param {object} plugin An object with this plugin's prototype. Valid keys are: - * name : The display name for this plugin. - * create (required) : The creation function - may return a `Promise`. - * delete : The deletion function. - * mode : The selection mode - may be "toggle" or "select". - */ - registerPlugin: function(name, plugin) { - RENDERERS[name] = plugin; - }, - getPlugin(name) { - return RENDERERS[name]; - } - }; -})(); diff --git a/packages/perspective-viewer/src/js/view.js b/packages/perspective-viewer/src/js/viewer.js similarity index 80% rename from packages/perspective-viewer/src/js/view.js rename to packages/perspective-viewer/src/js/viewer.js index f545e32905..bc5df0e077 100755 --- a/packages/perspective-viewer/src/js/view.js +++ b/packages/perspective-viewer/src/js/viewer.js @@ -13,80 +13,28 @@ import "@webcomponents/shadycss/custom-style-interface.min.js"; import _ from "underscore"; import {polyfill} from "mobile-drag-drop"; -import perspective from "@jpmorganchase/perspective"; -import {ViewPrivate} from "./view/ViewPrivate.js"; -import "./row.js"; - import {bindTemplate, json_attribute, array_attribute, copy_to_clipboard} from "./utils.js"; -import {renderers} from "./view/renderers.js"; -import {_hide_context_menu} from "./view/dom.js"; +import {renderers, register_debug_plugin} from "./viewer/renderers.js"; import {COMPUTATIONS} from "./computed_column.js"; +import "./row.js"; -import template from "../html/view.html"; +import template from "../html/viewer.html"; -import view_style from "../less/view.less"; +import view_style from "../less/viewer.less"; import default_style from "../less/default.less"; -polyfill({}); +import {ActionElement} from "./viewer/action_element.js"; -/****************************************************************************** - * - * Plugin API - * - */ - -global.registerPlugin = renderers.registerPlugin; - -global.getPlugin = renderers.getPlugin; - -function _register_debug_plugin() { - global.registerPlugin("debug", { - name: "Debug", - create: async function(div) { - const csv = await this._view.to_csv({config: {delimiter: "|"}}); - const timer = this._render_time(); - div.innerHTML = `
${csv}
`; - timer(); - }, - selectMode: "toggle", - resize: function() {}, - delete: function() {} - }); -} - -/****************************************************************************** - * - * Perspective Loading - * - */ - -let worker = (function() { - let __WORKER__; - return { - getInstance: function() { - if (__WORKER__ === undefined) { - __WORKER__ = perspective.worker(); - } - return __WORKER__; - } - }; -})(); - -if (document.currentScript && document.currentScript.hasAttribute("preload")) { - worker.getInstance(); -} +polyfill({}); /** * HTMLElement class for ` view_style.toString() + "\n" + default_style.toString()}) // eslint-disable-next-line no-unused-vars -class View extends ViewPrivate { +@bindTemplate(template, view_style, default_style) // eslint-disable-next-line no-unused-vars +class PerspectiveViewer extends ActionElement { constructor() { super(); this._register_debounce_instance(); @@ -100,7 +48,7 @@ class View extends ViewPrivate { connectedCallback() { if (Object.keys(renderers.getInstance()).length === 0) { - _register_debug_plugin(); + register_debug_plugin(); } this.setAttribute("settings", true); @@ -123,11 +71,11 @@ class View extends ViewPrivate { * names. * * @name sort - * @memberof View.prototype + * @memberof PerspectiveViewer.prototype * @type {array} Array of arrays tuples of column name and * direction, where the possible values are "asc", "desc", "asc abs", * "desc abs" and "none". - * @fires View#perspective-config-update + * @fires PerspectiveViewer#perspective-config-update * @example via Javascript DOM * let elem = document.getElementById('my_viewer'); * elem.setAttribute('sort', JSON.stringify([["x","desc"])); @@ -146,7 +94,7 @@ class View extends ViewPrivate { dir = s[1]; s = s[0]; } - let row = this.new_row(s, false, false, false, dir); + let row = this._new_row(s, false, false, false, dir); inner.appendChild(row); }.bind(this) ); @@ -159,9 +107,9 @@ class View extends ViewPrivate { * The set of visible columns. * * @name columns - * @memberof View.prototype + * @memberof PerspectiveViewer.prototype * @param {array} columns An array of strings, the names of visible columns. - * @fires View#perspective-config-update + * @fires PerspectiveViewer#perspective-config-update * @example via Javascript DOM * let elem = document.getElementById('my_viewer'); * elem.setAttribute('columns', JSON.stringify(["x", "y'"])); @@ -179,9 +127,9 @@ class View extends ViewPrivate { * The set of visible columns. * * @name computed-columns - * @memberof View.prototype + * @memberof PerspectiveViewer.prototype * @param {array} computed-columns An array of computed column objects - * @fires View#perspective-config-update + * @fires PerspectiveViewer#perspective-config-update * @example via Javascript DOM * let elem = document.getElementById('my_viewer'); * elem.setAttribute('computed-columns', JSON.stringify([{name: "x+y", func: "add", inputs: ["x", "y"]}])); @@ -213,13 +161,13 @@ class View extends ViewPrivate { * The set of column aggregate configurations. * * @name aggregates - * @memberof View.prototype + * @memberof PerspectiveViewer.prototype * @param {object} aggregates A dictionary whose keys are column names, and * values are valid aggregations. The `aggergates` attribute works as an * override; in lieu of a key for a column supplied by the developers, a * default will be selected and reflected to the attribute based on the * column's type. See {@link perspective/src/js/defaults.js} - * @fires View#perspective-config-update + * @fires PerspectiveViewer#perspective-config-update * @example via Javascript DOM * let elem = document.getElementById('my_viewer'); * elem.setAttribute('aggregates', JSON.stringify({x: "distinct count"})); @@ -243,7 +191,7 @@ class View extends ViewPrivate { * The set of column filter configurations. * * @name filters - * @memberof View.prototype + * @memberof PerspectiveViewer.prototype * @type {array} filters An arry of filter config objects. A filter * config object is an array of three elements: * * The column name. @@ -251,7 +199,7 @@ class View extends ViewPrivate { * {@link perspective/src/js/defaults.js} * * The filter argument, as a string, float or Array as the * filter operation demands. - * @fires View#perspective-config-update + * @fires PerspectiveViewer#perspective-config-update * @example via Javascript DOM * let filters = [ * ["x", "<", 3], @@ -273,7 +221,7 @@ class View extends ViewPrivate { operator: pivot[1], operand: pivot[2] }); - const row = this.new_row(pivot[0], undefined, undefined, fterms); + const row = this._new_row(pivot[0], undefined, undefined, fterms); inner.appendChild(row); }); } @@ -286,7 +234,7 @@ class View extends ViewPrivate { * Sets the currently selected plugin, via its `name` field. * * @type {string} - * @fires View#perspective-config-update + * @fires PerspectiveViewer#perspective-config-update */ set view(v) { this._vis_selector.value = this.getAttribute("view"); @@ -298,9 +246,9 @@ class View extends ViewPrivate { * Sets this `perspective.table.view`'s `column_pivots` property. * * @name column-pivots - * @memberof View.prototype + * @memberof PerspectiveViewer.prototype * @type {array} Array of column names - * @fires View#perspective-config-update + * @fires PerspectiveViewer#perspective-config-update */ @array_attribute set "column-pivots"(pivots) { @@ -309,7 +257,7 @@ class View extends ViewPrivate { if (pivots.length > 0) { pivots.map( function(pivot) { - let row = this.new_row(pivot); + let row = this._new_row(pivot); inner.appendChild(row); }.bind(this) ); @@ -322,9 +270,9 @@ class View extends ViewPrivate { * Sets this `perspective.table.view`'s `row_pivots` property. * * @name row-pivots - * @memberof View.prototype + * @memberof PerspectiveViewer.prototype * @type {array} Array of column names - * @fires View#perspective-config-update + * @fires PerspectiveViewer#perspective-config-update */ @array_attribute set "row-pivots"(pivots) { @@ -333,7 +281,7 @@ class View extends ViewPrivate { if (pivots.length > 0) { pivots.map( function(pivot) { - let row = this.new_row(pivot); + let row = this._new_row(pivot); inner.appendChild(row); }.bind(this) ); @@ -378,15 +326,12 @@ class View extends ViewPrivate { * elem.load(table); */ get worker() { - if (this._table) { - return this._table._worker; - } - return worker.getInstance(); + return this._get_worker(); } /** * This element's `perspective.table.view` instance. The instance itself - * will change after every `View#perspective-config-update` event. + * will change after every `PerspectiveViewer#perspective-config-update` event. * * @readonly */ @@ -402,7 +347,7 @@ class View extends ViewPrivate { * supported by `perspective.table`. * @returns {Promise} A promise which resolves once the data is * loaded and a `perspective.view` has been created. - * @fires View#perspective-view-update + * @fires PerspectiveViewer#perspective-view-update * @example Load JSON * const my_viewer = document.getElementById('#my_viewer'); * my_viewer.load([ @@ -425,12 +370,12 @@ class View extends ViewPrivate { if (data.hasOwnProperty("_name")) { table = data; } else { - table = worker.getInstance().table(data, options); + table = this.worker.table(data, options); table._owner_viewer = this; } - let _promises = [this.load_table(table)]; + let _promises = [this._load_table(table)]; for (let slave of this._slaves) { - _promises.push(this.load_table.call(slave, table)); + _promises.push(this._load_table.call(slave, table)); } this._slaves = []; return Promise.all(_promises); @@ -441,7 +386,7 @@ class View extends ViewPrivate { * * @param {any} data The data to load. Works with the same input types * supported by `perspective.table.update`. - * @fires View#perspective-view-update + * @fires PerspectiveViewer#perspective-view-update * @example * const my_viewer = document.getElementById('#my_viewer'); * my_viewer.update([ @@ -489,7 +434,7 @@ class View extends ViewPrivate { } if (widget._table) { - this.load_table(widget._table); + this._load_table(widget._table); } else { widget._slaves.push(this); } @@ -568,7 +513,7 @@ class View extends ViewPrivate { } this.setAttribute("view", Object.keys(renderers.getInstance())[0]); this.dispatchEvent(new Event("perspective-config-update")); - _hide_context_menu.call(this); + this._hide_context_menu(); } /** @@ -576,7 +521,7 @@ class View extends ViewPrivate { * * @param {boolean} [flat=false] Whether to use the element's current view * config, or to use a default "flat" view. - * @memberof View + * @memberof PerspectiveViewer */ async download(flat = false) { const view = flat ? this._table.view() : this._view; @@ -595,7 +540,7 @@ class View extends ViewPrivate { document.body.appendChild(element); element.click(); document.body.removeChild(element); - _hide_context_menu.call(this); + this._hide_context_menu(); } /** @@ -615,19 +560,19 @@ class View extends ViewPrivate { console.error(err); data = ""; }); - let count = 0, - f = () => { - if (typeof data !== "undefined") { - copy_to_clipboard(data); - } else if (count < 200) { - count++; - setTimeout(f, 50); - } else { - console.warn("Timeout expired - copy to clipboard cancelled."); - } - }; + let count = 0; + let f = () => { + if (typeof data !== "undefined") { + copy_to_clipboard(data); + } else if (count < 200) { + count++; + setTimeout(f, 50); + } else { + console.warn("Timeout expired - copy to clipboard cancelled."); + } + }; f(); - _hide_context_menu.call(this); + this._hide_context_menu(); } /** @@ -642,7 +587,7 @@ class View extends ViewPrivate { * `perspective-config-update` is fired whenever an configuration attribute has * been modified, by the user or otherwise. * - * @event View#perspective-config-update + * @event PerspectiveViewer#perspective-config-update * @type {string} */ @@ -650,6 +595,6 @@ class View extends ViewPrivate { * `perspective-view-update` is fired whenever underlying `view`'s data has * updated, including every invocation of `load` and `update`. * - * @event View#perspective-view-update + * @event PerspectiveViewer#perspective-view-update * @type {string} */ diff --git a/packages/perspective-viewer/src/js/viewer/action_element.js b/packages/perspective-viewer/src/js/viewer/action_element.js new file mode 100644 index 0000000000..382c337a28 --- /dev/null +++ b/packages/perspective-viewer/src/js/viewer/action_element.js @@ -0,0 +1,235 @@ +/****************************************************************************** + * + * Copyright (c) 2018, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ + +import {undrag, column_undrag, column_dragleave, column_dragover, column_drop, drop, drag_enter, allow_drop, disallow_drop} from "./dragdrop.js"; + +import {DomElement} from "./dom_element.js"; + +export class ActionElement extends DomElement { + _show_context_menu(event) { + this.shadowRoot.querySelector("#app").classList.toggle("show_menu"); + event.stopPropagation(); + event.preventDefault(); + return false; + } + + _hide_context_menu() { + this.shadowRoot.querySelector("#app").classList.remove("show_menu"); + } + + _toggle_config() { + if (this._show_config) { + this._side_panel.style.display = "none"; + this._top_panel.style.display = "none"; + this.removeAttribute("settings"); + } else { + this._side_panel.style.display = "flex"; + this._top_panel.style.display = "flex"; + this.setAttribute("settings", true); + } + this._show_config = !this._show_config; + this._plugin.resize.call(this, true); + this._hide_context_menu(); + this.dispatchEvent(new CustomEvent("perspective-toggle-settings", {detail: this._show_config})); + } + + // UI action + _open_computed_column(event) { + //const data = event.detail; + event.stopImmediatePropagation(); + /*if (event.type === 'perspective-computed-column-edit') { + this._computed_column._edit_computed_column(data); + }*/ + this._computed_column.style.display = "flex"; + this._side_panel_actions.style.display = "none"; + } + + // edits state + _set_computed_column_input(event) { + event.detail.target.appendChild(this._new_row(event.detail.column.name, event.detail.column.type)); + this._update_column_view(); + } + + // edits state + _validate_computed_column(event) { + const new_column = event.detail; + let computed_columns = JSON.parse(this.getAttribute("computed-columns")); + if (computed_columns === null) { + computed_columns = []; + } + // names cannot be duplicates + for (let col of computed_columns) { + if (new_column.name === col.name) { + return; + } + } + computed_columns.push(new_column); + this.setAttribute("computed-columns", JSON.stringify(computed_columns)); + } + + // edits state, calls reload + async _create_computed_column(event) { + const data = event.detail; + let computed_column_name = data.column_name; + + const cols = await this._table.columns(); + // edit overwrites last column, otherwise avoid name collision + if (cols.includes(computed_column_name)) { + console.log(computed_column_name); + computed_column_name += ` ${Math.round(Math.random() * 100)}`; + } + + const params = [ + { + computation: data.computation, + column: computed_column_name, + func: data.computation.func, + inputs: data.input_columns.map(col => col.name), + input_type: data.computation.input_type, + type: data.computation.return_type + } + ]; + + const table = this._table.add_computed(params); + await this._load_table(table, true); + this._update_column_view(); + } + + _column_visibility_clicked(ev) { + let parent = ev.currentTarget; + let is_active = parent.parentElement.getAttribute("id") === "active_columns"; + if (is_active) { + if (this._get_visible_column_count() === 1) { + return; + } + if (ev.detail.shiftKey) { + for (let child of Array.prototype.slice.call(this._active_columns.children)) { + if (child !== parent) { + this._active_columns.removeChild(child); + } + } + } else { + this._active_columns.removeChild(parent); + } + } else { + // check if we're manipulating computed column input + if (ev.path && ev.path[1].classList.contains("psp-cc-computation__input-column")) { + // this._computed_column._register_inputs(); + this._computed_column.deselect_column(ev.currentTarget.getAttribute("name")); + this._update_column_view(); + return; + } + if ((ev.detail.shiftKey && this._plugin.selectMode === "toggle") || (!ev.detail.shiftKey && this._plugin.selectMode === "select")) { + for (let child of Array.prototype.slice.call(this._active_columns.children)) { + this._active_columns.removeChild(child); + } + } + let row = this._new_row(parent.getAttribute("name"), parent.getAttribute("type")); + this._active_columns.appendChild(row); + } + let cols = this._get_view_columns(); + this._update_column_view(cols); + } + + _column_aggregate_clicked() { + let aggregates = this.get_aggregate_attribute(); + let new_aggregates = this._get_view_aggregates(); + for (let aggregate of aggregates) { + let updated_agg = new_aggregates.find(x => x.column === aggregate.column); + if (updated_agg) { + aggregate.op = updated_agg.op; + } + } + this.set_aggregate_attribute(aggregates); + this._update_column_view(); + this._debounce_update(); + } + + _column_filter_clicked() { + let new_filters = this._get_view_filters(); + this._updating_filter = true; + this.setAttribute("filters", JSON.stringify(new_filters)); + this._updating_filter = false; + this._debounce_update(); + } + + _sort_order_clicked() { + let sort = JSON.parse(this.getAttribute("sort")); + let new_sort = this._get_view_sorts(); + for (let s of sort) { + let updated_sort = new_sort.find(x => x[0] === s[0]); + if (updated_sort) { + s[1] = updated_sort[1]; + } + } + this.setAttribute("sort", JSON.stringify(sort)); + } + + // edits state + _transpose() { + let row_pivots = this.getAttribute("row-pivots"); + this.setAttribute("row-pivots", this.getAttribute("column-pivots")); + this.setAttribute("column-pivots", row_pivots); + } + + // most of these are drag and drop handlers - how to clean up? + _register_callbacks() { + this._sort.addEventListener("drop", drop.bind(this)); + this._sort.addEventListener("dragend", undrag.bind(this)); + this._sort.addEventListener("dragenter", drag_enter.bind(this)); + this._sort.addEventListener("dragover", allow_drop.bind(this)); + this._sort.addEventListener("dragleave", disallow_drop.bind(this)); + this._row_pivots.addEventListener("drop", drop.bind(this)); + this._row_pivots.addEventListener("dragend", undrag.bind(this)); + this._row_pivots.addEventListener("dragenter", drag_enter.bind(this)); + this._row_pivots.addEventListener("dragover", allow_drop.bind(this)); + this._row_pivots.addEventListener("dragleave", disallow_drop.bind(this)); + this._column_pivots.addEventListener("drop", drop.bind(this)); + this._column_pivots.addEventListener("dragend", undrag.bind(this)); + this._column_pivots.addEventListener("dragenter", drag_enter.bind(this)); + this._column_pivots.addEventListener("dragover", allow_drop.bind(this)); + this._column_pivots.addEventListener("dragleave", disallow_drop.bind(this)); + this._filters.addEventListener("drop", drop.bind(this)); + this._filters.addEventListener("dragend", undrag.bind(this)); + this._filters.addEventListener("dragenter", drag_enter.bind(this)); + this._filters.addEventListener("dragover", allow_drop.bind(this)); + this._filters.addEventListener("dragleave", disallow_drop.bind(this)); + this._active_columns.addEventListener("drop", column_drop.bind(this)); + this._active_columns.addEventListener("dragenter", drag_enter.bind(this)); + this._active_columns.addEventListener("dragend", column_undrag.bind(this)); + this._active_columns.addEventListener("dragover", column_dragover.bind(this)); + this._active_columns.addEventListener("dragleave", column_dragleave.bind(this)); + this._add_computed_column.addEventListener("click", this._open_computed_column.bind(this)); + this._computed_column.addEventListener("perspective-computed-column-save", this._validate_computed_column.bind(this)); + this._computed_column.addEventListener("perspective-computed-column-update", this._set_computed_column_input.bind(this)); + //this._side_panel.addEventListener('perspective-computed-column-edit', this._open_computed_column.bind(this)); + this._config_button.addEventListener("click", this._toggle_config.bind(this)); + this._config_button.addEventListener("contextmenu", this._show_context_menu.bind(this)); + this._reset_button.addEventListener("click", this.reset.bind(this)); + this._copy_button.addEventListener("click", event => this.copy(event.shiftKey)); + this._download_button.addEventListener("click", event => this.download(event.shiftKey)); + this._transpose_button.addEventListener("click", this._transpose.bind(this)); + this._drop_target.addEventListener("dragover", allow_drop.bind(this)); + + this._vis_selector.addEventListener("change", () => { + this.setAttribute("view", this._vis_selector.value); + this._debounce_update(); + }); + + this._plugin_information_action.addEventListener("click", () => { + this._debounce_update({ignore_size_check: true}); + this._plugin_information.classList.add("hidden"); + }); + this._plugin_information_dismiss.addEventListener("click", () => { + this._debounce_update({ignore_size_check: true}); + this._plugin_information.classList.add("hidden"); + this._show_warnings = false; + }); + } +} diff --git a/packages/perspective-viewer/src/js/view/CancelTask.js b/packages/perspective-viewer/src/js/viewer/cancel_task.js similarity index 88% rename from packages/perspective-viewer/src/js/view/CancelTask.js rename to packages/perspective-viewer/src/js/viewer/cancel_task.js index 86ab20a783..e77773c8fd 100644 --- a/packages/perspective-viewer/src/js/view/CancelTask.js +++ b/packages/perspective-viewer/src/js/viewer/cancel_task.js @@ -8,9 +8,10 @@ */ export class CancelTask { - constructor(on_cancel) { + constructor(on_cancel, initial = false) { this._on_cancel = on_cancel; this._cancelled = false; + this.initial = initial; } cancel() { @@ -23,4 +24,4 @@ export class CancelTask { get cancelled() { return this._cancelled; } -} \ No newline at end of file +} diff --git a/packages/perspective-viewer/src/js/viewer/dom_element.js b/packages/perspective-viewer/src/js/viewer/dom_element.js new file mode 100644 index 0000000000..ab82c5b048 --- /dev/null +++ b/packages/perspective-viewer/src/js/viewer/dom_element.js @@ -0,0 +1,242 @@ +/****************************************************************************** + * + * Copyright (c) 2018, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ + +import perspective from "@jpmorganchase/perspective"; +import {undrag} from "./dragdrop.js"; +import {renderers} from "./renderers.js"; + +import {PerspectiveElement} from "./perspective_element.js"; + +export class DomElement extends PerspectiveElement { + _clear_columns() { + this._inactive_columns.innerHTML = ""; + this._active_columns.innerHTML = ""; + } + + set_aggregate_attribute(aggs) { + this.setAttribute( + "aggregates", + JSON.stringify( + aggs.reduce((obj, agg) => { + obj[agg.column] = agg.op; + return obj; + }, {}) + ) + ); + } + + // Generates a new row in state + DOM + _new_row(name, type, aggregate, filter, sort, computed) { + let row = document.createElement("perspective-row"); + + if (!type) { + let all = this._get_view_dom_columns("#inactive_columns perspective-row"); + if (all.length > 0) { + type = all.find(x => x.getAttribute("name") === name); + if (type) { + type = type.getAttribute("type"); + } else { + type = "integer"; + } + } else { + type = ""; + } + } + + if (!aggregate) { + let aggregates = this.get_aggregate_attribute(); + if (aggregates) { + aggregate = aggregates.find(x => x.column === name); + if (aggregate) { + aggregate = aggregate.op; + } else { + aggregate = perspective.AGGREGATE_DEFAULTS[type]; + } + } else { + aggregate = perspective.AGGREGATE_DEFAULTS[type]; + } + } + + if (filter) { + row.setAttribute("filter", filter); + if (type === "string") { + const v = this._table.view({row_pivot: [name], aggregate: []}); + v.to_json().then(json => { + row.choices(json.slice(1, json.length).map(x => x.__ROW_PATH__)); + v.delete(); + }); + } + } + + if (sort) { + row.setAttribute("sort-order", sort); + } else { + row.setAttribute("sort-order", "asc"); + } + + row.setAttribute("type", type); + row.setAttribute("name", name); + row.setAttribute("aggregate", aggregate); + + row.addEventListener("visibility-clicked", this._column_visibility_clicked.bind(this)); + row.addEventListener("aggregate-selected", this._column_aggregate_clicked.bind(this)); + row.addEventListener("filter-selected", this._column_filter_clicked.bind(this)); + row.addEventListener("close-clicked", event => undrag.call(this, event.detail)); + row.addEventListener("sort-order", this._sort_order_clicked.bind(this)); + + row.addEventListener("row-drag", () => { + this.classList.add("dragging"); + this._original_index = Array.prototype.slice.call(this._active_columns.children).findIndex(x => x.getAttribute("name") === name); + if (this._original_index !== -1) { + this._drop_target_hover = this._active_columns.children[this._original_index]; + setTimeout(() => row.setAttribute("drop-target", true)); + } else { + this._drop_target_hover = this._new_row(name, type, aggregate); + } + }); + row.addEventListener("row-dragend", () => this.classList.remove("dragging")); + + if (computed) { + row.setAttribute("computed_column", JSON.stringify(computed)); + row.classList.add("computed"); + } + + return row; + } + + _update_column_view(columns, reset = false) { + if (!columns) { + columns = this._get_view_columns(); + } + this.setAttribute("columns", JSON.stringify(columns)); + const lis = this._get_view_dom_columns("#inactive_columns perspective-row"); + if (columns.length === lis.length) { + this._inactive_columns.parentElement.classList.add("collapse"); + } else { + this._inactive_columns.parentElement.classList.remove("collapse"); + } + lis.forEach(x => { + const index = columns.indexOf(x.getAttribute("name")); + if (index === -1) { + x.classList.remove("active"); + } else { + x.classList.add("active"); + } + }); + if (reset) { + this._active_columns.innerHTML = ""; + columns.map(y => { + let ref = lis.find(x => x.getAttribute("name") === y); + if (ref) { + this._active_columns.appendChild(this._new_row(ref.getAttribute("name"), ref.getAttribute("type"))); + } + }); + } + } + + _show_column_selectors() { + this.shadowRoot.querySelector("#columns_container").style.visibility = "visible"; + this.shadowRoot.querySelector("#side_panel__actions").style.visibility = "visible"; + } + + // set viewer state + _set_column_defaults() { + let cols = this._get_view_dom_columns("#inactive_columns perspective-row"); + let active_cols = this._get_view_dom_columns(); + if (cols.length > 0) { + if (this._plugin.initial) { + let pref = []; + let count = this._plugin.initial.count || 2; + if (active_cols.length === count) { + pref = active_cols.map(x => x.getAttribute("name")); + } else if (active_cols.length < count) { + pref = active_cols.map(x => x.getAttribute("name")); + this._fill_numeric(cols, pref); + if (pref.length < count) { + this._fill_numeric(cols, pref, true); + } + } else { + if (this._plugin.initial.type === "number") { + this._fill_numeric(active_cols, pref); + if (pref.length < count) { + this._fill_numeric(cols, pref); + } + if (pref.length < count) { + this._fill_numeric(cols, pref, true); + } + } + } + this.setAttribute("columns", JSON.stringify(pref.slice(0, count))); + } else if (this._plugin.selectMode === "select") { + this.setAttribute("columns", JSON.stringify([cols[0].getAttribute("name")])); + } + } + } + + _fill_numeric(cols, pref, bypass = false) { + for (let col of cols) { + let type = col.getAttribute("type"); + let name = col.getAttribute("name"); + if (bypass || (["float", "integer"].indexOf(type) > -1 && pref.indexOf(name) === -1)) { + pref.push(name); + } + } + } + + // setup functions + _register_ids() { + this._aggregate_selector = this.shadowRoot.querySelector("#aggregate_selector"); + this._vis_selector = this.shadowRoot.querySelector("#vis_selector"); + this._filters = this.shadowRoot.querySelector("#filters"); + this._row_pivots = this.shadowRoot.querySelector("#row_pivots"); + this._column_pivots = this.shadowRoot.querySelector("#column_pivots"); + this._datavis = this.shadowRoot.querySelector("#pivot_chart"); + this._active_columns = this.shadowRoot.querySelector("#active_columns"); + this._inactive_columns = this.shadowRoot.querySelector("#inactive_columns"); + this._side_panel_actions = this.shadowRoot.querySelector("#side_panel__actions"); + this._add_computed_column = this.shadowRoot.querySelector("#add-computed-column"); + this._computed_column = this.shadowRoot.querySelector("perspective-computed-column"); + this._computed_column_inputs = this._computed_column.querySelector("#psp-cc-computation-inputs"); + this._inner_drop_target = this.shadowRoot.querySelector("#drop_target_inner"); + this._drop_target = this.shadowRoot.querySelector("#drop_target"); + this._config_button = this.shadowRoot.querySelector("#config_button"); + this._reset_button = this.shadowRoot.querySelector("#reset_button"); + this._download_button = this.shadowRoot.querySelector("#download_button"); + this._copy_button = this.shadowRoot.querySelector("#copy_button"); + this._side_panel = this.shadowRoot.querySelector("#side_panel"); + this._top_panel = this.shadowRoot.querySelector("#top_panel"); + this._sort = this.shadowRoot.querySelector("#sort"); + this._transpose_button = this.shadowRoot.querySelector("#transpose_button"); + this._plugin_information = this.shadowRoot.querySelector(".plugin_information"); + this._plugin_information_action = this.shadowRoot.querySelector(".plugin_information__action"); + this._plugin_information_dismiss = this.shadowRoot.querySelector(".plugin_information__action--dismiss"); + } + + // sets state, manipulates DOM + _register_view_options() { + let current_renderers = renderers.getInstance(); + for (let name in current_renderers) { + const display_name = current_renderers[name].name || name; + const opt = ``; + this._vis_selector.innerHTML += opt; + } + } + + // sets state + _register_data_attribute() { + // TODO this feature needs to become a real attribute. + if (this.getAttribute("data")) { + let data = this.getAttribute("data"); + try { + data = JSON.parse(data); + } catch (e) {} + this.load(data); + } + } +} diff --git a/packages/perspective-viewer/src/js/dragdrop.js b/packages/perspective-viewer/src/js/viewer/dragdrop.js similarity index 100% rename from packages/perspective-viewer/src/js/dragdrop.js rename to packages/perspective-viewer/src/js/viewer/dragdrop.js diff --git a/packages/perspective-viewer/src/js/viewer/perspective_element.js b/packages/perspective-viewer/src/js/viewer/perspective_element.js new file mode 100644 index 0000000000..c4871299c0 --- /dev/null +++ b/packages/perspective-viewer/src/js/viewer/perspective_element.js @@ -0,0 +1,331 @@ +/****************************************************************************** + * + * Copyright (c) 2018, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ + +import _ from "underscore"; + +import perspective from "@jpmorganchase/perspective"; +import {CancelTask} from "./cancel_task.js"; +import {COMPUTATIONS} from "../computed_column.js"; + +import {StateElement} from "./state_element.js"; + +/****************************************************************************** + * + * Web Worker Singleton + * + */ + +const WORKER_SINGLETON = (function() { + let __WORKER__; + return { + getInstance: function() { + if (__WORKER__ === undefined) { + __WORKER__ = perspective.worker(); + } + return __WORKER__; + } + }; +})(); + +if (document.currentScript && document.currentScript.hasAttribute("preload")) { + WORKER_SINGLETON.getInstance(); +} + +/****************************************************************************** + * + * Helpers + * + */ + +let TYPE_ORDER = {integer: 2, string: 0, float: 3, boolean: 4, datetime: 1}; + +const column_sorter = schema => (a, b) => { + const s1 = TYPE_ORDER[schema[a]]; + const s2 = TYPE_ORDER[schema[b]]; + let r = 0; + if (s1 == s2) { + r = a.toLowerCase() < b.toLowerCase() ? -1 : 1; + } else { + r = s1 < s2 ? -1 : 1; + } + return r; +}; + +function get_aggregates_with_defaults(aggregate_attribute, schema, cols) { + const found = new Set(); + const aggregates = []; + for (const col of aggregate_attribute) { + const type = schema[col.column]; + found.add(col.column); + if (type) { + if (col.op === "" || perspective.TYPE_AGGREGATES[type].indexOf(col.op) === -1) { + col.op = perspective.AGGREGATE_DEFAULTS[type]; + } + aggregates.push(col); + } else { + console.warn(`No column "${col.column}" found (specified in aggregates attribute).`); + } + } + + // Add columns detected from dataset. + for (const col of cols) { + if (!found.has(col)) { + aggregates.push({ + column: col, + op: perspective.AGGREGATE_DEFAULTS[schema[col]] + }); + } + } + + return aggregates; +} + +function calculate_throttle_timeout(render_time) { + const view_count = document.getElementsByTagName("perspective-viewer").length; + const timeout = render_time * view_count * 2; + return Math.min(10000, Math.max(0, timeout)); +} + +/****************************************************************************** + * + * PerspectiveElement + * + */ + +export class PerspectiveElement extends StateElement { + async _check_recreate_computed_columns() { + const computed_columns = JSON.parse(this.getAttribute("computed-columns")); + if (computed_columns.length > 0) { + for (const col of computed_columns) { + await this._create_computed_column({ + detail: { + column_name: col.name, + input_columns: col.inputs.map(x => ({name: x})), + computation: COMPUTATIONS[col.func] + } + }); + } + this._debounce_update({ignore_size_check: false}); + return true; + } + return false; + } + + async _load_table(table, computed = false) { + this.shadowRoot.querySelector("#app").classList.add("hide_message"); + this.setAttribute("updating", true); + + if (this._table && !computed) { + this.removeAttribute("computed-columns"); + } + + this._clear_state(); + this._table = table; + + if (this.hasAttribute("computed-columns") && !computed) { + if (await this._check_recreate_computed_columns()) { + return; + } + } + + const [cols, schema, computed_schema] = await Promise.all([table.columns(), table.schema(true), table.computed_schema()]); + + this._clear_columns(); + + this._initial_col_order = cols.slice(); + if (!this.hasAttribute("columns")) { + this.setAttribute("columns", JSON.stringify(this._initial_col_order)); + } + + cols.sort(column_sorter(schema)); + + // Update aggregates + const computed_aggregates = Object.entries(computed_schema).map(([column, op]) => ({ + column, + op + })); + + const all_cols = cols.concat(Object.keys(computed_schema)); + const aggregates = get_aggregates_with_defaults(this.get_aggregate_attribute().concat(computed_aggregates), schema, all_cols); + + let shown = JSON.parse(this.getAttribute("columns")).filter(x => all_cols.indexOf(x) > -1); + if (shown.length === 0) { + shown = this._initial_col_order; + } + + this.set_aggregate_attribute(aggregates); + + for (const name of all_cols) { + const aggregate = aggregates.find(a => a.column === name).op; + const row = this._new_row(name, schema[name], aggregate, null, null, computed_schema[name]); + this._inactive_columns.appendChild(row); + if (shown.includes(name)) { + row.classList.add("active"); + } + } + + for (const x of shown) { + const active_row = this._new_row(x, schema[x]); + this._active_columns.appendChild(active_row); + } + + if (all_cols.length === shown.length) { + this._inactive_columns.parentElement.classList.add("collapse"); + } else { + this._inactive_columns.parentElement.classList.remove("collapse"); + } + + this._show_column_selectors(); + await this._debounce_update(); + } + + async _warn_render_size_exceeded() { + if (this._show_warnings && typeof this._plugin.max_size !== "undefined") { + const num_columns = await this._view.num_columns(); + const num_rows = await this._view.num_rows(); + const count = num_columns * num_rows; + if (count >= this._plugin.max_size) { + this._plugin_information.classList.remove("hidden"); + this.removeAttribute("updating"); + return true; + } else { + this._plugin_information.classList.add("hidden"); + } + } + return false; + } + + _view_on_update() { + if (!this._debounced) { + this._debounced = setTimeout(async () => { + this._debounced = undefined; + const timer = this._render_time(); + if (this._task && !this._task.initial) { + this._task.cancel(); + } + const task = (this._task = new CancelTask()); + const updater = this._plugin.update || this._plugin.create; + try { + await updater.call(this, this._datavis, this._view, task); + timer(); + task.cancel(); + } catch (err) { + console.error("Error rendering plugin.", err); + } finally { + this.dispatchEvent(new Event("perspective-view-update")); + } + }, calculate_throttle_timeout(this.getAttribute("render_time"))); + } + } + + async _new_view(ignore_size_check = false) { + if (!this._table) return; + const row_pivots = this._get_view_row_pivots(); + const column_pivots = this._get_view_column_pivots(); + const filters = this._get_view_filters(); + const aggregates = this._get_view_aggregates(); + if (aggregates.length === 0) return; + const sort = this._get_view_sorts(); + const hidden = this._get_view_hidden(aggregates, sort); + for (const s of hidden) { + const all = this.get_aggregate_attribute(); + if (column_pivots.indexOf(s) > -1 || row_pivots.indexOf(s) > -1) { + aggregates.push({column: s, op: "any"}); + } else { + aggregates.push(all.reduce((obj, y) => (y.column === s ? y : obj))); + } + } + + if (this._view) { + this._view.delete(); + this._view = undefined; + } + this._view = this._table.view({ + filter: filters, + row_pivot: row_pivots, + column_pivot: column_pivots, + aggregate: aggregates, + sort: sort + }); + + if (!ignore_size_check) { + if (await this._warn_render_size_exceeded()) { + return; + } + } + + this._view.on_update(() => this._view_on_update()); + + const timer = this._render_time(); + this._render_count = (this._render_count || 0) + 1; + if (this._task) { + this._task.cancel(); + } + + const task = (this._task = new CancelTask(() => this._render_count--, true)); + + try { + await this._plugin.create.call(this, this._datavis, this._view, task); + } catch (err) { + console.warn(err); + } finally { + if (!this.hasAttribute("render_time")) { + this.dispatchEvent(new Event("perspective-view-update")); + } + timer(); + task.cancel(); + if (this._render_count === 0) { + this.removeAttribute("updating"); + } + } + } + + _render_time() { + const t = performance.now(); + return () => this.setAttribute("render_time", performance.now() - t); + } + + _clear_state() { + if (this._task) { + this._task.cancel(); + } + const all = []; + if (this._view) { + const view = this._view; + this._view = undefined; + all.push(view.delete()); + } + if (this._table) { + const table = this._table; + this._table = undefined; + if (table._owner_viewer && table._owner_viewer === this) { + all.push(table.delete()); + } + } + return Promise.all(all); + } + + // setup for update + _register_debounce_instance() { + const _update = _.debounce((resolve, ignore_size_check) => { + this._new_view(ignore_size_check).then(resolve); + }, 10); + this._debounce_update = async ({ignore_size_check = false} = {}) => { + this.setAttribute("updating", true); + await new Promise(resolve => _update(resolve, ignore_size_check)); + }; + } + + _get_worker() { + if (this._table) { + return this._table._worker; + } + return WORKER_SINGLETON.getInstance(); + } +} diff --git a/packages/perspective-viewer/src/js/viewer/renderers.js b/packages/perspective-viewer/src/js/viewer/renderers.js new file mode 100644 index 0000000000..b94cf215e0 --- /dev/null +++ b/packages/perspective-viewer/src/js/viewer/renderers.js @@ -0,0 +1,54 @@ +/****************************************************************************** + * + * Copyright (c) 2018, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ + +const RENDERERS = {}; + +export const renderers = new class { + /** + * Register a plugin with the component. + * + * @param {string} name The logical unique name of the plugin. This will be + * used to set the component's `view` attribute. + * @param {object} plugin An object with this plugin's prototype. Valid keys are: + * name : The display name for this plugin. + * create (required) : The creation function - may return a `Promise`. + * delete : The deletion function. + * mode : The selection mode - may be "toggle" or "select". + */ + registerPlugin(name, plugin) { + RENDERERS[name] = plugin; + } + + getPlugin(name) { + return RENDERERS[name]; + } + + getInstance() { + return RENDERERS; + } +}(); + +global.registerPlugin = renderers.registerPlugin; + +global.getPlugin = renderers.getPlugin; + +export function register_debug_plugin() { + global.registerPlugin("debug", { + name: "Debug", + create: async function(div) { + const csv = await this._view.to_csv({config: {delimiter: "|"}}); + const timer = this._render_time(); + div.innerHTML = `
${csv}
`; + timer(); + }, + selectMode: "toggle", + resize: function() {}, + delete: function() {} + }); +} diff --git a/packages/perspective-viewer/src/js/viewer/state_element.js b/packages/perspective-viewer/src/js/viewer/state_element.js new file mode 100644 index 0000000000..695f9954fd --- /dev/null +++ b/packages/perspective-viewer/src/js/viewer/state_element.js @@ -0,0 +1,100 @@ +/****************************************************************************** + * + * Copyright (c) 2018, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ + +import {renderers} from "./renderers.js"; + +export class StateElement extends HTMLElement { + get _plugin() { + let current_renderers = renderers.getInstance(); + let view = this.getAttribute("view"); + if (!view) { + view = Object.keys(current_renderers)[0]; + } + this.setAttribute("view", view); + return current_renderers[view] || current_renderers[Object.keys(current_renderers)[0]]; + } + + _get_view_dom_columns(selector, callback) { + selector = selector || "#active_columns perspective-row"; + let columns = Array.prototype.slice.call(this.shadowRoot.querySelectorAll(selector)); + if (!callback) { + return columns; + } + return columns.map(callback); + } + + _get_view_columns({active = true} = {}) { + let selector; + if (active) { + selector = "#active_columns perspective-row"; + } else { + selector = "#inactive_columns perspective-row"; + } + return this._get_view_dom_columns(selector, col => { + return col.getAttribute("name"); + }); + } + + _get_view_aggregates(selector) { + selector = selector || "#active_columns perspective-row"; + return this._get_view_dom_columns(selector, s => { + return { + op: s.getAttribute("aggregate"), + column: s.getAttribute("name") + }; + }); + } + + _get_view_row_pivots() { + return this._get_view_dom_columns("#row_pivots perspective-row", col => { + return col.getAttribute("name"); + }); + } + + _get_view_column_pivots() { + return this._get_view_dom_columns("#column_pivots perspective-row", col => { + return col.getAttribute("name"); + }); + } + + _get_view_filters() { + return this._get_view_dom_columns("#filters perspective-row", col => { + let {operator, operand} = JSON.parse(col.getAttribute("filter")); + return [col.getAttribute("name"), operator, operand]; + }); + } + + _get_view_sorts() { + return this._get_view_dom_columns("#sort perspective-row", col => { + let order = col.getAttribute("sort-order") || "asc"; + return [col.getAttribute("name"), order]; + }); + } + + _get_view_hidden(aggregates, sort) { + aggregates = aggregates || this._get_view_aggregates(); + let hidden = []; + sort = sort || this._get_view_sorts(); + for (let s of sort) { + if (aggregates.map(agg => agg.column).indexOf(s[0]) === -1) { + hidden.push(s[0]); + } + } + return hidden; + } + + _get_visible_column_count() { + return this._get_view_dom_columns().length; + } + + get_aggregate_attribute() { + const aggs = JSON.parse(this.getAttribute("aggregates")) || {}; + return Object.keys(aggs).map(col => ({column: col, op: aggs[col]})); + } +} diff --git a/packages/perspective-viewer/src/less/view.less b/packages/perspective-viewer/src/less/viewer.less similarity index 100% rename from packages/perspective-viewer/src/less/view.less rename to packages/perspective-viewer/src/less/viewer.less diff --git a/packages/perspective-viewer/test/html/blank.html b/packages/perspective-viewer/test/html/blank.html index 1d63a4f52d..cb78641a6d 100644 --- a/packages/perspective-viewer/test/html/blank.html +++ b/packages/perspective-viewer/test/html/blank.html @@ -12,7 +12,7 @@ - + diff --git a/packages/perspective-viewer/test/html/superstore.html b/packages/perspective-viewer/test/html/superstore.html index 962ee27161..f2617da75d 100644 --- a/packages/perspective-viewer/test/html/superstore.html +++ b/packages/perspective-viewer/test/html/superstore.html @@ -12,7 +12,7 @@ - + diff --git a/packages/perspective-viewer/test/js/computed_column_tests.js b/packages/perspective-viewer/test/js/computed_column_tests.js deleted file mode 100644 index 97b93226b9..0000000000 --- a/packages/perspective-viewer/test/js/computed_column_tests.js +++ /dev/null @@ -1,195 +0,0 @@ -/****************************************************************************** - * - * Copyright (c) 2017, the Perspective Authors. - * - * This file is part of the Perspective library, distributed under the terms of - * the Apache License 2.0. The full license can be found in the LICENSE file. - * - */ - -const add_computed_column = async page => { - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); - await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); - await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); - await page.evaluate(element => { - let com = element.shadowRoot.querySelector("perspective-computed-column"); - const columns = [{name: "Order Date", type: "datetime"}]; - com._apply_state(columns, com.computations["day_of_week"], "new_cc"); - }, viewer); - await page.evaluate( - element => - element.shadowRoot - .querySelector("perspective-computed-column") - .shadowRoot.querySelector("#psp-cc-button-save") - .click(), - viewer - ); - await page.waitForSelector("perspective-viewer:not([updating])"); - await page.evaluate(element => element.setAttribute("aggregates", '{"new_cc":"dominant"}'), viewer); - await page.waitForSelector("perspective-viewer:not([updating])"); -}; - -exports.default = function() { - // basic UI tests - test.capture("click on add computed column button opens the UI.", async page => { - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); - await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); - await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); - }); - - test.capture("click on close button closes the UI.", async page => { - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); - await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); - await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); - await page.evaluate( - element => - element.shadowRoot - .querySelector("perspective-computed-column") - .shadowRoot.querySelector("#psp-cc__close") - .click(), - viewer - ); - }); - - // input column - test.capture("setting a valid column should set it as input.", async page => { - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); - await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); - await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); - await page.evaluate(element => { - let com = element.shadowRoot.querySelector("perspective-computed-column"); - const columns = [{name: "State", type: "string"}]; - com._apply_state(columns, com.computations["lowercase"], "new_cc"); - }, viewer); - }); - - test.capture("setting multiple column parameters should set input.", async page => { - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); - await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); - await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); - await page.evaluate(element => { - let com = element.shadowRoot.querySelector("perspective-computed-column"); - const columns = [{name: "Quantity", type: "integer"}, {name: "Row ID", type: "integer"}]; - com._apply_state(columns, com.computations["add"], "new_cc"); - }, viewer); - }); - - // computation - test.capture("computations should clear input column.", async page => { - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); - await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); - await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); - await page.evaluate(element => { - let com = element.shadowRoot.querySelector("perspective-computed-column"); - const columns = [{name: "State", type: "string"}]; - com._apply_state(columns, com.computations["lowercase"], "new_cc"); - }, viewer); - await page.evaluate(element => { - const select = element.shadowRoot.querySelector("perspective-computed-column").shadowRoot.querySelector("#psp-cc-computation__select"); - select.value = "subtract"; - select.dispatchEvent(new Event("change")); - }, viewer); - // await page.select("#psp-cc-computation__select", "subtract"); - }); - - // save - test.capture("saving a computed column should add it to inactive columns.", async page => { - await add_computed_column(page); - }); - - test.capture("saving without parameters should fail as button is disabled.", async page => { - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); - await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); - await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); - await page.evaluate( - element => - element.shadowRoot - .querySelector("perspective-computed-column") - .shadowRoot.querySelector("#psp-cc-button-save") - .click(), - viewer - ); - }); - - test.capture("saving a duplicate column should fail with error message.", async page => { - await add_computed_column(page); - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); - await page.evaluate(element => { - let com = element.shadowRoot.querySelector("perspective-computed-column"); - const columns = [{name: "Order Date", type: "datetime"}]; - com._apply_state(columns, com.computations["day_of_week"], "new_cc"); - }, viewer); - await page.evaluate( - element => - element.shadowRoot - .querySelector("perspective-computed-column") - .shadowRoot.querySelector("#psp-cc-button-save") - .click(), - viewer - ); - }); - - // edit - test.skip("clicking on the edit button should bring up the UI", async page => { - await add_computed_column(page); - await page.click("#row_edit"); - }); - - // usage - test.capture("aggregates by computed column.", async page => { - await add_computed_column(page); - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.setAttribute("row-pivots", '["Quantity"]'), viewer); - await page.evaluate(element => element.setAttribute("columns", '["Row ID", "Quantity", "new_cc"]'), viewer); - }); - - test.capture("pivots by computed column.", async page => { - await add_computed_column(page); - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.setAttribute("row-pivots", '["new_cc"]'), viewer); - }); - - test.capture("adds computed column via attribute", async page => { - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); - await page.evaluate(element => element.setAttribute("computed-columns", '[{"name":"test","func":"month_bucket","inputs":["Order Date"]}]'), viewer); - await page.waitForSelector("perspective-viewer:not([updating])"); - await page.evaluate(element => element.setAttribute("columns", '["Quantity", "test"]'), viewer); - await page.waitForSelector("perspective-viewer:not([updating])"); - }); - - test.capture("sorts by computed column.", async page => { - await add_computed_column(page); - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.setAttribute("sort", '["new_cc"]'), viewer); - }); - - test.capture("filters by computed column.", async page => { - await add_computed_column(page); - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.setAttribute("filters", '[["new_cc", "==", "2 Monday"]]'), viewer); - }); - - test.capture("computed column aggregates should persist.", async page => { - await add_computed_column(page); - const viewer = await page.$("perspective-viewer"); - await page.evaluate(element => element.setAttribute("row-pivots", '["Quantity"]'), viewer); - await page.evaluate(element => element.setAttribute("columns", '["Row ID", "Quantity", "new_cc"]'), viewer); - await page.evaluate(element => element.setAttribute("aggregates", '{"new_cc":"any"}'), viewer); - await page.evaluate(element => element.setAttribute("columns", '["Row ID", "Quantity"]'), viewer); - await page.evaluate(element => element.setAttribute("columns", '["Row ID", "Quantity", "new_cc"]'), viewer); - }); -}; diff --git a/packages/perspective-viewer/test/js/computed_columns.spec.js b/packages/perspective-viewer/test/js/computed_columns.spec.js new file mode 100644 index 0000000000..741ad8f6aa --- /dev/null +++ b/packages/perspective-viewer/test/js/computed_columns.spec.js @@ -0,0 +1,207 @@ +/****************************************************************************** + * + * Copyright (c) 2017, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ + +/****************************************************************************** + * + * Copyright (c) 2017, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ + +const utils = require("./utils.js"); + +const add_computed_column = async page => { + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); + await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); + await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); + await page.evaluate(element => { + let com = element.shadowRoot.querySelector("perspective-computed-column"); + const columns = [{name: "Order Date", type: "datetime"}]; + com._apply_state(columns, com.computations["day_of_week"], "new_cc"); + }, viewer); + await page.evaluate( + element => + element.shadowRoot + .querySelector("perspective-computed-column") + .shadowRoot.querySelector("#psp-cc-button-save") + .click(), + viewer + ); + await page.waitForSelector("perspective-viewer:not([updating])"); + await page.evaluate(element => element.setAttribute("aggregates", '{"new_cc":"dominant"}'), viewer); + await page.waitForSelector("perspective-viewer:not([updating])"); +}; + +utils.with_server({}, () => { + describe.page("superstore.html", () => { + test.capture("click on add computed column button opens the UI.", async page => { + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); + await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); + await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); + }); + + test.capture("click on close button closes the UI.", async page => { + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); + await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); + await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); + await page.evaluate( + element => + element.shadowRoot + .querySelector("perspective-computed-column") + .shadowRoot.querySelector("#psp-cc__close") + .click(), + viewer + ); + }); + + // input column + test.capture("setting a valid column should set it as input.", async page => { + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); + await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); + await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); + await page.evaluate(element => { + let com = element.shadowRoot.querySelector("perspective-computed-column"); + const columns = [{name: "State", type: "string"}]; + com._apply_state(columns, com.computations["lowercase"], "new_cc"); + }, viewer); + }); + + test.capture("setting multiple column parameters should set input.", async page => { + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); + await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); + await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); + await page.evaluate(element => { + let com = element.shadowRoot.querySelector("perspective-computed-column"); + const columns = [{name: "Quantity", type: "integer"}, {name: "Row ID", type: "integer"}]; + com._apply_state(columns, com.computations["add"], "new_cc"); + }, viewer); + }); + + // computation + test.capture("computations should clear input column.", async page => { + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); + await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); + await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); + await page.evaluate(element => { + let com = element.shadowRoot.querySelector("perspective-computed-column"); + const columns = [{name: "State", type: "string"}]; + com._apply_state(columns, com.computations["lowercase"], "new_cc"); + }, viewer); + await page.evaluate(element => { + const select = element.shadowRoot.querySelector("perspective-computed-column").shadowRoot.querySelector("#psp-cc-computation__select"); + select.value = "subtract"; + select.dispatchEvent(new Event("change")); + }, viewer); + // await page.select("#psp-cc-computation__select", "subtract"); + }); + + // save + test.capture("saving a computed column should add it to inactive columns.", async page => { + await add_computed_column(page); + }); + + test.capture("saving without parameters should fail as button is disabled.", async page => { + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); + await page.evaluate(element => element.setAttribute("columns", '["Row ID","Quantity"]'), viewer); + await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); + await page.evaluate( + element => + element.shadowRoot + .querySelector("perspective-computed-column") + .shadowRoot.querySelector("#psp-cc-button-save") + .click(), + viewer + ); + }); + + test.capture("saving a duplicate column should fail with error message.", async page => { + await add_computed_column(page); + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#add-computed-column").click(), viewer); + await page.evaluate(element => { + let com = element.shadowRoot.querySelector("perspective-computed-column"); + const columns = [{name: "Order Date", type: "datetime"}]; + com._apply_state(columns, com.computations["day_of_week"], "new_cc"); + }, viewer); + await page.evaluate( + element => + element.shadowRoot + .querySelector("perspective-computed-column") + .shadowRoot.querySelector("#psp-cc-button-save") + .click(), + viewer + ); + }); + + // edit + test.skip("clicking on the edit button should bring up the UI", async page => { + await add_computed_column(page); + await page.click("#row_edit"); + }); + + // usage + test.capture("aggregates by computed column.", async page => { + await add_computed_column(page); + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.setAttribute("row-pivots", '["Quantity"]'), viewer); + await page.evaluate(element => element.setAttribute("columns", '["Row ID", "Quantity", "new_cc"]'), viewer); + }); + + test.capture("pivots by computed column.", async page => { + await add_computed_column(page); + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.setAttribute("row-pivots", '["new_cc"]'), viewer); + }); + + test.capture("adds computed column via attribute", async page => { + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.shadowRoot.querySelector("#config_button").click(), viewer); + await page.evaluate(element => element.setAttribute("computed-columns", '[{"name":"test","func":"month_bucket","inputs":["Order Date"]}]'), viewer); + await page.waitForSelector("perspective-viewer:not([updating])"); + await page.evaluate(element => element.setAttribute("columns", '["Quantity", "test"]'), viewer); + await page.waitForSelector("perspective-viewer:not([updating])"); + }); + + test.capture("sorts by computed column.", async page => { + await add_computed_column(page); + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.setAttribute("sort", '["new_cc"]'), viewer); + }); + + test.capture("filters by computed column.", async page => { + await add_computed_column(page); + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.setAttribute("filters", '[["new_cc", "==", "2 Monday"]]'), viewer); + }); + + test.capture("computed column aggregates should persist.", async page => { + await add_computed_column(page); + const viewer = await page.$("perspective-viewer"); + await page.evaluate(element => element.setAttribute("row-pivots", '["Quantity"]'), viewer); + await page.evaluate(element => element.setAttribute("columns", '["Row ID", "Quantity", "new_cc"]'), viewer); + await page.evaluate(element => element.setAttribute("aggregates", '{"new_cc":"any"}'), viewer); + await page.evaluate(element => element.setAttribute("columns", '["Row ID", "Quantity"]'), viewer); + await page.evaluate(element => element.setAttribute("columns", '["Row ID", "Quantity", "new_cc"]'), viewer); + }); + }); +}); diff --git a/packages/perspective-viewer/test/js/superstore.spec.js b/packages/perspective-viewer/test/js/superstore.spec.js index d9a0a81f4b..d4728909c8 100644 --- a/packages/perspective-viewer/test/js/superstore.spec.js +++ b/packages/perspective-viewer/test/js/superstore.spec.js @@ -10,17 +10,12 @@ const utils = require("./utils.js"); const simple_tests = require("./simple_tests.js"); -const computed_column_tests = require("./computed_column_tests.js"); const responsive_tests = require("./responsive_tests"); utils.with_server({}, () => { describe.page("superstore.html", () => { simple_tests.default(); - describe("Computed Columns", () => { - computed_column_tests.default(); - }); - describe("Responsive Layout", () => { responsive_tests.default(); }); diff --git a/packages/perspective-viewer/test/js/utils.js b/packages/perspective-viewer/test/js/utils.js index edfa889245..afd850510c 100644 --- a/packages/perspective-viewer/test/js/utils.js +++ b/packages/perspective-viewer/test/js/utils.js @@ -69,7 +69,8 @@ exports.with_server = function with_server({paths = DEFAULT}, body) { beforeAll(() => server.listen(0, () => { __PORT__ = server.address().port; - })); + }) + ); afterAll(() => server.close()); @@ -97,7 +98,7 @@ let browser, __name = ""; beforeAll(async () => { - browser = await puppeteer.launch({args: ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage"]}); + browser = await puppeteer.launch({args: ["--no-sandbox", "--disable-setuid-sandbox", "--disable-dev-shm-usage", '--proxy-server="direct://"', "--proxy-bypass-list=*"]}); page = await browser.newPage(); // CSS Animations break our screenshot tests, so set the diff --git a/packages/perspective/bench/js/report.js b/packages/perspective/bench/js/report.js index ae4afa6d1c..6fcd842908 100644 --- a/packages/perspective/bench/js/report.js +++ b/packages/perspective/bench/js/report.js @@ -7,7 +7,7 @@ * */ -import "../less/benchmark.less"; +import "!!style-loader!css-loader!less-loader!../less/benchmark.less"; import CodeMirror from 'codemirror'; import '!!style-loader!css-loader!codemirror/lib/codemirror.css'; diff --git a/packages/perspective/bench/results/results.json b/packages/perspective/bench/results/results.json index 6c74b5a960..346802b65c 100644 --- a/packages/perspective/bench/results/results.json +++ b/packages/perspective/bench/results/results.json @@ -1 +1 @@ -{"data":[{"code":"\n_perspectiveNode2.default.table(csv);\nresolve();\n","bins":[[315.76230300962925,3],[320,8],[325,5],[330,16],[335,35],[340,22],[345,11],[350,3],[355,4],[360,3],[365,8],[370,8],[375,6],[380,24],[385,26],[390,5],[395,4],[400,3],[405,1],[410,0],[415,0],[420,1],[425,1],[430,1],[435,0],[440,0],[445,1],[450,0],[455,0],[460,0],[465,0],[470,0],[475,0],[480,0],[485,0],[490,0],[495,0],[500,0],[505,0],[510,0],[515,0],[520,1],[525,0],[530,0],[535,0],[540,0],[545,0],[550,0],[555,0],[560,0],[565,0],[570,0],[575,0],[580,0],[585,0],[590,0],[595,0],[600,0],[605,0],[610,0],[615,0],[620,0],[625,0],[630,0],[635,0],[640,0],[645,0],[650,0],[655,0],[660,0],[665,0],[670,0],[675,0],[680,0],[685,0],[690,0],[695,0],[700,0],[705,0],[710,0],[715,0],[720,0],[725,0],[730,0],[735,0],[740,0],[745,0],[750,0],[755,0],[760,0],[765,0],[770,0],[775,0],[780,0],[785,0],[790,0],[795,0],[800,0],[805,0],[810,0],[815,1]],"avg":362.7220161792947,"std":42.69973266375656,"avg_diff":"-2.48%","std_diff":"-20.77%","old_avg":371.7116352541382,"old_std":51.567505990570744},{"code":"\nvar view = table.view({\n filter: [[\"Origin\", \"contains\", \"P\"]],\n aggregate: \"count\"\n});\nview.delete();\nresolve();\n","bins":[[14.16473700106144,1],[14.200000000000001,1],[14.3,1],[14.4,1],[14.500000000000002,0],[14.600000000000001,0],[14.700000000000001,1],[14.8,0],[14.9,2],[15.000000000000002,2],[15.100000000000001,2],[15.200000000000001,2],[15.3,3],[15.400000000000002,1],[15.500000000000002,0],[15.600000000000001,0],[15.700000000000001,3],[15.8,6],[15.900000000000002,4],[16,6],[16.1,5],[16.200000000000003,5],[16.3,5],[16.400000000000002,8],[16.5,7],[16.6,13],[16.700000000000003,9],[16.8,17],[16.900000000000002,22],[17,27],[17.1,26],[17.200000000000003,27],[17.3,26],[17.400000000000002,28],[17.5,33],[17.6,30],[17.700000000000003,24],[17.8,22],[17.900000000000002,28],[18,22],[18.1,32],[18.200000000000003,35],[18.3,40],[18.400000000000002,37],[18.5,51],[18.6,43],[18.700000000000003,56],[18.8,68],[18.900000000000002,78],[19,73],[19.1,99],[19.200000000000003,75],[19.3,77],[19.400000000000002,76],[19.5,86],[19.6,74],[19.700000000000003,73],[19.8,79],[19.900000000000002,58],[20,64],[20.1,59],[20.200000000000003,63],[20.3,50],[20.400000000000002,30],[20.5,25],[20.6,15],[20.700000000000003,18],[20.8,7],[20.900000000000002,8],[21,7],[21.1,2],[21.200000000000003,3],[21.3,2],[21.400000000000002,2],[21.5,1],[21.6,2],[21.700000000000003,0],[21.8,3],[21.900000000000002,2],[22,0],[22.1,1],[22.200000000000003,0],[22.3,1],[22.400000000000002,1],[22.5,1],[22.6,1],[22.700000000000003,0],[22.8,0],[22.900000000000002,0],[23,0],[23.1,0],[23.200000000000003,0],[23.3,0],[23.400000000000002,0],[23.5,0],[23.6,0],[23.700000000000003,0],[23.800000000000004,0],[23.900000000000002,0],[24,0],[24.1,0],[24.200000000000003,0],[24.300000000000004,0],[24.400000000000002,0],[24.5,0],[24.6,1],[24.700000000000003,0],[24.800000000000004,0],[24.900000000000002,0],[25,0],[25.1,0],[25.200000000000003,0],[25.300000000000004,0],[25.400000000000002,0],[25.5,0],[25.6,0],[25.700000000000003,0],[25.800000000000004,0],[25.900000000000002,0],[26,0],[26.1,0],[26.200000000000003,0],[26.300000000000004,0],[26.400000000000002,0],[26.5,0],[26.6,0],[26.700000000000003,1],[26.800000000000004,1]],"avg":18.99126514641271,"std":1.2033541007277022,"avg_diff":"+1.55%","std_diff":"-0.07%","old_avg":18.697776202125468,"old_std":1.204138619303178},{"code":"\nvar view = table.view({\n filter: [[\"Origin\", \"contains\", \"P\"]],\n aggregate: \"count\"\n});\nview.to_json({\n end_row: 10\n}).then(function () {\n view.delete();\n resolve();\n});\n","bins":[[14.451049998402596,1],[14.5,0],[14.6,1],[14.7,0],[14.8,1],[14.9,0],[15,2],[15.1,2],[15.2,0],[15.3,3],[15.4,5],[15.5,1],[15.6,3],[15.7,4],[15.8,4],[15.9,2],[16,6],[16.1,0],[16.2,5],[16.3,2],[16.4,3],[16.5,10],[16.6,11],[16.7,11],[16.8,11],[16.9,10],[17,16],[17.1,22],[17.2,14],[17.3,28],[17.4,29],[17.5,25],[17.6,38],[17.7,33],[17.8,42],[17.9,29],[18,40],[18.1,38],[18.2,28],[18.3,27],[18.4,34],[18.5,41],[18.6,50],[18.7,35],[18.8,42],[18.9,32],[19,54],[19.1,60],[19.2,75],[19.3,70],[19.4,86],[19.5,80],[19.6,87],[19.7,69],[19.8,94],[19.9,68],[20,86],[20.1,72],[20.2,61],[20.3,58],[20.4,51],[20.5,40],[20.6,38],[20.7,28],[20.8,17],[20.9,20],[21,9],[21.1,8],[21.2,4],[21.3,4],[21.4,3],[21.5,3],[21.6,2],[21.7,2],[21.8,0],[21.9,0],[22,0],[22.1,2],[22.2,1],[22.3,1],[22.4,2],[22.5,0],[22.6,0],[22.700000000000003,1],[22.8,0],[22.9,0],[23,0],[23.1,0],[23.200000000000003,0],[23.3,0],[23.4,0],[23.5,1],[23.6,0],[23.700000000000003,0],[23.8,1],[23.9,0],[24,0],[24.1,0],[24.200000000000003,0],[24.3,0],[24.4,0],[24.5,0],[24.6,0],[24.700000000000003,0],[24.8,1],[24.9,1]],"avg":19.159274226535803,"std":1.2176060814371967,"avg_diff":"+1.45%","std_diff":"-2.25%","old_avg":18.882164844732056,"old_std":1.2450121970538315},{"code":"\nvar view = table.view({\n filter: [[\"Origin\", \"contains\", \"P\"]],\n row_pivot: ['Dest'],\n aggregate: \"count\"\n});\nview.to_json().then(function () {\n view.delete();\n resolve();\n});\n","bins":[[42.097451999783516,1],[42.5,0],[43,3],[43.5,10],[44,23],[44.5,29],[45,56],[45.5,92],[46,141],[46.5,189],[47,237],[47.5,283],[48,302],[48.5,228],[49,151],[49.5,87],[50,55],[50.5,34],[51,26],[51.5,12],[52,13],[52.5,8],[53,3],[53.5,3],[54,1],[54.5,0],[55,4],[55.5,2],[56,0],[56.5,0],[57,1],[57.5,1],[58,2],[58.5,0],[59,0],[59.5,0],[60,0],[60.5,0],[61,0],[61.5,0],[62,1],[62.5,1],[63,0],[63.5,0],[64,0],[64.5,0],[65,0],[65.5,1],[66,0],[66.5,0],[67,0],[67.5,0],[68,0],[68.5,0],[69,0],[69.5,0],[70,0],[70.5,0],[71,0],[71.5,0],[72,0],[72.5,0],[73,0],[73.5,1]],"avg":47.93699074158634,"std":1.87018674304559,"avg_diff":"-2.65%","std_diff":"-99.69%","old_avg":49.20633878484778,"old_std":3.7345842105732636},{"code":"\nvar view = table.view({\n filter: [[\"Origin\", \"contains\", \"P\"]],\n aggregate: \"count\"\n});\nview.to_json().then(function () {\n view.delete();\n resolve();\n});\n","bins":[[72.860749989748,1],[73,0],[73.2,1],[73.4,0],[73.6,0],[73.8,0],[74,0],[74.2,0],[74.4,0],[74.6,0],[74.8,0],[75,0],[75.2,0],[75.4,0],[75.6,3],[75.8,2],[76,3],[76.2,4],[76.4,5],[76.6,4],[76.8,3],[77,8],[77.2,4],[77.4,3],[77.6,13],[77.8,10],[78,12],[78.2,8],[78.4,12],[78.6,10],[78.8,11],[79,10],[79.2,5],[79.4,20],[79.6,11],[79.8,7],[80,18],[80.2,12],[80.4,6],[80.6,8],[80.8,12],[81,11],[81.2,9],[81.4,10],[81.6,13],[81.8,18],[82,9],[82.2,10],[82.4,5],[82.6,6],[82.8,12],[83,16],[83.2,10],[83.4,11],[83.6,10],[83.8,10],[84,2],[84.2,6],[84.4,9],[84.6,7],[84.8,8],[85,5],[85.2,3],[85.4,3],[85.6,5],[85.8,3],[86,2],[86.2,1],[86.4,0],[86.6,1],[86.8,2],[87,1],[87.2,2],[87.4,0],[87.6,0],[87.8,0],[88,0],[88.2,0],[88.4,0],[88.6,0],[88.8,0],[89,0],[89.2,0],[89.4,0],[89.6,0],[89.8,0],[90,1],[90.2,0],[90.4,0],[90.6,0],[90.8,0],[91,0],[91.2,2],[91.4,0],[91.6,2],[91.8,3],[92,2],[92.2,2],[92.4,4],[92.6,1],[92.8,5],[93,8],[93.2,5],[93.4,8],[93.6,14],[93.8,12],[94,5],[94.2,12],[94.4,1],[94.6,5],[94.8,6],[95,8],[95.2,6],[95.4,4],[95.6,3],[95.8,6],[96,2],[96.2,3],[96.4,4],[96.6,3],[96.8,2],[97,3],[97.2,5],[97.4,2],[97.6,1],[97.8,0],[98,1],[98.2,0],[98.4,0],[98.6,1],[98.8,0],[99,0],[99.2,0],[99.4,0],[99.6,1],[99.8,0],[100,0],[100.2,0],[100.4,0],[100.6,0],[100.8,0],[101,0],[101.2,0],[101.4,0],[101.6,0],[101.8,0],[102,0],[102.2,0],[102.4,0],[102.6,0],[102.8,0],[103,0],[103.2,1],[103.4,1]],"avg":84.49366870381174,"std":6.470284726490922,"avg_diff":"-4.47%","std_diff":"-29.02%","old_avg":88.26828702014814,"old_std":8.348132426753132},{"code":"\nvar view = table.view({\n row_pivot: ['Dest'],\n aggregate: \"count\",\n row_pivot_depth: 1\n});\nview.to_json().then(function () {\n view.delete();\n resolve();\n});\n","bins":[[95.60043799877167,2],[95.80000000000001,0],[96.00000000000001,0],[96.20000000000002,0],[96.4,0],[96.60000000000001,2],[96.80000000000001,0],[97.00000000000001,0],[97.20000000000002,2],[97.4,1],[97.60000000000001,3],[97.80000000000001,0],[98.00000000000001,1],[98.20000000000002,0],[98.4,1],[98.60000000000001,7],[98.80000000000001,4],[99.00000000000001,2],[99.20000000000002,7],[99.4,4],[99.60000000000001,10],[99.80000000000001,9],[100.00000000000001,10],[100.20000000000002,18],[100.4,21],[100.60000000000001,8],[100.80000000000001,18],[101.00000000000001,20],[101.20000000000002,26],[101.4,29],[101.60000000000001,21],[101.80000000000001,18],[102.00000000000001,27],[102.20000000000002,23],[102.4,25],[102.60000000000001,21],[102.80000000000001,21],[103.00000000000001,16],[103.20000000000002,18],[103.4,14],[103.60000000000001,17],[103.80000000000001,17],[104.00000000000001,13],[104.20000000000002,17],[104.4,9],[104.60000000000001,10],[104.80000000000001,5],[105.00000000000001,6],[105.20000000000002,5],[105.4,6],[105.60000000000001,9],[105.80000000000001,8],[106.00000000000001,9],[106.20000000000002,5],[106.4,7],[106.60000000000001,6],[106.80000000000001,1],[107.00000000000001,3],[107.20000000000002,2],[107.4,6],[107.60000000000001,5],[107.80000000000001,1],[108.00000000000001,1],[108.20000000000002,6],[108.4,1],[108.60000000000001,1],[108.80000000000001,2],[109.00000000000001,2],[109.20000000000002,0],[109.4,1],[109.60000000000001,0],[109.80000000000001,0],[110.00000000000001,3],[110.20000000000002,0],[110.4,0],[110.60000000000001,0],[110.80000000000001,1],[111.00000000000001,0],[111.20000000000002,0],[111.4,2],[111.60000000000001,0],[111.80000000000001,0],[112.00000000000001,0],[112.20000000000002,0],[112.4,1],[112.60000000000001,0],[112.80000000000001,1],[113.00000000000001,0],[113.20000000000002,0],[113.4,0],[113.60000000000001,0],[113.80000000000001,0],[114.00000000000001,0],[114.20000000000002,0],[114.4,0],[114.60000000000001,0],[114.80000000000001,0],[115.00000000000001,0],[115.20000000000002,1],[115.4,0],[115.60000000000001,0],[115.80000000000001,0],[116.00000000000001,0],[116.20000000000002,0],[116.4,0],[116.60000000000001,0],[116.80000000000001,0],[117.00000000000001,0],[117.20000000000002,0],[117.4,0],[117.60000000000001,0],[117.80000000000001,0],[118.00000000000001,0],[118.20000000000002,0],[118.4,0],[118.60000000000001,0],[118.80000000000001,0],[119.00000000000001,0],[119.20000000000002,0],[119.4,1],[119.60000000000001,1]],"avg":102.87101471719151,"std":2.7302324819374606,"avg_diff":"-2.50%","std_diff":"-301.57%","old_avg":105.43767548601659,"old_std":10.963836692839665},{"code":"\nvar view = table.view({\n row_pivot: ['Dest'],\n aggregate: \"mean\",\n row_pivot_depth: 1\n});\nview.to_json().then(function () {\n view.delete();\n resolve();\n});\n","bins":[[95.58404798805714,1],[96,0],[96.5,1],[97,2],[97.5,4],[98,8],[98.5,9],[99,22],[99.5,34],[100,40],[100.5,51],[101,54],[101.5,46],[102,55],[102.5,55],[103,42],[103.5,34],[104,26],[104.5,17],[105,22],[105.5,10],[106,15],[106.5,14],[107,7],[107.5,8],[108,3],[108.5,5],[109,4],[109.5,1],[110,1],[110.5,0],[111,3],[111.5,1],[112,0],[112.5,0],[113,0],[113.5,0],[114,0],[114.5,0],[115,1],[115.5,0],[116,0],[116.5,1],[117,0],[117.5,0],[118,0],[118.5,0],[119,0],[119.5,0],[120,1],[120.5,0],[121,0],[121.5,0],[122,0],[122.5,0],[123,0],[123.5,0],[124,0],[124.5,0],[125,0],[125.5,0],[126,2],[126.5,0],[127,0],[127.5,0],[128,0],[128.5,0],[129,0],[129.5,0],[130,0],[130.5,1]],"avg":102.7086531924775,"std":3.2325664504347027,"avg_diff":"-20.97%","std_diff":"-598.67%","old_avg":124.24623680072597,"old_std":22.585001138121537}],"model":"Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz","release":"17.5.0","platform":"darwin"} \ No newline at end of file +{"data":[{"code":"\n__WEBPACK_IMPORTED_MODULE_8__src_js_perspective_node_js___default.a.table(csv);\nresolve();\n","bins":[[310.46748304367065,5],[315,29],[320,5],[325,17],[330,30],[335,22],[340,13],[345,6],[350,10],[355,3],[360,11],[365,9],[370,6],[375,11],[380,6],[385,10],[390,2],[395,2],[400,1],[405,0],[410,1],[415,0],[420,0],[425,0],[430,0],[435,0],[440,0],[445,0],[450,0],[455,0],[460,0],[465,0],[470,0],[475,0],[480,1],[485,0],[490,0],[495,0],[500,0],[505,0],[510,0],[515,0],[520,0],[525,0],[530,0],[535,0],[540,0],[545,0],[550,0],[555,0],[560,0],[565,0],[570,0],[575,0],[580,0],[585,0],[590,0],[595,0],[600,0],[605,0],[610,0],[615,0],[620,0],[625,0],[630,0],[635,0],[640,0],[645,1]],"avg":347.62833713061775,"std":32.89927603645628,"avg_diff":"-4.34%","std_diff":"-29.79%","old_avg":362.7220161792947,"old_std":42.69973266375656},{"code":"\nvar view = table.view({\n filter: [[\"Origin\", \"contains\", \"P\"]],\n aggregate: \"count\"\n});\nview.delete();\nresolve();\n","bins":[[14.115973949432373,1],[14.200000000000001,0],[14.3,1],[14.4,1],[14.500000000000002,0],[14.600000000000001,2],[14.700000000000001,1],[14.8,3],[14.9,0],[15.000000000000002,0],[15.100000000000001,2],[15.200000000000001,1],[15.3,1],[15.400000000000002,3],[15.500000000000002,6],[15.600000000000001,3],[15.700000000000001,4],[15.8,5],[15.900000000000002,8],[16,6],[16.1,8],[16.200000000000003,11],[16.3,11],[16.400000000000002,24],[16.5,27],[16.6,25],[16.700000000000003,20],[16.8,18],[16.900000000000002,21],[17,25],[17.1,25],[17.200000000000003,17],[17.3,19],[17.400000000000002,20],[17.5,20],[17.6,10],[17.700000000000003,19],[17.8,11],[17.900000000000002,25],[18,50],[18.1,67],[18.200000000000003,55],[18.3,76],[18.400000000000002,80],[18.5,69],[18.6,100],[18.700000000000003,106],[18.8,109],[18.900000000000002,107],[19,106],[19.1,75],[19.200000000000003,77],[19.3,81],[19.400000000000002,77],[19.5,67],[19.6,42],[19.700000000000003,48],[19.8,37],[19.900000000000002,32],[20,29],[20.1,23],[20.200000000000003,20],[20.3,18],[20.400000000000002,10],[20.5,6],[20.6,12],[20.700000000000003,2],[20.8,1],[20.900000000000002,3],[21,2],[21.1,1],[21.200000000000003,0],[21.3,0],[21.400000000000002,0],[21.5,1],[21.6,0],[21.700000000000003,2],[21.8,1],[21.900000000000002,0],[22,0],[22.1,1],[22.200000000000003,0],[22.3,0],[22.400000000000002,0],[22.5,0],[22.6,0],[22.700000000000003,0],[22.8,0],[22.900000000000002,0],[23,0],[23.1,0],[23.200000000000003,0],[23.3,0],[23.400000000000002,0],[23.5,0],[23.6,2],[23.700000000000003,0],[23.800000000000004,1],[23.900000000000002,0],[24,0],[24.1,0],[24.200000000000003,0],[24.300000000000004,0],[24.400000000000002,0],[24.5,0],[24.6,1]],"avg":18.627741933107256,"std":1.1101927477930364,"avg_diff":"-1.95%","std_diff":"-8.39%","old_avg":18.99126514641271,"old_std":1.2033541007277022},{"code":"\nvar view = table.view({\n filter: [[\"Origin\", \"contains\", \"P\"]],\n aggregate: \"count\"\n});\nview.to_json({\n end_row: 10\n}).then(function () {\n view.delete();\n resolve();\n});\n","bins":[[14.48419201374054,3],[14.600000000000001,1],[14.8,5],[15.000000000000002,5],[15.200000000000001,7],[15.400000000000002,4],[15.600000000000001,7],[15.8,8],[16,13],[16.200000000000003,15],[16.400000000000002,22],[16.6,34],[16.8,58],[17,42],[17.200000000000003,43],[17.400000000000002,57],[17.6,45],[17.8,42],[18,44],[18.200000000000003,77],[18.400000000000002,162],[18.6,176],[18.8,184],[19,247],[19.200000000000003,208],[19.400000000000002,160],[19.6,114],[19.8,80],[20,49],[20.200000000000003,27],[20.400000000000002,25],[20.6,14],[20.8,7],[21,2],[21.200000000000003,3],[21.400000000000002,2],[21.6,1],[21.8,0],[22,1],[22.200000000000003,0],[22.400000000000002,2],[22.6,0],[22.800000000000004,0],[23,0],[23.200000000000003,0],[23.400000000000002,0],[23.6,0],[23.800000000000004,0],[24,0],[24.200000000000003,2],[24.400000000000002,0],[24.6,0],[24.800000000000004,0],[25,1],[25.200000000000003,0],[25.400000000000002,0],[25.6,0],[25.800000000000004,0],[26,0],[26.200000000000003,0],[26.400000000000002,0],[26.6,0],[26.800000000000004,1],[27,0],[27.200000000000003,0],[27.400000000000002,0],[27.6,0],[27.800000000000004,0],[28,0],[28.200000000000003,0],[28.400000000000002,0],[28.6,0],[28.800000000000004,1]],"avg":18.74041039797141,"std":1.1377211279674802,"avg_diff":"-2.24%","std_diff":"-7.02%","old_avg":19.159274226535803,"old_std":1.2176060814371967},{"code":"\nvar view = table.view({\n filter: [[\"Origin\", \"contains\", \"P\"]],\n row_pivot: ['Dest'],\n aggregate: \"count\"\n});\nview.to_json().then(function () {\n view.delete();\n resolve();\n});\n","bins":[[41.31120491027832,1],[41.400000000000006,0],[41.60000000000001,0],[41.800000000000004,0],[42.00000000000001,1],[42.2,0],[42.400000000000006,1],[42.60000000000001,2],[42.800000000000004,2],[43.00000000000001,4],[43.2,4],[43.400000000000006,5],[43.60000000000001,4],[43.800000000000004,14],[44.00000000000001,21],[44.2,26],[44.400000000000006,18],[44.60000000000001,18],[44.800000000000004,32],[45.00000000000001,41],[45.2,45],[45.400000000000006,57],[45.60000000000001,60],[45.800000000000004,71],[46.00000000000001,93],[46.2,84],[46.400000000000006,105],[46.60000000000001,107],[46.800000000000004,126],[47.00000000000001,122],[47.2,125],[47.400000000000006,131],[47.60000000000001,128],[47.800000000000004,106],[48.00000000000001,81],[48.2,84],[48.400000000000006,55],[48.60000000000001,41],[48.800000000000004,31],[49.00000000000001,29],[49.2,19],[49.400000000000006,17],[49.60000000000001,15],[49.800000000000004,6],[50.00000000000001,8],[50.2,8],[50.400000000000006,4],[50.60000000000001,5],[50.800000000000004,6],[51.00000000000001,3],[51.2,3],[51.400000000000006,5],[51.60000000000001,1],[51.800000000000004,2],[52.00000000000001,0],[52.2,2],[52.400000000000006,0],[52.60000000000001,0],[52.800000000000004,0],[53.00000000000001,1],[53.2,0],[53.400000000000006,1],[53.60000000000001,1],[53.800000000000004,0],[54.00000000000001,0],[54.2,0],[54.400000000000006,2],[54.60000000000001,0],[54.800000000000004,1],[55.00000000000001,1],[55.2,2],[55.400000000000006,0],[55.60000000000001,0],[55.800000000000004,0],[56.00000000000001,0],[56.2,0],[56.400000000000006,0],[56.60000000000001,0],[56.800000000000004,1],[57.00000000000001,0],[57.2,1],[57.400000000000006,2],[57.60000000000001,0],[57.80000000000001,0],[58.00000000000001,0],[58.2,0],[58.400000000000006,0],[58.60000000000001,1],[58.80000000000001,0],[59.00000000000001,0],[59.2,0],[59.400000000000006,2],[59.60000000000001,1],[59.80000000000001,0],[60.00000000000001,0],[60.2,0],[60.400000000000006,0],[60.60000000000001,1],[60.80000000000001,1],[61.00000000000001,0],[61.2,0],[61.400000000000006,0],[61.60000000000001,0],[61.80000000000001,0],[62.00000000000001,0],[62.2,0],[62.400000000000006,0],[62.60000000000001,0],[62.80000000000001,1],[63.00000000000001,0],[63.2,0],[63.400000000000006,0],[63.60000000000001,1],[63.80000000000001,1]],"avg":47.136033334176815,"std":1.8227427805791003,"avg_diff":"-1.70%","std_diff":"-2.60%","old_avg":47.93699074158634,"old_std":1.87018674304559},{"code":"\nvar view = table.view({\n filter: [[\"Origin\", \"contains\", \"P\"]],\n aggregate: \"count\"\n});\nview.to_json().then(function () {\n view.delete();\n resolve();\n});\n","bins":[[74.34956908226013,1],[74.5,1],[75,3],[75.5,3],[76,10],[76.5,8],[77,17],[77.5,31],[78,31],[78.5,30],[79,30],[79.5,31],[80,37],[80.5,30],[81,29],[81.5,25],[82,27],[82.5,24],[83,22],[83.5,17],[84,9],[84.5,8],[85,9],[85.5,4],[86,3],[86.5,2],[87,1],[87.5,0],[88,0],[88.5,2],[89,0],[89.5,0],[90,0],[90.5,0],[91,0],[91.5,3],[92,10],[92.5,20],[93,20],[93.5,27],[94,27],[94.5,21],[95,9],[95.5,6],[96,3],[96.5,2],[97,1],[97.5,0],[98,1],[98.5,1],[99,0],[99.5,2],[100,0],[100.5,0],[101,1],[101.5,0],[102,0],[102.5,0],[103,1],[103.5,0],[104,0],[104.5,0],[105,0],[105.5,0],[106,0],[106.5,0],[107,0],[107.5,0],[108,0],[108.5,0],[109,0],[109.5,0],[110,0],[110.5,0],[111,0],[111.5,0],[112,0],[112.5,0],[113,0],[113.5,0],[114,0],[114.5,0],[115,0],[115.5,0],[116,0],[116.5,0],[117,0],[117.5,0],[118,0],[118.5,0],[119,0],[119.5,0],[120,1]],"avg":84.11479964014298,"std":6.566337533803833,"avg_diff":"-0.45%","std_diff":"+1.46%","old_avg":84.49366870381174,"old_std":6.470284726490922},{"code":"\nvar view = table.view({\n row_pivot: ['Dest'],\n aggregate: \"count\",\n row_pivot_depth: 1\n});\nview.to_json().then(function () {\n view.delete();\n resolve();\n});\n","bins":[[95.92658197879791,1],[96,1],[96.5,0],[97,6],[97.5,24],[98,17],[98.5,34],[99,57],[99.5,71],[100,66],[100.5,84],[101,78],[101.5,61],[102,27],[102.5,19],[103,16],[103.5,11],[104,7],[104.5,3],[105,1],[105.5,1],[106,1],[106.5,2],[107,1],[107.5,1],[108,2],[108.5,0],[109,1],[109.5,0],[110,0],[110.5,0],[111,0],[111.5,0],[112,1],[112.5,1],[113,0],[113.5,0],[114,0],[114.5,1],[115,1],[115.5,0],[116,0],[116.5,1],[117,0],[117.5,0],[118,0],[118.5,0],[119,0],[119.5,0],[120,0],[120.5,0],[121,1],[121.5,0],[122,0],[122.5,0],[123,0],[123.5,0],[124,0],[124.5,0],[125,0],[125.5,0],[126,0],[126.5,0],[127,0],[127.5,1],[128,0],[128.5,0],[129,1]],"avg":100.90800720284663,"std":2.7708893548086655,"avg_diff":"-1.95%","std_diff":"+1.47%","old_avg":102.87101471719151,"old_std":2.7302324819374606},{"code":"\nvar view = table.view({\n row_pivot: ['Dest'],\n aggregate: \"mean\",\n row_pivot_depth: 1\n});\nview.to_json().then(function () {\n view.delete();\n resolve();\n});\n","bins":[[95.03880703449249,1],[95.5,0],[96,4],[96.5,1],[97,3],[97.5,8],[98,12],[98.5,33],[99,32],[99.5,59],[100,89],[100.5,64],[101,91],[101.5,55],[102,49],[102.5,29],[103,26],[103.5,8],[104,5],[104.5,4],[105,6],[105.5,4],[106,2],[106.5,0],[107,2],[107.5,0],[108,2],[108.5,0],[109,0],[109.5,0],[110,0],[110.5,0],[111,0],[111.5,0],[112,0],[112.5,0],[113,0],[113.5,1],[114,0],[114.5,1],[115,1],[115.5,1],[116,0],[116.5,0],[117,0],[117.5,0],[118,1],[118.5,0],[119,0],[119.5,0],[120,1],[120.5,1],[121,0],[121.5,0],[122,0],[122.5,0],[123,0],[123.5,1],[124,0],[124.5,0],[125,1],[125.5,0],[126,0],[126.5,0],[127,0],[127.5,0],[128,0],[128.5,1],[129,0],[129.5,0],[130,0],[130.5,0],[131,0],[131.5,0],[132,1],[132.5,0],[133,0],[133.5,0],[134,1]],"avg":101.37813304625017,"std":3.502667380642886,"avg_diff":"-1.31%","std_diff":"+7.71%","old_avg":102.7086531924775,"old_std":3.2325664504347027}],"model":"Intel(R) Core(TM) i7-7700HQ CPU @ 2.80GHz","release":"17.5.0","platform":"darwin"} \ No newline at end of file diff --git a/packages/perspective/index.d.ts b/packages/perspective/index.d.ts index b250f14d18..1c4289bc56 100644 --- a/packages/perspective/index.d.ts +++ b/packages/perspective/index.d.ts @@ -1,3 +1,4 @@ + declare module '@jpmorganchase/perspective' { /**** object types ****/ export enum TypeNames { @@ -133,6 +134,7 @@ declare module '@jpmorganchase/perspective' { table(data: TableData, options?: TableOptions): Table; } + type perspective = { TYPE_AGGREGATES: ValuesByType, TYPE_FILTERS: ValuesByType, @@ -140,10 +142,27 @@ declare module '@jpmorganchase/perspective' { FILTER_DEFAULTS: ValueByType, SORT_ORDERS: SortOrders, table(): Table, - worker(): PerspectiveWorker + worker(): PerspectiveWorker, + override: (x: any) => void } const impl: perspective; export default impl; } + + + +declare module "@jpmorganchase/perspective/build/psp.async.wasm" { + const impl: ArrayBuffer; + export default impl; +} + +declare module "@jpmorganchase/perspective/build/psp.sync.wasm" { + const impl: ArrayBuffer; + export default impl; +} + +declare module "@jpmorganchase/perspective/build/perspective.wasm.worker.js" {} +declare module "@jpmorganchase/perspective/build/perspective.asmjs.worker.js" {} + diff --git a/packages/perspective/package.json b/packages/perspective/package.json index fd0dd2cabb..09a74d17ed 100644 --- a/packages/perspective/package.json +++ b/packages/perspective/package.json @@ -1,6 +1,6 @@ { "name": "@jpmorganchase/perspective", - "version": "0.2.7", + "version": "0.2.8", "description": "Perspective.js", "main": "build/perspective.node.js", "browser": "src/js/perspective.parallel.js", @@ -23,7 +23,7 @@ "bench:run": "node build/benchmark.js", "bench": "npm-run-all bench:build bench:run", "prebuild": "mkdir -p build && CLICOLOR_FORCE=1 npm run compile", - "precompile": "mkdir -p obj", + "precompile": "mkdir -p obj && mkdir -p ../../obj/", "compile": "cd ../../obj/ && emcmake cmake ../scripts/ && emmake make -j${PSP_CPU_COUNT-8}", "postcompile": "node ../../scripts/compile", "build": "npm-run-all build:webpack", diff --git a/packages/perspective/src/js/parse_data.js b/packages/perspective/src/js/parse_data.js new file mode 100644 index 0000000000..08beb06e30 --- /dev/null +++ b/packages/perspective/src/js/parse_data.js @@ -0,0 +1,253 @@ +/****************************************************************************** + * + * Copyright (c) 2018, the Perspective Authors. + * + * This file is part of the Perspective library, distributed under the terms of + * the Apache License 2.0. The full license can be found in the LICENSE file. + * + */ + +import {DateParser, is_valid_date} from "./date_parser.js"; + +/** + * Infer the t_dtype of a value. + * @private + * @returns A t_dtype. + */ +function infer_type(__MODULE__, x) { + let t = __MODULE__.t_dtype.DTYPE_FLOAT64; + if (x === null) { + t = null; + } else if (typeof x === "number" && x % 1 === 0 && x < 10000 && x !== 0) { + t = __MODULE__.t_dtype.DTYPE_INT32; + } else if (typeof x === "number") { + t = __MODULE__.t_dtype.DTYPE_FLOAT64; + } else if (typeof x === "boolean") { + t = __MODULE__.t_dtype.DTYPE_BOOL; + } else if (x instanceof Date) { + if (x.getHours() === 0 && x.getMinutes() === 0 && x.getSeconds() === 0 && x.getMilliseconds() === 0) { + t = __MODULE__.t_dtype.DTYPE_DATE; + } else { + t = __MODULE__.t_dtype.DTYPE_TIME; + } + } else if (!isNaN(Number(x)) && x !== "") { + t = __MODULE__.t_dtype.DTYPE_FLOAT64; + } else if (typeof x === "string" && is_valid_date(x)) { + t = __MODULE__.t_dtype.DTYPE_TIME; + } else if (typeof x === "string") { + let lower = x.toLowerCase(); + if (lower === "true" || lower === "false") { + t = __MODULE__.t_dtype.DTYPE_BOOL; + } else { + t = __MODULE__.t_dtype.DTYPE_STR; + } + } + return t; +} + +/** + * Coerce string null into value null. + * @private + * @param {*} value + */ +export function clean_data(value) { + if (value === null || value === "null") { + return null; + } else { + return value; + } +} + +/** + * Converts any supported input type into a canonical representation for + * interfacing with perspective. + * + * @private + * @param {object} data See docs + * @returns An object with 3 properties: + * names - the column names. + * types - the column t_dtypes. + * cdata - an array of columnar data. + */ +export function parse_data(__MODULE__, data, names, types) { + // todo: refactor, treat columnar/row data as the same to marshal values + fix null handling + let preloaded = types ? true : false; + if (types === undefined) { + types = []; + } else { + let _types = []; + for (let t = 0; t < types.size() - 1; t++) { + _types.push(types.get(t)); + } + types = _types; + } + let cdata = []; + + let row_count = 0; + + if (Array.isArray(data)) { + // Row oriented + if (data.length === 0) { + throw "Not yet implemented: instantiate empty grid without column type"; + } + let max_check = 50; + if (names === undefined) { + names = Object.keys(data[0]); + for (let ix = 0; ix < Math.min(max_check, data.length); ix++) { + let next = Object.keys(data[ix]); + if (names.length !== next.length) { + if (next.length > names.length) { + if (max_check === 50) console.warn("Array data has inconsistent rows"); + console.warn("Extending from " + names.length + " to " + next.length); + names = next; + max_check *= 2; + } + } + } + } + for (let n in names) { + let name = names[n]; + let i = 0, + inferredType = undefined; + // type inferrence + if (!preloaded) { + while (!inferredType && i < 100 && i < data.length) { + if (data[i].hasOwnProperty(name)) { + inferredType = infer_type(__MODULE__, data[i][name]); + } + i++; + } + inferredType = inferredType || __MODULE__.t_dtype.DTYPE_STR; + types.push(inferredType); + } else { + inferredType = types[parseInt(n)]; + } + if (inferredType === undefined) { + console.warn(`Could not infer type for column ${name}`); + inferredType = __MODULE__.t_dtype.DTYPE_STR; + } + let col = []; + const parser = new DateParser(); + // data transformation + for (let x = 0; x < data.length; x++) { + if (!(name in data[x]) || clean_data(data[x][name]) === undefined) { + col.push(undefined); + continue; + } + if (inferredType.value === __MODULE__.t_dtype.DTYPE_FLOAT64.value) { + let val = clean_data(data[x][name]); + if (val !== null) { + val = Number(val); + } + col.push(val); + } else if (inferredType.value === __MODULE__.t_dtype.DTYPE_INT32.value) { + let val = clean_data(data[x][name]); + if (val !== null) val = Number(val); + col.push(val); + if (val > 2147483647 || val < -2147483648) { + types[n] = __MODULE__.t_dtype.DTYPE_FLOAT64; + } + } else if (inferredType.value === __MODULE__.t_dtype.DTYPE_BOOL.value) { + let cell = clean_data(data[x][name]); + if (cell === null) { + col.push(null); + continue; + } + + if (typeof cell === "string") { + if (cell.toLowerCase() === "true") { + col.push(true); + } else { + col.push(false); + } + } else { + col.push(!!cell); + } + } else if (inferredType.value === __MODULE__.t_dtype.DTYPE_TIME.value || inferredType.value === __MODULE__.t_dtype.DTYPE_DATE.value) { + let val = clean_data(data[x][name]); + if (val !== null) { + col.push(parser.parse(val)); + } else { + col.push(null); + } + } else { + let val = clean_data(data[x][name]); + // types[types.length - 1].value === 19 ? "" : 0 + col.push(val === null ? null : "" + val); // TODO this is not right - might not be a string. Need a data cleaner + } + } + cdata.push(col); + row_count = col.length; + } + } else if (Array.isArray(data[Object.keys(data)[0]])) { + // Column oriented update. Extending schema not supported here. + + const names_in_update = Object.keys(data); + row_count = data[names_in_update[0]].length; + names = names || names_in_update; + + for (let col_num = 0; col_num < names.length; col_num++) { + const name = names[col_num]; + + // Infer column type if necessary + if (!preloaded) { + let i = 0; + let inferredType = null; + while (inferredType === null && i < 100 && i < data[name].length) { + inferredType = infer_type(__MODULE__, data[name][i]); + i++; + } + inferredType = inferredType || __MODULE__.t_dtype.DTYPE_STR; + types.push(inferredType); + } + + // Extract the data or fill with undefined if column doesn't exist (nothing in column changed) + let transformed; // data transformation + if (data.hasOwnProperty(name)) { + transformed = data[name].map(clean_data); + } else { + transformed = new Array(row_count); + } + cdata.push(transformed); + } + } else if (typeof data[Object.keys(data)[0]] === "string" || typeof data[Object.keys(data)[0]] === "function") { + // Arrow data/' + + //if (this.initialized) { + // throw "Cannot update already initialized table with schema."; + // } + names = []; + + // Empty type dict + for (let name in data) { + names.push(name); + if (data[name] === "integer") { + types.push(__MODULE__.t_dtype.DTYPE_INT32); + } else if (data[name] === "float") { + types.push(__MODULE__.t_dtype.DTYPE_FLOAT64); + } else if (data[name] === "string") { + types.push(__MODULE__.t_dtype.DTYPE_STR); + } else if (data[name] === "boolean") { + types.push(__MODULE__.t_dtype.DTYPE_BOOL); + } else if (data[name] === "datetime") { + types.push(__MODULE__.t_dtype.DTYPE_TIME); + } else if (data[name] === "date") { + types.push(__MODULE__.t_dtype.DTYPE_DATE); + } else { + throw `Unknown type ${data[name]}`; + } + cdata.push([]); + } + } else { + throw "Unknown data type"; + } + + // separate methods to return each property + return { + row_count: row_count, + is_arrow: false, + names: names, + types: types, + cdata: cdata + }; +} diff --git a/packages/perspective/src/js/perspective.asmjs.js b/packages/perspective/src/js/perspective.asmjs.js index 7dc8818c0a..0777a96832 100644 --- a/packages/perspective/src/js/perspective.asmjs.js +++ b/packages/perspective/src/js/perspective.asmjs.js @@ -14,7 +14,6 @@ const perspective = require("./perspective.js").default; const Module = load_perspective({ wasmJSMethod: "asmjs", - filePackagePrefixURL: "", printErr: x => console.error(x), print: x => console.log(x) }); diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index 9c18cd23e2..f61b23296b 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -8,7 +8,8 @@ */ import * as defaults from "./defaults.js"; -import {DateParser, is_valid_date} from "./date_parser.js"; +import {parse_data, clean_data} from "./parse_data.js"; +import {DateParser} from "./date_parser.js"; import {bindall} from "./utils.js"; import {Precision} from "@apache-arrow/es5-esm/type"; @@ -37,42 +38,6 @@ export default function(Module) { * */ - /** - * Infer the t_dtype of a value. - * @private - * @returns A t_dtype. - */ - function infer_type(x) { - let t = __MODULE__.t_dtype.DTYPE_FLOAT64; - if (x === null) { - t = null; - } else if (typeof x === "number" && x % 1 === 0 && x < 10000 && x !== 0) { - t = __MODULE__.t_dtype.DTYPE_INT32; - } else if (typeof x === "number") { - t = __MODULE__.t_dtype.DTYPE_FLOAT64; - } else if (typeof x === "boolean") { - t = __MODULE__.t_dtype.DTYPE_BOOL; - } else if (x instanceof Date) { - if (x.getHours() === 0 && x.getMinutes() === 0 && x.getSeconds() === 0 && x.getMilliseconds() === 0) { - t = __MODULE__.t_dtype.DTYPE_DATE; - } else { - t = __MODULE__.t_dtype.DTYPE_TIME; - } - } else if (!isNaN(Number(x)) && x !== "") { - t = __MODULE__.t_dtype.DTYPE_FLOAT64; - } else if (typeof x === "string" && is_valid_date(x)) { - t = __MODULE__.t_dtype.DTYPE_TIME; - } else if (typeof x === "string") { - let lower = x.toLowerCase(); - if (lower === "true" || lower === "false") { - t = __MODULE__.t_dtype.DTYPE_BOOL; - } else { - t = __MODULE__.t_dtype.DTYPE_STR; - } - } - return t; - } - /** * Gets human-readable types for a column * @private @@ -95,205 +60,39 @@ export default function(Module) { } /** - * Coerce string null into value null + * Determines a table's limit index. * @private - * @param {*} value + * @param {int} limit_index + * @param {int} new_length + * @param {int} options_limit */ - function clean_data(value) { - if (value === null || value === "null") { - return null; - } else { - return value; + function calc_limit_index(limit_index, new_length, options_limit) { + limit_index += new_length; + if (options_limit) { + limit_index = limit_index % options_limit; } + return limit_index; } /** - * Converts any supported input type into a canonical representation for - * interfacing with perspective. - * - * @private - * @param {object} data See docs - * @returns An object with 3 properties: - * names - the column names. - * types - the column t_dtypes. - * cdata - an array of columnar data. + * Common logic for creating and registering a gnode/t_table. + * + * @param {*} pdata + * @param {*} pool + * @param {*} gnode + * @param {*} computed + * @param {*} index + * @param {*} limit + * @param {*} limit_index + * @param {*} is_delete + * @returns */ - function parse_data(data, names, types) { - // todo: refactor, treat columnar/row data as the same to marshal values + fix null handling - let preloaded = types ? true : false; - if (types === undefined) { - types = []; - } else { - let _types = []; - for (let t = 0; t < types.size() - 1; t++) { - _types.push(types.get(t)); - } - types = _types; - } - let cdata = []; - - let row_count = 0; - - if (Array.isArray(data)) { - // Row oriented - if (data.length === 0) { - throw "Not yet implemented: instantiate empty grid without column type"; - } - let max_check = 50; - if (names === undefined) { - names = Object.keys(data[0]); - for (let ix = 0; ix < Math.min(max_check, data.length); ix++) { - let next = Object.keys(data[ix]); - if (names.length !== next.length) { - if (next.length > names.length) { - if (max_check === 50) console.warn("Array data has inconsistent rows"); - console.warn("Extending from " + names.length + " to " + next.length); - names = next; - max_check *= 2; - } - } - } - } - for (let n in names) { - let name = names[n]; - let i = 0, - inferredType = undefined; - if (!preloaded) { - while (!inferredType && i < 100 && i < data.length) { - if (data[i].hasOwnProperty(name)) { - inferredType = infer_type(data[i][name]); - } - i++; - } - inferredType = inferredType || __MODULE__.t_dtype.DTYPE_STR; - types.push(inferredType); - } else { - inferredType = types[parseInt(n)]; - } - if (inferredType === undefined) { - console.warn(`Could not infer type for column ${name}`); - inferredType = __MODULE__.t_dtype.DTYPE_STR; - } - let col = []; - const parser = new DateParser(); - for (let x = 0; x < data.length; x++) { - if (!(name in data[x]) || clean_data(data[x][name]) === undefined) { - col.push(undefined); - continue; - } - if (inferredType.value === __MODULE__.t_dtype.DTYPE_FLOAT64.value) { - let val = clean_data(data[x][name]); - if (val !== null) { - val = Number(val); - } - col.push(val); - } else if (inferredType.value === __MODULE__.t_dtype.DTYPE_INT32.value) { - let val = clean_data(data[x][name]); - if (val !== null) val = Number(val); - col.push(val); - if (val > 2147483647 || val < -2147483648) { - types[n] = __MODULE__.t_dtype.DTYPE_FLOAT64; - } - } else if (inferredType.value === __MODULE__.t_dtype.DTYPE_BOOL.value) { - let cell = clean_data(data[x][name]); - if (cell === null) { - col.push(null); - continue; - } - - if (typeof cell === "string") { - if (cell.toLowerCase() === "true") { - col.push(true); - } else { - col.push(false); - } - } else { - col.push(!!cell); - } - } else if (inferredType.value === __MODULE__.t_dtype.DTYPE_TIME.value || inferredType.value === __MODULE__.t_dtype.DTYPE_DATE.value) { - let val = clean_data(data[x][name]); - if (val !== null) { - col.push(parser.parse(val)); - } else { - col.push(null); - } - } else { - let val = clean_data(data[x][name]); - // types[types.length - 1].value === 19 ? "" : 0 - col.push(val === null ? null : "" + val); // TODO this is not right - might not be a string. Need a data cleaner - } - } - cdata.push(col); - row_count = col.length; - } - } else if (Array.isArray(data[Object.keys(data)[0]])) { - // Column oriented update. Extending schema not supported here. - - const names_in_update = Object.keys(data); - row_count = data[names_in_update[0]].length; - names = names || names_in_update; - - for (let col_num = 0; col_num < names.length; col_num++) { - const name = names[col_num]; - - // Infer column type if necessary - if (!preloaded) { - let i = 0; - let inferredType = null; - while (inferredType === null && i < 100 && i < data[name].length) { - inferredType = infer_type(data[name][i]); - i++; - } - inferredType = inferredType || __MODULE__.t_dtype.DTYPE_STR; - types.push(inferredType); - } - - // Extract the data or fill with undefined if column doesn't exist (nothing in column changed) - let transformed; - if (data.hasOwnProperty(name)) { - transformed = data[name].map(clean_data); - } else { - transformed = new Array(row_count); - } - cdata.push(transformed); - } - } else if (typeof data[Object.keys(data)[0]] === "string" || typeof data[Object.keys(data)[0]] === "function") { - //if (this.initialized) { - // throw "Cannot update already initialized table with schema."; - // } - names = []; - - // Empty type dict - for (let name in data) { - names.push(name); - if (data[name] === "integer") { - types.push(__MODULE__.t_dtype.DTYPE_INT32); - } else if (data[name] === "float") { - types.push(__MODULE__.t_dtype.DTYPE_FLOAT64); - } else if (data[name] === "string") { - types.push(__MODULE__.t_dtype.DTYPE_STR); - } else if (data[name] === "boolean") { - types.push(__MODULE__.t_dtype.DTYPE_BOOL); - } else if (data[name] === "datetime") { - types.push(__MODULE__.t_dtype.DTYPE_TIME); - } else if (data[name] === "date") { - types.push(__MODULE__.t_dtype.DTYPE_DATE); - } else { - throw `Unknown type ${data[name]}`; - } - cdata.push([]); - } - } else { - throw "Unknown data type"; + function make_table(pdata, pool, gnode, computed, index, limit, limit_index, is_delete) { + for (let chunk of pdata) { + gnode = __MODULE__.make_table(pool, gnode, chunk, computed, limit_index, limit || 4294967295, index, is_delete); + limit_index = calc_limit_index(limit_index, chunk.cdata[0].length, limit); } - - return { - row_count: row_count, - is_arrow: false, - names: names, - types: types, - cdata: cdata - }; + return [gnode, limit_index]; } /** @@ -482,7 +281,7 @@ export default function(Module) { return this.nsides; }; - view.prototype._column_names = function() { + view.prototype._column_names = function(skip_depth = false) { let col_names = []; let aggs = this.ctx.get_column_names(); for (let key = 0; key < this.ctx.unity_get_column_count(); key++) { @@ -498,6 +297,10 @@ export default function(Module) { continue; } let col_path = this.ctx.unity_get_column_path(key + 1); + if (skip_depth && col_path.size() < skip_depth) { + col_path.delete(); + continue; + } col_name = []; for (let cnix = 0; cnix < col_path.size(); cnix++) { col_name.push(__MODULE__.scalar_vec_to_val(col_path, cnix)); @@ -587,8 +390,9 @@ export default function(Module) { let start_row = options.start_row || (viewport.top ? viewport.top : 0); let end_row = options.end_row || (viewport.height ? start_row + viewport.height : this.ctx.get_row_count()); let start_col = options.start_col || (viewport.left ? viewport.left : 0); - let end_col = options.end_col || (viewport.width ? start_row + viewport.width : this.ctx.unity_get_column_count() + (this.sides() === 0 ? 0 : 1)); + let end_col = options.end_col || (viewport.width ? start_col + viewport.width : this.ctx.unity_get_column_count() + (this.sides() === 0 ? 0 : 1)); let slice; + const sorted = typeof this.config.sort !== "undefined" && this.config.sort.length > 0; if (this.config.row_pivot[0] === "psp_okey") { end_row += this.config.column_pivot.length; } @@ -596,17 +400,19 @@ export default function(Module) { slice = __MODULE__.get_data_zero(this.ctx, start_row, end_row, start_col, end_col); } else if (this.sides() === 1) { slice = __MODULE__.get_data_one(this.ctx, start_row, end_row, start_col, end_col); - } else { + } else if (!sorted) { slice = __MODULE__.get_data_two(this.ctx, start_row, end_row, start_col, end_col); + } else { + slice = __MODULE__.get_data_two_skip_headers(this.ctx, this.config.column_pivot.length, start_row, end_row, start_col, end_col); } let data = formatter.initDataValue(); - let col_names = [[]].concat(this._column_names()); + let col_names = [[]].concat(this._column_names(this.sides() === 2 && sorted ? this.config.column_pivot.length : false)); let row; let ridx = -1; for (let idx = 0; idx < slice.length; idx++) { - let cidx = idx % (end_col - start_col); + let cidx = idx % Math.min(end_col - start_col, col_names.slice(start_col, end_col - start_col + 1).length); if (cidx === 0) { if (row) { formatter.addRow(data, row); @@ -945,43 +751,6 @@ export default function(Module) { } }; - table.prototype._calculate_computed = function(tbl, computed_defs) { - // tbl is the pointer to the C++ t_table - - for (let i = 0; i < computed_defs.length; ++i) { - let coldef = computed_defs[i]; - let name = coldef["column"]; - let func = coldef["func"]; - let inputs = coldef["inputs"]; - let type = coldef["type"] || "string"; - - let dtype; - switch (type) { - case "integer": - dtype = __MODULE__.t_dtype.DTYPE_INT32; - break; - case "float": - dtype = __MODULE__.t_dtype.DTYPE_FLOAT64; - break; - case "boolean": - dtype = __MODULE__.t_dtype.DTYPE_BOOL; - break; - case "date": - dtype = __MODULE__.t_dtype.DTYPE_DATE; - break; - case "datetime": - dtype = __MODULE__.t_dtype.DTYPE_TIME; - break; - case "string": - default: - dtype = __MODULE__.t_dtype.DTYPE_STR; - break; - } - - __MODULE__.table_add_computed_column(tbl, name, dtype, func, inputs); - } - }; - /** * Delete this {@link table} and clean up all resources associated with it. * Table objects do not stop consuming resources or processing updates when @@ -1024,16 +793,18 @@ export default function(Module) { return this.gnode.get_table().size(); }; - table.prototype._schema = function() { + table.prototype._schema = function(computed) { let schema = this.gnode.get_tblschema(); let columns = schema.columns(); let types = schema.types(); let new_schema = {}; + const computed_schema = this.computed_schema(); for (let key = 0; key < columns.size(); key++) { - if (columns.get(key) === "psp_okey") { + const name = columns.get(key); + if (name === "psp_okey" && (typeof computed_schema[name] === "undefined" || computed)) { continue; } - new_schema[columns.get(key)] = get_column_type(types.get(key).value); + new_schema[name] = get_column_type(types.get(key).value); } schema.delete(); columns.delete(); @@ -1046,41 +817,33 @@ export default function(Module) { * columns of this {@link table}, and whose values are their string type names. * * @async - * + * @param {boolean} computed Should computed columns be included? + * (default false) * @returns {Promise} A Promise of this {@link table}'s schema. */ - table.prototype.schema = async function() { - return this._schema(); + table.prototype.schema = async function(computed = false) { + return this._schema(computed); }; table.prototype._computed_schema = function() { - let computed = this.computed; - - if (computed.length < 0) return {}; + if (this.computed.length < 0) return {}; - let schema = this.gnode.get_tblschema(); - let columns = schema.columns(); - let types = schema.types(); + const computed_schema = {}; - let computed_schema = {}; - - for (let i = 0; i < computed.length; i++) { - const column_name = computed[i].column; - const column_type = computed[i].type; + for (let i = 0; i < this.computed.length; i++) { + const column_name = this.computed[i].column; + const column_type = this.computed[i].type; const column = {}; column.type = column_type; - column.input_columns = computed[i].inputs; - column.input_type = computed[i].input_type; - column.computation = computed[i].computation; + column.input_columns = this.computed[i].inputs; + column.input_type = this.computed[i].input_type; + column.computation = this.computed[i].computation; computed_schema[column_name] = column; } - schema.delete(); - columns.delete(); - types.delete(); return computed_schema; }; @@ -1211,21 +974,6 @@ export default function(Module) { } } - // Sort - let sort = []; - if (config.sort) { - sort = config.sort.map(x => { - if (!Array.isArray(x)) { - return [config.aggregate.map(agg => agg.column).indexOf(x), 1]; - } else { - return [config.aggregate.map(agg => agg.column).indexOf(x[0]), defaults.SORT_ORDERS.indexOf(x[1])]; - } - }); - if (config.column_pivot.length > 0 && config.row_pivot.length > 0) { - config.sort = config.sort.filter(x => config.row_pivot.indexOf(x[0]) === -1); - } - } - let schema = this.gnode.get_tblschema(); // Row Pivots @@ -1249,13 +997,14 @@ export default function(Module) { aggregates.push([agg.name || agg.column.join(defaults.COLUMN_SEPARATOR_STRING), agg_op, agg.column]); } } else { - let agg_op = __MODULE__.t_aggtype.AGGTYPE_DISTINCT_COUNT; - if (config.column_only) { - agg_op = __MODULE__.t_aggtype.AGGTYPE_ANY; - } let t_aggs = schema.columns(); + let t_aggtypes = schema.types(); for (let aidx = 0; aidx < t_aggs.size(); aidx++) { let column = t_aggs.get(aidx); + let agg_op = __MODULE__.t_aggtype.AGGTYPE_ANY; + if (!config.column_only) { + agg_op = _string_to_aggtype[defaults.AGGREGATE_DEFAULTS[get_column_type(t_aggtypes.get(aidx).value)]]; + } if (column !== "psp_okey") { aggregates.push([column, agg_op, [column]]); } @@ -1263,12 +1012,36 @@ export default function(Module) { t_aggs.delete(); } + // Sort + let sort = [], + col_sort = []; + if (config.sort) { + sort = config.sort + .filter(x => config.column_pivot.indexOf(x[0]) === -1) + .map(x => { + if (!Array.isArray(x)) { + return [aggregates.map(agg => agg[0]).indexOf(x), 1]; + } else { + return [aggregates.map(agg => agg[0]).indexOf(x[0]), defaults.SORT_ORDERS.indexOf(x[1])]; + } + }); + col_sort = config.sort + .filter(x => config.column_pivot.indexOf(x[0]) > -1) + .map(x => { + if (!Array.isArray(x)) { + return [aggregates.map(agg => agg[0]).indexOf(x), 1]; + } else { + return [aggregates.map(agg => agg[0]).indexOf(x[0]), defaults.SORT_ORDERS.indexOf(x[1])]; + } + }); + } + let context; let sides = 0; if (config.row_pivot.length > 0 || config.column_pivot.length > 0) { if (config.column_pivot && config.column_pivot.length > 0) { config.row_pivot = config.row_pivot || []; - context = __MODULE__.make_context_two(schema, config.row_pivot, config.column_pivot, filter_op, filters, aggregates, []); + context = __MODULE__.make_context_two(schema, config.row_pivot, config.column_pivot, filter_op, filters, aggregates, sort.length > 0); sides = 2; this.pool.register_context(this.gnode.get_id(), name, __MODULE__.t_ctx_type.TWO_SIDED_CONTEXT, context.$$.ptr); @@ -1284,17 +1057,8 @@ export default function(Module) { context.set_depth(__MODULE__.t_header.HEADER_COLUMN, config.column_pivot.length); } - const groups = context.unity_get_column_count() / aggregates.length; - const new_sort = []; - - for (let z = 0; z < groups; z++) { - for (let s of sort) { - new_sort.push([s[0] + z * aggregates.length, s[1]]); - } - } - - if (sort.length > 0) { - __MODULE__.sort(context, new_sort); + if (sort.length > 0 || col_sort.length > 0) { + __MODULE__.sort(context, sort, col_sort); } } else { context = __MODULE__.make_context_one(schema, config.row_pivot, filter_op, filters, aggregates, sort); @@ -1354,9 +1118,9 @@ export default function(Module) { if (data[0] === ",") { data = "_" + data; } - pdata = [parse_data(papaparse.parse(data.trim(), {dynamicTyping: true, header: true}).data, cols, types)]; + pdata = [parse_data(__MODULE__, papaparse.parse(data.trim(), {dynamicTyping: true, header: true}).data, cols, types)]; } else { - pdata = [parse_data(data, cols, types)]; + pdata = [parse_data(__MODULE__, data, cols, types)]; } for (let i = names.size() - 1; i >= 0; i--) { @@ -1367,29 +1131,12 @@ export default function(Module) { } } - let tbl; try { - for (let chunk of pdata) { - tbl = __MODULE__.make_table(chunk.row_count || 0, chunk.names, chunk.types, chunk.cdata, this.limit_index, this.limit || 4294967295, this.index || "", chunk.is_arrow, false); - - this.limit_index += chunk.cdata[0].length; - if (this.limit) { - this.limit_index = this.limit_index % this.limit; - } - - // Add any computed columns - this._calculate_computed(tbl, this.computed); - - this.pool.send(this.gnode.get_id(), 0, tbl); - this.pool.process(); - this.initialized = true; - } + [, this.limit_index] = make_table(pdata, this.pool, this.gnode, this.computed, this.index || "", this.limit, this.limit_index, false); + this.initialized = true; } catch (e) { console.error(e); } finally { - if (tbl) { - tbl.delete(); - } schema.delete(); names.delete(); types.delete(); @@ -1408,62 +1155,37 @@ export default function(Module) { let pdata; let schema = this.gnode.get_tblschema(); let types = schema.types(); - schema.delete(); data = data.map(idx => ({[this.index]: idx})); if (data instanceof ArrayBuffer) { pdata = load_arrow_buffer(data, [this.index], types); } else { - pdata = [parse_data(data, [this.index], types)]; + pdata = [parse_data(__MODULE__, data, [this.index], types)]; } - let tbl; try { - for (let chunk of pdata) { - tbl = __MODULE__.make_table(chunk.row_count || 0, chunk.names, chunk.types, chunk.cdata, this.limit_index, this.limit || 4294967295, this.index || "", chunk.is_arrow, true); - - this.limit_index += chunk.cdata[0].length; - if (this.limit) { - this.limit_index = this.limit_index % this.limit; - } - - this.pool.send(this.gnode.get_id(), 0, tbl); - this.pool.process(); - this.initialized = true; - } + [, this.limit_index] = make_table(pdata, this.pool, this.gnode, undefined, this.index || "", this.limit, this.limit_index, true); + this.initialized = true; } catch (e) { console.error(e); } finally { - if (tbl) { - tbl.delete(); - } types.delete(); + schema.delete(); } }; /** * Create a new table with the addition of new computed columns (defined as javascript functions) + * + * @param {Computation} computed A computation specification object */ table.prototype.add_computed = function(computed) { let pool, gnode, tbl; try { - // Create perspective pool - pool = new __MODULE__.t_pool({_update_callback: function() {}}); - - // Pull out the t_table from the current gnode - tbl = __MODULE__.clone_gnode_table(this.gnode); - - // Add new computed columns in place to tbl - this._calculate_computed(tbl, computed); - - gnode = __MODULE__.make_gnode(tbl); - pool.register_gnode(gnode); - pool.send(gnode.get_id(), 0, tbl); - pool.process(); - - // Merge in definition of previous computed columns + pool = new __MODULE__.t_pool(); + gnode = __MODULE__.clone_gnode_table(pool, this.gnode, computed); if (this.computed.length > 0) { computed = this.computed.concat(computed); } @@ -1484,14 +1206,14 @@ export default function(Module) { } }; - table.prototype._columns = function() { + table.prototype._columns = function(computed = false) { let schema = this.gnode.get_tblschema(); let computed_schema = this._computed_schema(); let cols = schema.columns(); let names = []; for (let cidx = 0; cidx < cols.size(); cidx++) { let name = cols.get(cidx); - if (name !== "psp_okey" && typeof computed_schema[name] === "undefined") { + if (name !== "psp_okey" && (typeof computed_schema[name] === "undefined" || computed)) { names.push(name); } } @@ -1504,11 +1226,12 @@ export default function(Module) { * The column names of this table. * * @async - * + * @param {boolean} computed Should computed columns be included? + * (default false) * @returns {Array} An array of column names for this table. */ - table.prototype.columns = async function() { - return this._columns(); + table.prototype.columns = async function(computed = false) { + return this._columns(computed); }; table.prototype._column_metadata = function() { @@ -1836,7 +1559,7 @@ export default function(Module) { } data = papaparse.parse(data.trim(), {dynamicTyping: true, header: true}).data; } - pdata = parse_data(data); + pdata = parse_data(__MODULE__, data); if (pdata.row_count > CHUNKED_THRESHOLD) { let new_pdata = []; while (pdata.cdata[0].length > 0) { @@ -1857,27 +1580,13 @@ export default function(Module) { throw `Specified index '${options.index}' does not exist in data.`; } - let tbl, - gnode, + let gnode, pool, limit_index = 0; try { - pool = new __MODULE__.t_pool({_update_callback: function() {}}); - for (let chunk of pdata) { - tbl = __MODULE__.make_table(chunk.cdata[0].length || 0, chunk.names, chunk.types, chunk.cdata, limit_index, options.limit || 4294967295, options.index, chunk.is_arrow, false); - limit_index += chunk.cdata[0].length; - if (options.limit) { - limit_index = limit_index % options.limit; - } - if (!gnode) { - gnode = __MODULE__.make_gnode(tbl); - pool.register_gnode(gnode); - } - pool.send(gnode.get_id(), 0, tbl); - pool.process(); - } - + pool = new __MODULE__.t_pool(); + [gnode, limit_index] = make_table(pdata, pool, gnode, undefined, options.index, options.limit, limit_index, false); return new table(gnode, pool, options.index, undefined, options.limit, limit_index); } catch (e) { if (pool) { @@ -1887,10 +1596,6 @@ export default function(Module) { gnode.delete(); } throw e; - } finally { - if (tbl) { - tbl.delete(); - } } } }; diff --git a/packages/perspective/src/js/perspective.node.js b/packages/perspective/src/js/perspective.node.js index 538643de6e..b5879d7c78 100644 --- a/packages/perspective/src/js/perspective.node.js +++ b/packages/perspective/src/js/perspective.node.js @@ -30,8 +30,7 @@ const buffer = fs.readFileSync(path.join(__dirname, wasm)).buffer; module.exports = perspective( load_perspective({ wasmBinary: buffer, - wasmJSMethod: "native-wasm", - ENVIRONMENT: "NODE" + wasmJSMethod: "native-wasm" }) ); diff --git a/packages/perspective/src/js/perspective.parallel.js b/packages/perspective/src/js/perspective.parallel.js index df74e8e28d..141b71896e 100644 --- a/packages/perspective/src/js/perspective.parallel.js +++ b/packages/perspective/src/js/perspective.parallel.js @@ -27,17 +27,38 @@ function detect_iphone() { return /iPad|iPhone|iPod/.test(navigator.userAgent) && !window.MSStream; } -function fetch(url) { - return new Promise(resolve => { - let wasmXHR = new XMLHttpRequest(); - wasmXHR.open("GET", url, true); - wasmXHR.responseType = "arraybuffer"; - wasmXHR.onload = () => { - resolve(wasmXHR.response); - }; - wasmXHR.send(null); - }); -} +/** + * Singleton WASM file download cache. + */ +const override = new class { + _fetch(url) { + return new Promise(resolve => { + let wasmXHR = new XMLHttpRequest(); + wasmXHR.open("GET", url, true); + wasmXHR.responseType = "arraybuffer"; + wasmXHR.onload = () => { + resolve(wasmXHR.response); + }; + wasmXHR.send(null); + }); + } + + set({wasm, worker}) { + this._wasm = wasm || this._wasm; + this._worker = worker || this._worker; + } + + worker() { + return (this._worker || wasm_worker)(); + } + + async wasm() { + if (!this._wasm) { + this._wasm = await this._fetch(wasm); + } + return this._wasm; + } +}(); class WebWorker extends worker { constructor() { @@ -51,7 +72,7 @@ class WebWorker extends worker { if (typeof WebAssembly === "undefined" || detect_iphone()) { worker = await asmjs_worker(); } else { - [worker, msg.buffer] = await Promise.all([wasm_worker(), fetch(wasm)]); + [worker, msg.buffer] = await Promise.all([override.worker(), override.wasm()]); } for (var key in this._worker) { worker[key] = this._worker[key]; @@ -109,6 +130,8 @@ class WebSocketWorker extends worker { } const mod = { + override: x => override.set(x), + worker(url) { if (url) { return new WebSocketWorker(url); diff --git a/packages/perspective/src/loader/blob_worker_loader.js b/packages/perspective/src/loader/blob_worker_loader.js index 01a9c84c7c..07c194d229 100644 --- a/packages/perspective/src/loader/blob_worker_loader.js +++ b/packages/perspective/src/loader/blob_worker_loader.js @@ -14,8 +14,6 @@ const NodeTargetPlugin = require("webpack/lib/node/NodeTargetPlugin"); const SingleEntryPlugin = require("webpack/lib/SingleEntryPlugin"); const WebWorkerTemplatePlugin = require("webpack/lib/webworker/WebWorkerTemplatePlugin"); -const path = require("path"); - class BlobWorkerLoaderError extends Error { constructor(err) { super(err); @@ -84,7 +82,6 @@ exports.pitch = function pitch(request) { if (!compilation.cache[subCache]) { compilation.cache[subCache] = {}; } - compilation.cache = compilation.cache[subCache]; } }; @@ -101,21 +98,7 @@ exports.pitch = function pitch(request) { if (entries[0]) { worker.file = entries[0].files[0]; - - const utils_path = JSON.stringify(`!!${path.join(__dirname, "utils.js")}`); - - return cb( - null, - `module.exports = function() { - var utils = require(${utils_path}); - - if (window.location.origin === utils.host.slice(0, window.location.origin.length)) { - return new Promise(function(resolve) { resolve(new Worker(utils.path + __webpack_public_path__ + ${JSON.stringify(worker.file)})); }); - } else { - return new Promise(function(resolve) { new utils.XHRWorker(utils.path + __webpack_public_path__ + ${JSON.stringify(worker.file)}, resolve); }); - } - };` - ); + return cb(null, this._compilation.assets[worker.file].children[0]._value); } return cb(null, null); diff --git a/packages/perspective/src/loader/file_worker_loader.js b/packages/perspective/src/loader/file_worker_loader.js index 8fc34ab758..c379c83683 100644 --- a/packages/perspective/src/loader/file_worker_loader.js +++ b/packages/perspective/src/loader/file_worker_loader.js @@ -7,18 +7,24 @@ * */ -var path = require("path"); +const path = require("path"); -var loaderUtils = require("loader-utils"); -var validateOptions = require("schema-utils"); +const loaderUtils = require("loader-utils"); +const validateOptions = require("schema-utils"); -var fs = require("fs"); +const fs = require("fs"); -var schema = { +const schema = { type: "object", properties: { name: {}, regExp: {}, + compiled: { + type: "boolean" + }, + inline: { + type: "boolean" + }, context: { type: "string" } @@ -27,41 +33,55 @@ var schema = { }; exports.default = function loader(content) { - var options = loaderUtils.getOptions(this) || {}; - + const options = loaderUtils.getOptions(this) || {}; validateOptions(schema, options, "File Worker Loader"); - - var context = options.context || this.rootContext || (this.options && this.options.context); - - var url = loaderUtils.interpolateName(this, options.name, { + const context = options.context || this.rootContext || (this.options && this.options.context); + const url = loaderUtils.interpolateName(this, options.name, { context, content, regExp: options.regExp }); + const outputPath = url.replace(/\.js/, ".worker.js"); - var outputPath = url.replace(/\.js/, ".worker.js"); - var inputPath = this.resourcePath - .replace(/\.js/, ".worker.js") - .replace(/\/build/, "") - .replace(/(src\/js|es5)/, "build"); - - var new_content = fs.readFileSync(inputPath); + if (!options.compiled) { + var inputPath = this.resourcePath; + if (!options.inline) { + inputPath = inputPath + .replace("build", "") + .replace(/\.js/, ".worker.js") + .replace(/(src\/js)/, "build"); + } + content = fs.readFileSync(inputPath).toString(); + if (!options.compiled) { + this.emitFile(outputPath, "" + content); + const map_file = `${inputPath}.map`; + if (fs.existsSync(map_file)) { + const map_content = fs.readFileSync(map_file).toString(); + this.emitFile(`${outputPath}.map`, "" + map_content); + } + } + } - var publicPath = `__webpack_public_path__ + ${JSON.stringify(outputPath)}`; + const publicPath = `__webpack_public_path__ + ${JSON.stringify(outputPath)}`; + const utils_path = JSON.stringify(`!!${path.join(__dirname, "utils.js")}`); - this.emitFile(outputPath, new_content); + if (options.inline) { + const worker_text = JSON.stringify(content.toString()) + .replace(/\u2028/g, "\\u2028") + .replace(/\u2029/g, "\\u2029"); - const utils_path = JSON.stringify(`!!${path.join(__dirname, "utils.js")}`); + return `module.exports = function() { + var utils = require(${utils_path}); + return new Promise(function(resolve) { utils.BlobWorker(${worker_text}, resolve); }); + };`; + } return `module.exports = function() { var utils = require(${utils_path}); - if (window.location.origin === utils.host.slice(0, window.location.origin.length)) { return new Promise(function(resolve) { resolve(new Worker(utils.path + ${publicPath})); }); } else { - return new Promise(function(resolve) { new utils.XHRWorker(utils.path + ${publicPath}, resolve); }); + return new Promise(function(resolve) { utils.XHRWorker(utils.path + ${publicPath}, resolve); }); } };`; }; - -exports.raw = true; diff --git a/packages/perspective/src/loader/utils.js b/packages/perspective/src/loader/utils.js index d78a7467f8..2574a5734c 100644 --- a/packages/perspective/src/loader/utils.js +++ b/packages/perspective/src/loader/utils.js @@ -55,17 +55,21 @@ module.exports.host = __SCRIPT_PATH__.host(); module.exports.path = __SCRIPT_PATH__.path(); +module.exports.BlobWorker = function(responseText, ready) { + var blob = new Blob([responseText]); + var obj = window.URL.createObjectURL(blob); + var worker = new Worker(obj); + if (ready) { + ready(worker); + } +}; + module.exports.XHRWorker = function XHRWorker(url, ready) { var oReq = new XMLHttpRequest(); oReq.addEventListener( "load", function() { - var blob = new Blob([this.responseText]); - var obj = window.URL.createObjectURL(blob); - var worker = new Worker(obj); - if (ready) { - ready(worker); - } + module.exports.BlobWorker(oReq.responseText, ready); }, oReq ); diff --git a/packages/perspective/test/js/constructors.js b/packages/perspective/test/js/constructors.js index e119b5e50f..eb2d03ed71 100644 --- a/packages/perspective/test/js/constructors.js +++ b/packages/perspective/test/js/constructors.js @@ -159,8 +159,11 @@ module.exports = perspective => { table.execute(t => { t.update([{x: 1, y: "a", z: true}, {x: 2, y: "b", z: false}, {x: 3, y: "c", z: true}, {x: 4, y: "d", z: false}]); }); - let js = await table.view({}).to_json(); + let view = table.view({}); + let js = await view.to_json(); expect(js).toEqual([{x: 1, y: "a", z: true}, {x: 2, y: "b", z: false}, {x: 3, y: "c", z: true}, {x: 4, y: "d", z: false}]); + view.delete(); + table.delete(); }); }); @@ -196,6 +199,8 @@ module.exports = perspective => { var view = table.view(); const result = await view.col_to_js_typed_array("int"); expect(result.byteLength).toEqual(16); + view.delete(); + table.delete(); }); it("Float, 0-sided view", async function() { @@ -203,6 +208,8 @@ module.exports = perspective => { var view = table.view(); const result = await view.col_to_js_typed_array("float"); expect(result.byteLength).toEqual(32); + view.delete(); + table.delete(); }); it("Datetime, 0-sided view", async function() { @@ -210,6 +217,8 @@ module.exports = perspective => { var view = table.view(); const result = await view.col_to_js_typed_array("datetime"); expect(result.byteLength).toEqual(32); + view.delete(); + table.delete(); }); it("Int, 1-sided view", async function() { @@ -221,6 +230,8 @@ module.exports = perspective => { const result = await view.col_to_js_typed_array("int"); // should include aggregate row expect(result.byteLength).toEqual(20); + view.delete(); + table.delete(); }); it("Float, 1-sided view", async function() { @@ -231,6 +242,8 @@ module.exports = perspective => { }); const result = await view.col_to_js_typed_array("float"); expect(result.byteLength).toEqual(40); + view.delete(); + table.delete(); }); it("Datetime, 1-sided view", async function() { @@ -241,6 +254,8 @@ module.exports = perspective => { }); const result = await view.col_to_js_typed_array("datetime"); expect(result.byteLength).toEqual(24); + view.delete(); + table.delete(); }); it("Int, 2-sided view with row pivot", async function() { @@ -252,6 +267,8 @@ module.exports = perspective => { }); const result = await view.col_to_js_typed_array("3.5|int"); expect(result.byteLength).toEqual(20); + view.delete(); + table.delete(); }); it("Float, 2-sided view with row pivot", async function() { @@ -263,6 +280,8 @@ module.exports = perspective => { }); const result = await view.col_to_js_typed_array("3.5|float"); expect(result.byteLength).toEqual(40); + view.delete(); + table.delete(); }); it("Int, 2-sided view, no row pivot", async function() { @@ -271,6 +290,8 @@ module.exports = perspective => { const result = await view.col_to_js_typed_array("3.5|int"); // bytelength should not include the aggregate row expect(result.byteLength).toEqual(16); + view.delete(); + table.delete(); }); it("Float, 2-sided view, no row pivot", async function() { @@ -278,6 +299,8 @@ module.exports = perspective => { var view = table.view({column_pivot: ["float"]}); const result = await view.col_to_js_typed_array("3.5|float"); expect(result.byteLength).toEqual(32); + view.delete(); + table.delete(); }); it("Undefined for non-int/float columns", async function() { @@ -285,6 +308,8 @@ module.exports = perspective => { var view = table.view(); const result = await view.col_to_js_typed_array("string"); expect(result).toBeUndefined(); + view.delete(); + table.delete(); }); it("Symmetric output with to_columns, 0-sided", async function() { @@ -300,6 +325,8 @@ module.exports = perspective => { expect(validate_typed_array(ta, cols[col])).toEqual(true); } } + view.delete(); + table.delete(); }); it("Symmetric output with to_columns, 1-sided", async function() { @@ -318,6 +345,8 @@ module.exports = perspective => { expect(validate_typed_array(ta, cols[col])).toEqual(true); } } + view.delete(); + table.delete(); }); }); @@ -328,6 +357,8 @@ module.exports = perspective => { var answer = `x,y,z\r\n1,a,true\r\n2,b,false\r\n3,c,true\r\n4,d,false`; let result2 = await view.to_csv(); expect(answer).toEqual(result2); + view.delete(); + table.delete(); }); it("Serializes 1 sided view to CSV", async function() { @@ -339,6 +370,8 @@ module.exports = perspective => { var answer = `__ROW_PATH__,x\r\n,10\r\nfalse,6\r\ntrue,4`; let result2 = await view.to_csv(); expect(answer).toEqual(result2); + view.delete(); + table.delete(); }); it("Serializes a 2 sided view to CSV", async function() { @@ -351,6 +384,8 @@ module.exports = perspective => { var answer = `__ROW_PATH__,\"a,x\",\"b,x\",\"c,x\",\"d,x\"\r\n,1,2,3,4\r\nfalse,,2,,4\r\ntrue,1,,3,`; let result2 = await view.to_csv(); expect(answer).toEqual(result2); + view.delete(); + table.delete(); }); it("Serializes a simple view to column-oriented JSON", async function() { @@ -358,6 +393,8 @@ module.exports = perspective => { var view = table.view({}); let result2 = await view.to_columns(); expect(data_7).toEqual(result2); + view.delete(); + table.delete(); }); }); @@ -367,6 +404,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(data).toEqual(result); + view.delete(); + table.delete(); }); it("JSON column oriented constructor", async function() { @@ -374,6 +413,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(data).toEqual(result); + view.delete(); + table.delete(); }); it("Arrow constructor", async function() { @@ -381,6 +422,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(arrow_result).toEqual(result); + view.delete(); + table.delete(); }); it("Arrow (chunked) constructor", async function() { @@ -388,6 +431,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(result.length).toEqual(10); + view.delete(); + table.delete(); }); it("CSV constructor", async function() { @@ -395,6 +440,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(papaparse.parse(csv, {header: true, dynamicTyping: true}).data).toEqual(result); + view.delete(); + table.delete(); }); it("Meta constructor", async function() { @@ -402,6 +449,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect([]).toEqual(result); + view.delete(); + table.delete(); }); it("Handles floats", async function() { @@ -409,73 +458,95 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(data_3).toEqual(result); + view.delete(); + table.delete(); }); it("has correct size", async function() { var table = perspective.table(data); let result = await table.size(); expect(result).toEqual(4); + table.delete(); }); it("has a schema", async function() { var table = perspective.table(data); let result = await table.schema(); expect(result).toEqual(meta); + table.delete(); }); it("has columns", async function() { var table = perspective.table(data); let result = await table.columns(); expect(result).toEqual(["x", "y", "z"]); + table.delete(); }); it("Handles floats schemas", async function() { var table = perspective.table(data_3); let result = await table.schema(); expect(meta_3).toEqual(result); + table.delete(); }); it("Generates correct date schemas", async function() { var table = perspective.table(data_4); let result = await table.schema(); expect(meta_4).toEqual(result); + table.delete(); }); it("Handles date updates when constructed from a schema", async function() { var table = perspective.table(meta_4); table.update(data_4); - let result = await table.view({}).to_json(); + let view = table.view(); + let result = await view.to_json(); expect([{v: +data_4[0]["v"]}]).toEqual(result); + view.delete(); + table.delete(); }); it("Handles datetime values", async function() { var table = perspective.table(data_4); - let result2 = await table.view({}).to_json(); + let view = table.view(); + let result2 = await view.to_json(); expect([{v: +data_4[0]["v"]}]).toEqual(result2); + view.delete(); + table.delete(); }); it("Handles datetime strings", async function() { var table = perspective.table(data_5); - let result2 = await table.view({}).to_json(); + let view = table.view(); + let result2 = await view.to_json(); expect([{v: +moment(data_5[0]["v"], "MM-DD-YYYY")}]).toEqual(result2); + view.delete(); + table.delete(); }); it("Handles date values", async function() { var table = perspective.table({v: "date"}); table.update(data_4); - let result2 = await table.view({}).to_json(); + let view = table.view(); + let result2 = await view.to_json(); let d = new Date(data_4[0]["v"]); d.setHours(0); d.setMinutes(0); d.setSeconds(0); d.setMilliseconds(0); expect([{v: +d}]).toEqual(result2); + view.delete(); + table.delete(); }); it("Handles utf16", async function() { var table = perspective.table(data_6); - let result = await table.view({}).to_json(); + let view = table.view({}); + let result = await view.to_json(); expect(data_6).toEqual(result); + view.delete(); + table.delete(); }); it("Computed column of arity 0", async function() { @@ -489,9 +560,13 @@ module.exports = perspective => { inputs: [] } ]); - let result = await table2.view({aggregate: [{op: "count", column: "const"}]}).to_json(); + let view = table2.view({aggregate: [{op: "count", column: "const"}]}); + let result = await view.to_json(); let expected = [{const: 1}, {const: 1}, {const: 1}, {const: 1}]; expect(expected).toEqual(result); + view.delete(); + table2.delete(); + table.delete(); }); it("Computed column of arity 2", async function() { @@ -505,9 +580,13 @@ module.exports = perspective => { inputs: ["w", "x"] } ]); - let result = await table2.view({aggregate: [{op: "count", column: "ratio"}]}).to_json(); + let view = table2.view({aggregate: [{op: "count", column: "ratio"}]}); + let result = await view.to_json(); let expected = [{ratio: 1.5}, {ratio: 1.25}, {ratio: 1.1666666666666667}, {ratio: 1.125}]; expect(expected).toEqual(result); + view.delete(); + table2.delete(); + table.delete(); }); it("Computed column of arity 2 with updates on non-dependent columns", async function() { @@ -531,10 +610,13 @@ module.exports = perspective => { let delta_upd = [{y: "a", z: false}, {y: "b", z: true}, {y: "c", z: false}, {y: "d", z: true}]; table2.update(delta_upd); - - let result = await table2.view({aggregate: [{op: "count", column: "y"}, {op: "count", column: "ratio"}]}).to_json(); + let view = table2.view({aggregate: [{op: "count", column: "y"}, {op: "count", column: "ratio"}]}); + let result = await view.to_json(); let expected = [{y: "a", ratio: 1.5}, {y: "b", ratio: 1.25}, {y: "c", ratio: 1.1666666666666667}, {y: "d", ratio: 1.125}]; expect(expected).toEqual(result); + view.delete(); + table2.delete(); + table.delete(); }); it("String computed column of arity 1", async function() { @@ -548,9 +630,13 @@ module.exports = perspective => { inputs: ["z"] } ]); - let result = await table2.view({aggregate: [{op: "count", column: "yes/no"}]}).to_json(); + let view = table2.view({aggregate: [{op: "count", column: "yes/no"}]}); + let result = await view.to_json(); let expected = [{"yes/no": "yes"}, {"yes/no": "no"}, {"yes/no": "yes"}, {"yes/no": "no"}]; expect(expected).toEqual(result); + view.delete(); + table2.delete(); + table.delete(); }); it("Computed schema returns names and metadata", async function() { @@ -587,33 +673,34 @@ module.exports = perspective => { }; expect(expected).toEqual(result); + table2.delete(); + table.delete(); }); it("Column metadata returns names and type", async function() { let table = perspective.table(data); let result = await table.column_metadata(); expect(result).toEqual(column_meta); + table.delete(); }); - it( - "allocates a large tables", - async function() { - function makeid() { - var text = ""; - var possible = Array.from(Array(26).keys()).map(x => String.fromCharCode(x + 65)); - for (var i = 0; i < 15; i++) text += possible[Math.floor(Math.random() * possible.length)]; - return text; - } - let data = []; - for (let i = 0; i < 35000; i++) { - data.push([{a: makeid(), b: makeid(), c: makeid(), d: makeid(), w: i + 0.5, x: i, y: makeid()}]); - } - let table = perspective.table(data); - let view = table.view(); - let result = await view.to_json(); - expect(result.length).toEqual(35000); - }, - 3000 - ); + it("allocates a large tables", async function() { + function makeid() { + var text = ""; + var possible = Array.from(Array(26).keys()).map(x => String.fromCharCode(x + 65)); + for (var i = 0; i < 15; i++) text += possible[Math.floor(Math.random() * possible.length)]; + return text; + } + let data = []; + for (let i = 0; i < 35000; i++) { + data.push([{a: makeid(), b: makeid(), c: makeid(), d: makeid(), w: i + 0.5, x: i, y: makeid()}]); + } + let table = perspective.table(data); + let view = table.view(); + let result = await view.to_json(); + expect(result.length).toEqual(35000); + view.delete(); + table.delete(); + }, 3000); }); }; diff --git a/packages/perspective/test/js/filters.js b/packages/perspective/test/js/filters.js index b3a84aa9bc..fc6e88f161 100644 --- a/packages/perspective/test/js/filters.js +++ b/packages/perspective/test/js/filters.js @@ -25,6 +25,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect(json.length).toEqual(2); + view.delete(); + table.delete(); }); it("x > 2", async function() { @@ -34,6 +36,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect(rdata.slice(2)).toEqual(json); + view.delete(); + table.delete(); }); it("x < 3", async function() { @@ -43,6 +47,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect(rdata.slice(0, 2)).toEqual(json); + view.delete(); + table.delete(); }); it("x > 4", async function() { @@ -52,6 +58,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect([]).toEqual(json); + view.delete(); + table.delete(); }); it("x < 0", async function() { @@ -61,6 +69,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect([]).toEqual(json); + view.delete(); + table.delete(); }); }); @@ -72,6 +82,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect(rdata.slice(0, 1)).toEqual(json); + view.delete(); + table.delete(); }); it("x == 5", async function() { @@ -81,6 +93,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect([]).toEqual(json); + view.delete(); + table.delete(); }); it("y == 'a'", async function() { @@ -90,6 +104,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect(rdata.slice(0, 1)).toEqual(json); + view.delete(); + table.delete(); }); it("y == 'e'", async function() { @@ -99,6 +115,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect([]).toEqual(json); + view.delete(); + table.delete(); }); it("z == true", async function() { @@ -108,6 +126,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect([rdata[0], rdata[2]]).toEqual(json); + view.delete(); + table.delete(); }); it("z == false", async function() { @@ -117,6 +137,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect([rdata[1], rdata[3]]).toEqual(json); + view.delete(); + table.delete(); }); it("w == yesterday", async function() { @@ -126,6 +148,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect([rdata[3]]).toEqual(json); + view.delete(); + table.delete(); }); it("w != yesterday", async function() { @@ -135,6 +159,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect(rdata.slice(0, 3)).toEqual(json); + view.delete(); + table.delete(); }); }); @@ -146,6 +172,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect(rdata.slice(0, 2)).toEqual(json); + view.delete(); + table.delete(); }); }); @@ -157,6 +185,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect(rdata.slice(0, 3)).toEqual(json); + view.delete(); + table.delete(); }); }); @@ -168,6 +198,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect(rdata.slice(0, 1)).toEqual(json); + view.delete(); + table.delete(); }); }); @@ -179,6 +211,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect(rdata.slice(1, 3)).toEqual(json); + view.delete(); + table.delete(); }); it("y contains 'a' | y contains 'b'", async function() { @@ -189,6 +223,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect(rdata.slice(0, 2)).toEqual(json); + view.delete(); + table.delete(); }); }); @@ -201,6 +237,8 @@ module.exports = perspective => { var answer = [{x: 3, y: 1}, {x: 4, y: 2}]; let result = await view.to_json(); expect(result).toEqual(answer); + view.delete(); + table.delete(); }); it("x < 3", async function() { @@ -211,6 +249,8 @@ module.exports = perspective => { var answer = [{x: 2, y: 1}]; let result = await view.to_json(); expect(result).toEqual(answer); + view.delete(); + table.delete(); }); it("x > 2", async function() { @@ -222,6 +262,8 @@ module.exports = perspective => { var answer = [{x: 3.5, y: 1}, {x: 4.5, y: 2}]; let result = await view.to_json(); expect(answer).toEqual(result); + view.delete(); + table.delete(); }); }); }); diff --git a/packages/perspective/test/js/internal.js b/packages/perspective/test/js/internal.js index 6439ad5af8..4badd00096 100644 --- a/packages/perspective/test/js/internal.js +++ b/packages/perspective/test/js/internal.js @@ -26,6 +26,7 @@ module.exports = (perspective, mode) => { }); }; expect(anon).toThrow(); + table.delete(); }); it("Arrow schema types are mapped correctly", async function() { @@ -49,6 +50,7 @@ module.exports = (perspective, mode) => { stypes.delete(); } } + table.delete(); }); }); }; diff --git a/packages/perspective/test/js/perspective.spec.js b/packages/perspective/test/js/perspective.spec.js index e38fe42cfa..e22a3f480d 100644 --- a/packages/perspective/test/js/perspective.spec.js +++ b/packages/perspective/test/js/perspective.spec.js @@ -15,7 +15,6 @@ const RUNTIMES = { ASMJS: perspective( asmjs.load_perspective({ wasmJSMethod: "asmjs", - filePackagePrefixURL: "", printErr: x => console.error(x), print: x => console.log(x) }) diff --git a/packages/perspective/test/js/pivots.js b/packages/perspective/test/js/pivots.js index 0e6a22c31b..c875f32b14 100644 --- a/packages/perspective/test/js/pivots.js +++ b/packages/perspective/test/js/pivots.js @@ -35,6 +35,8 @@ module.exports = perspective => { var answer = [{__ROW_PATH__: [], x: 10}, {__ROW_PATH__: [false], x: 6}, {__ROW_PATH__: [true], x: 4}]; let result = await view.to_json(); expect(answer).toEqual(result); + view.delete(); + table.delete(); }); it("['z'], sum with new column syntax", async function() { @@ -46,6 +48,8 @@ module.exports = perspective => { var answer = [{__ROW_PATH__: [], x: 10}, {__ROW_PATH__: [false], x: 6}, {__ROW_PATH__: [true], x: 4}]; let result = await view.to_json(); expect(answer).toEqual(result); + view.delete(); + table.delete(); }); it("['z'], weighted_mean", async function() { @@ -57,6 +61,8 @@ module.exports = perspective => { var answer = [{__ROW_PATH__: [], x: 2.5, "x|y": 2.8333333333333335}, {__ROW_PATH__: [false], x: 3, "x|y": 3.3333333333333335}, {__ROW_PATH__: [true], x: 2, "x|y": 2.3333333333333335}]; let result = await view.to_json(); expect(answer).toEqual(result); + view.delete(); + table.delete(); }); it("['z'], mean", async function() { @@ -68,6 +74,8 @@ module.exports = perspective => { var answer = [{__ROW_PATH__: [], x: 2.5}, {__ROW_PATH__: [false], x: 3}, {__ROW_PATH__: [true], x: 2}]; let result = await view.to_json(); expect(answer).toEqual(result); + view.delete(); + table.delete(); }); it("['z'], first by index", async function() { @@ -79,6 +87,8 @@ module.exports = perspective => { var answer = [{__ROW_PATH__: [], x: 1}, {__ROW_PATH__: [false], x: 2}, {__ROW_PATH__: [true], x: 1}]; let result = await view.to_json(); expect(answer).toEqual(result); + view.delete(); + table.delete(); }); it("['z'], last by index", async function() { @@ -90,6 +100,8 @@ module.exports = perspective => { var answer = [{__ROW_PATH__: [], x: 4}, {__ROW_PATH__: [false], x: 4}, {__ROW_PATH__: [true], x: 3}]; let result = await view.to_json(); expect(answer).toEqual(result); + view.delete(); + table.delete(); }); it("['z'], last", async function() { @@ -106,6 +118,8 @@ module.exports = perspective => { var answerAfterUpdate = [{__ROW_PATH__: [], x: 1}, {__ROW_PATH__: [false], x: 2}, {__ROW_PATH__: [true], x: 1}]; let result2 = await view.to_json(); expect(answerAfterUpdate).toEqual(result2); + view.delete(); + table.delete(); }); }); @@ -119,6 +133,8 @@ module.exports = perspective => { var answer = [{__ROW_PATH__: [], x: 3}, {__ROW_PATH__: [1], x: 2.5}, {__ROW_PATH__: [2], x: 4}]; let result = await view.to_json(); expect(answer).toEqual(result); + view.delete(); + table.delete(); }); it("mean with 0", async function() { @@ -130,6 +146,8 @@ module.exports = perspective => { var answer = [{__ROW_PATH__: [], x: 2.5}, {__ROW_PATH__: [1], x: 2}, {__ROW_PATH__: [2], x: 4}]; let result = await view.to_json(); expect(answer).toEqual(result); + view.delete(); + table.delete(); }); it("mean with 0.0 (floats)", async function() { @@ -142,6 +160,8 @@ module.exports = perspective => { var answer = [{__ROW_PATH__: [], x: 2.5}, {__ROW_PATH__: [1], x: 2}, {__ROW_PATH__: [2], x: 4}]; let result = await view.to_json(); expect(answer).toEqual(result); + view.delete(); + table.delete(); }); it("sum", async function() { @@ -153,6 +173,8 @@ module.exports = perspective => { var answer = [{__ROW_PATH__: [], x: 9}, {__ROW_PATH__: [1], x: 5}, {__ROW_PATH__: [2], x: 4}]; let result = await view.to_json(); expect(answer).toEqual(result); + view.delete(); + table.delete(); }); it("mean after update", async function() { @@ -165,6 +187,8 @@ module.exports = perspective => { var answer = [{__ROW_PATH__: [], x: 3}, {__ROW_PATH__: [1], x: 2.5}, {__ROW_PATH__: [2], x: 4}]; let result = await view.to_json(); expect(answer).toEqual(result); + view.delete(); + table.delete(); }); it("mean at aggregate level", async function() { @@ -187,6 +211,8 @@ module.exports = perspective => { ]; let result = await view.to_json(); expect(answer).toEqual(result); + view.delete(); + table.delete(); }); it("null_in_pivot_column", async function() { @@ -198,6 +224,21 @@ module.exports = perspective => { var answer = [{__ROW_PATH__: [], x: 3}, {__ROW_PATH__: ["x"], x: 1}, {__ROW_PATH__: ["y"], x: 1}, {__ROW_PATH__: [null], x: 1}]; let result = await view.to_json(); expect(result).toEqual(answer); + view.delete(); + table.delete(); + }); + + it("weighted mean", async function() { + var table = perspective.table([{a: "a", x: 1, y: 200}, {a: "a", x: 2, y: 100}, {a: "a", x: 3, y: null}]); + var view = table.view({ + row_pivot: ["a"], + aggregate: [{op: "weighted mean", column: ["y", "x"], name: "y"}] + }); + var answer = [{__ROW_PATH__: [], y: (1 * 200 + 2 * 100) / (1 + 2)}, {__ROW_PATH__: ["a"], y: (1 * 200 + 2 * 100) / (1 + 2)}]; + let result = await view.to_json(); + expect(answer).toEqual(result); + view.delete(); + table.delete(); }); }); @@ -208,14 +249,16 @@ module.exports = perspective => { row_pivot: ["x"] }); var answer = [ - {__ROW_PATH__: [], x: 4, y: 4, z: 2}, + {__ROW_PATH__: [], x: 10, y: 4, z: 2}, {__ROW_PATH__: [1], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [2], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [3], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [4], x: 1, y: 1, z: 1} + {__ROW_PATH__: [2], x: 2, y: 1, z: 1}, + {__ROW_PATH__: [3], x: 3, y: 1, z: 1}, + {__ROW_PATH__: [4], x: 4, y: 1, z: 1} ]; let result2 = await view.to_json(); - expect(answer).toEqual(result2); + expect(result2).toEqual(answer); + view.delete(); + table.delete(); }); it("['x'] test update pkey column", async function() { @@ -237,6 +280,8 @@ module.exports = perspective => { let result2 = await view.to_json(); var answer = [{__ROW_PATH__: [], pos: 600}, {__ROW_PATH__: [1], pos: 100}, {__ROW_PATH__: [2], pos: 200}, {__ROW_PATH__: [3], pos: 300}]; expect(answer).toEqual(result2); + view.delete(); + table.delete(); }); it("['x'] has a schema", async function() { @@ -246,6 +291,8 @@ module.exports = perspective => { }); let result2 = await view.schema(); expect(result2).toEqual(meta); + view.delete(); + table.delete(); }); it("['x'] translates type `string` to `integer` when pivoted by row", async function() { @@ -256,6 +303,8 @@ module.exports = perspective => { }); let result2 = await view.schema(); expect(result2).toEqual({y: "integer"}); + view.delete(); + table.delete(); }); it("['x'] translates type `integer` to `float` when pivoted by row", async function() { @@ -266,6 +315,8 @@ module.exports = perspective => { }); let result2 = await view.schema(); expect(result2).toEqual({x: "float"}); + view.delete(); + table.delete(); }); it("['x'] does not translate type when only pivoted by column", async function() { @@ -276,6 +327,8 @@ module.exports = perspective => { }); let result2 = await view.schema(); expect(result2).toEqual({x: "integer"}); + view.delete(); + table.delete(); }); it("['x'] has the correct # of rows", async function() { @@ -285,6 +338,8 @@ module.exports = perspective => { }); let result2 = await view.num_rows(); expect(result2).toEqual(5); + view.delete(); + table.delete(); }); it("['x'] has the correct # of columns", async function() { @@ -294,6 +349,8 @@ module.exports = perspective => { }); let result2 = await view.num_columns(); expect(result2).toEqual(3); + view.delete(); + table.delete(); }); it("['z']", async function() { @@ -301,9 +358,11 @@ module.exports = perspective => { var view = table.view({ row_pivot: ["z"] }); - var answer = [{__ROW_PATH__: [], x: 4, y: 4, z: 2}, {__ROW_PATH__: [false], x: 2, y: 2, z: 1}, {__ROW_PATH__: [true], x: 2, y: 2, z: 1}]; + var answer = [{__ROW_PATH__: [], x: 10, y: 4, z: 2}, {__ROW_PATH__: [false], x: 6, y: 2, z: 1}, {__ROW_PATH__: [true], x: 4, y: 2, z: 1}]; let result2 = await view.to_json(); - expect(answer).toEqual(result2); + expect(result2).toEqual(answer); + view.delete(); + table.delete(); }); it("['x', 'z']", async function() { @@ -311,21 +370,21 @@ module.exports = perspective => { var view = table.view({ row_pivot: ["x", "z"] }); - var answer = [ - {__ROW_PATH__: [], x: 4, y: 4, z: 2}, + {__ROW_PATH__: [], x: 10, y: 4, z: 2}, {__ROW_PATH__: [1], x: 1, y: 1, z: 1}, {__ROW_PATH__: [1, true], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [2], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [2, false], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [3], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [3, true], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [4], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [4, false], x: 1, y: 1, z: 1} + {__ROW_PATH__: [2], x: 2, y: 1, z: 1}, + {__ROW_PATH__: [2, false], x: 2, y: 1, z: 1}, + {__ROW_PATH__: [3], x: 3, y: 1, z: 1}, + {__ROW_PATH__: [3, true], x: 3, y: 1, z: 1}, + {__ROW_PATH__: [4], x: 4, y: 1, z: 1}, + {__ROW_PATH__: [4, false], x: 4, y: 1, z: 1} ]; - let result2 = await view.to_json(); - expect(answer).toEqual(result2); + expect(result2).toEqual(answer); + view.delete(); + table.delete(); }); it("['x', 'z'] windowed", async function() { @@ -333,19 +392,19 @@ module.exports = perspective => { var view = table.view({ row_pivot: ["x", "z"] }); - var answer = [ {__ROW_PATH__: [1, true], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [2], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [2, false], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [3], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [3, true], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [4], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [4, false], x: 1, y: 1, z: 1} + {__ROW_PATH__: [2], x: 2, y: 1, z: 1}, + {__ROW_PATH__: [2, false], x: 2, y: 1, z: 1}, + {__ROW_PATH__: [3], x: 3, y: 1, z: 1}, + {__ROW_PATH__: [3, true], x: 3, y: 1, z: 1}, + {__ROW_PATH__: [4], x: 4, y: 1, z: 1}, + {__ROW_PATH__: [4, false], x: 4, y: 1, z: 1} ]; - let result2 = await view.to_json({start_row: 2}); - expect(answer).toEqual(result2); + expect(result2).toEqual(answer); + view.delete(); + table.delete(); }); it("['x', 'z'], pivot_depth = 1", async function() { @@ -354,17 +413,17 @@ module.exports = perspective => { row_pivot: ["x", "z"], row_pivot_depth: 1 }); - var answer = [ - {__ROW_PATH__: [], x: 4, y: 4, z: 2}, + {__ROW_PATH__: [], x: 10, y: 4, z: 2}, {__ROW_PATH__: [1], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [2], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [3], x: 1, y: 1, z: 1}, - {__ROW_PATH__: [4], x: 1, y: 1, z: 1} + {__ROW_PATH__: [2], x: 2, y: 1, z: 1}, + {__ROW_PATH__: [3], x: 3, y: 1, z: 1}, + {__ROW_PATH__: [4], x: 4, y: 1, z: 1} ]; - let result2 = await view.to_json(); - expect(answer).toEqual(result2); + expect(result2).toEqual(answer); + view.delete(); + table.delete(); }); }); @@ -376,6 +435,8 @@ module.exports = perspective => { }); let result2 = await view.schema(); expect(meta).toEqual(result2); + view.delete(); + table.delete(); }); it("['x'] only, column-oriented input", async function() { @@ -390,6 +451,8 @@ module.exports = perspective => { {"true|w": 3.5, "true|x": 3, "true|y": "c", "true|z": true, "false|w": null, "false|x": null, "false|y": null, "false|z": null}, {"true|w": null, "true|x": null, "true|y": null, "true|z": null, "false|w": 4.5, "false|x": 4, "false|y": "d", "false|z": false} ]); + view.delete(); + table.delete(); }); it("['x'] only, column-oriented output", async function() { @@ -408,6 +471,8 @@ module.exports = perspective => { "false|y": [null, "b", null, "d"], "false|z": [null, false, null, false] }); + view.delete(); + table.delete(); }); it("['x'] only", async function() { @@ -423,23 +488,90 @@ module.exports = perspective => { ]; let result2 = await view.to_json(); expect(answer).toEqual(result2); + view.delete(); + table.delete(); }); - it("['x']", async function() { + it("['x'] by ['y']", async function() { var table = perspective.table(data); var view = table.view({ column_pivot: ["y"], row_pivot: ["x"] }); var answer = [ - {__ROW_PATH__: [], "a|x": 1, "a|y": 1, "a|z": 1, "b|x": 1, "b|y": 1, "b|z": 1, "c|x": 1, "c|y": 1, "c|z": 1, "d|x": 1, "d|y": 1, "d|z": 1}, + {__ROW_PATH__: [], "a|x": 1, "a|y": 1, "a|z": 1, "b|x": 2, "b|y": 1, "b|z": 1, "c|x": 3, "c|y": 1, "c|z": 1, "d|x": 4, "d|y": 1, "d|z": 1}, {__ROW_PATH__: [1], "a|x": 1, "a|y": 1, "a|z": 1, "b|x": null, "b|y": null, "b|z": null, "c|x": null, "c|y": null, "c|z": null, "d|x": null, "d|y": null, "d|z": null}, - {__ROW_PATH__: [2], "a|x": null, "a|y": null, "a|z": null, "b|x": 1, "b|y": 1, "b|z": 1, "c|x": null, "c|y": null, "c|z": null, "d|x": null, "d|y": null, "d|z": null}, - {__ROW_PATH__: [3], "a|x": null, "a|y": null, "a|z": null, "b|x": null, "b|y": null, "b|z": null, "c|x": 1, "c|y": 1, "c|z": 1, "d|x": null, "d|y": null, "d|z": null}, - {__ROW_PATH__: [4], "a|x": null, "a|y": null, "a|z": null, "b|x": null, "b|y": null, "b|z": null, "c|x": null, "c|y": null, "c|z": null, "d|x": 1, "d|y": 1, "d|z": 1} + {__ROW_PATH__: [2], "a|x": null, "a|y": null, "a|z": null, "b|x": 2, "b|y": 1, "b|z": 1, "c|x": null, "c|y": null, "c|z": null, "d|x": null, "d|y": null, "d|z": null}, + {__ROW_PATH__: [3], "a|x": null, "a|y": null, "a|z": null, "b|x": null, "b|y": null, "b|z": null, "c|x": 3, "c|y": 1, "c|z": 1, "d|x": null, "d|y": null, "d|z": null}, + {__ROW_PATH__: [4], "a|x": null, "a|y": null, "a|z": null, "b|x": null, "b|y": null, "b|z": null, "c|x": null, "c|y": null, "c|z": null, "d|x": 4, "d|y": 1, "d|z": 1} ]; let result2 = await view.to_json(); - expect(answer).toEqual(result2); + expect(result2).toEqual(answer); + view.delete(); + table.delete(); + }); + }); + + describe("Column pivot w/sort", function() { + it("['y'] by ['z'], sorted by 'x'", async function() { + var table = perspective.table([ + {x: 7, y: "A", z: true}, + {x: 2, y: "A", z: false}, + {x: 5, y: "A", z: true}, + {x: 4, y: "A", z: false}, + {x: 1, y: "B", z: true}, + {x: 8, y: "B", z: false}, + {x: 3, y: "B", z: true}, + {x: 6, y: "B", z: false}, + {x: 9, y: "C", z: true}, + {x: 10, y: "C", z: false}, + {x: 11, y: "C", z: true}, + {x: 12, y: "C", z: false} + ]); + var view = table.view({ + column_pivot: ["z"], + row_pivot: ["y"], + sort: [["x", "desc"]] + }); + + let answer = [ + {__ROW_PATH__: [], "false|x": 42, "false|y": 3, "false|z": 1, "true|x": 36, "true|y": 3, "true|z": 1}, + {__ROW_PATH__: ["C"], "false|x": 22, "false|y": 1, "false|z": 1, "true|x": 20, "true|y": 1, "true|z": 1}, + {__ROW_PATH__: ["A"], "false|x": 6, "false|y": 1, "false|z": 1, "true|x": 12, "true|y": 1, "true|z": 1}, + {__ROW_PATH__: ["B"], "false|x": 14, "false|y": 1, "false|z": 1, "true|x": 4, "true|y": 1, "true|z": 1} + ]; + let result2 = await view.to_json(); + expect(result2).toEqual(answer); + view.delete(); + table.delete(); + }); + + it("['z'] by ['y'], sorted by 'y'", async function() { + var table = perspective.table([ + {x: 7, y: "A", z: true}, + {x: 2, y: "A", z: false}, + {x: 5, y: "A", z: true}, + {x: 4, y: "A", z: false}, + {x: 1, y: "B", z: true}, + {x: 8, y: "B", z: false}, + {x: 3, y: "B", z: true}, + {x: 6, y: "B", z: false}, + {x: 9, y: "C", z: true}, + {x: 10, y: "C", z: false}, + {x: 11, y: "C", z: true}, + {x: 12, y: "C", z: false} + ]); + var view = table.view({ + column_pivot: ["y"], + row_pivot: ["z"], + sort: [["y", "desc"]], + aggregate: [{column: "x", op: "sum"}, {column: "y", op: "any"}] + }); + + let result2 = await view.to_columns(); + expect(Object.keys(result2)).toEqual(["__ROW_PATH__", "C|x", "C|y", "B|x", "B|y", "A|x", "A|y"]); + view.delete(); + table.delete(); }); }); }; diff --git a/packages/perspective/test/js/test_browser.js b/packages/perspective/test/js/test_browser.js index 1346f88468..f961ae57d9 100644 --- a/packages/perspective/test/js/test_browser.js +++ b/packages/perspective/test/js/test_browser.js @@ -24,7 +24,6 @@ const RUNTIMES = { ASMJS: perspective( asmjs.load_perspective({ wasmJSMethod: "asmjs", - filePackagePrefixURL: "", printErr: x => console.error(x), print: x => console.log(x) }) diff --git a/packages/perspective/test/js/updates.js b/packages/perspective/test/js/updates.js index 3d7d1b7cf2..c3d73c94fc 100644 --- a/packages/perspective/test/js/updates.js +++ b/packages/perspective/test/js/updates.js @@ -51,6 +51,8 @@ module.exports = perspective => { let result = await view.to_json(); expect(result.length).toEqual(2); expect(data.slice(2, 4)).toEqual(result); + view.delete(); + table.delete(); }); it("after a regular data load`", async function() { @@ -60,6 +62,8 @@ module.exports = perspective => { let result = await view.to_json(); expect(result.length).toEqual(2); expect(data.slice(2, 4)).toEqual(result); + view.delete(); + table.delete(); }); it("multiple single element removes", async function() { @@ -74,6 +78,8 @@ module.exports = perspective => { let result = await view.to_json(); expect(result).toEqual([{x: 0, y: "test", z: false}]); expect(result.length).toEqual(1); + view.delete(); + table.delete(); }); }); @@ -84,6 +90,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(result).toEqual([{x: 1, y: "a"}, {x: 2, y: "b"}, {x: 3, y: "c"}, {x: 4, y: "d"}]); + view.delete(); + table.delete(); }); it("coerces to string", async function() { @@ -92,6 +100,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(result).toEqual([{x: "1", y: "a", z: "true"}, {x: "2", y: "b", z: "false"}, {x: "3", y: "c", z: "true"}, {x: "4", y: "d", z: "false"}]); + view.delete(); + table.delete(); }); }); @@ -102,6 +112,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(data).toEqual(result); + view.delete(); + table.delete(); }); it("Meta constructor then column oriented `update()`", async function() { @@ -110,6 +122,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(result).toEqual(data); + view.delete(); + table.delete(); }); it("Column oriented `update()` with columns in different order to schema", async function() { @@ -125,6 +139,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(result).toEqual(data); + view.delete(); + table.delete(); }); it("Column data constructor then column oriented `update()`", async function() { @@ -141,14 +157,19 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(result).toEqual(expected); + view.delete(); + table.delete(); }); it("`update()` unbound to table", async function() { var table = perspective.table(meta); var updater = table.update; updater(data); - let result = await table.view().to_json(); + let view = table.view(); + let result = await view.to_json(); expect(data).toEqual(result); + view.delete(); + table.delete(); }); it("Multiple `update()`s", async function() { @@ -158,6 +179,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(data.concat(data)).toEqual(result); + view.delete(); + table.delete(); }); it("`update()` called after `view()`", async function() { @@ -166,6 +189,8 @@ module.exports = perspective => { table.update(data); let result = await view.to_json(); expect(data).toEqual(result); + view.delete(); + table.delete(); }); it("Arrow `update()`s", async function() { @@ -174,6 +199,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(arrow_result.concat(arrow_result)).toEqual(result); + view.delete(); + table.delete(); }); }); @@ -190,9 +217,13 @@ module.exports = perspective => { } ]); table2.update(data); - let result = await table2.view({aggregate: [{op: "count", column: "yes/no"}]}).to_json(); + let view2 = table2.view({aggregate: [{op: "count", column: "yes/no"}]}); + let result = await view2.to_json(); let expected = [{"yes/no": "yes"}, {"yes/no": "no"}, {"yes/no": "yes"}, {"yes/no": "no"}, {"yes/no": "yes"}, {"yes/no": "no"}, {"yes/no": "yes"}, {"yes/no": "no"}]; expect(result).toEqual(expected); + view2.delete(); + table.delete(); + table2.delete(); }); }); @@ -202,6 +233,8 @@ module.exports = perspective => { var view = table.view(); view.on_update(function(new_data) { expect(data).toEqual(new_data); + view.delete(); + table.delete(); done(); }); table.update(data); @@ -216,6 +249,8 @@ module.exports = perspective => { if (!ran) { expect(new_data).toEqual(data); ran = true; + view.delete(); + table.delete(); done(); } }); @@ -231,6 +266,10 @@ module.exports = perspective => { table2.update(x); let result = await view2.to_json(); expect(data).toEqual(result); + view1.delete(); + view2.delete(); + table1.delete(); + table2.delete(); done(); }); table1.update(data); @@ -248,6 +287,10 @@ module.exports = perspective => { table2.update(x); let result = await view2.to_json(); expect(data.concat(data)).toEqual(result); + view1.delete(); + view2.delete(); + table1.delete(); + table2.delete(); done(); }); table1.update(data); @@ -260,6 +303,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(data.slice(2)).toEqual(result); + view.delete(); + table.delete(); }); it("{limit: 5} with 2 updates of size 4", async function() { @@ -273,6 +318,8 @@ module.exports = perspective => { .concat(data.slice(3, 4)) .concat(data.slice(0, 1)) ).toEqual(result); + view.delete(); + table.delete(); }); }); @@ -283,6 +330,8 @@ module.exports = perspective => { table.update(data); let result = await view.to_json(); expect(data).toEqual(result); + view.delete(); + table.delete(); }); it("{index: 'y'} (string)", async function() { @@ -291,6 +340,8 @@ module.exports = perspective => { table.update(data); let result = await view.to_json(); expect(data).toEqual(result); + view.delete(); + table.delete(); }); it("Arrow with {index: 'i64'} (int)", async function() { @@ -298,6 +349,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(arrow_result).toEqual(result); + view.delete(); + table.delete(); }); it("Arrow with {index: 'char'} (char)", async function() { @@ -305,6 +358,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(arrow_indexed_result).toEqual(result); + view.delete(); + table.delete(); }); it("Arrow with {index: 'dict'} (dict)", async function() { @@ -312,6 +367,8 @@ module.exports = perspective => { var view = table.view(); let result = await view.to_json(); expect(arrow_indexed_result).toEqual(result); + view.delete(); + table.delete(); }); it("multiple updates on {index: 'x'}", async function() { @@ -322,6 +379,8 @@ module.exports = perspective => { table.update(data); let result = await view.to_json(); expect(data).toEqual(result); + view.delete(); + table.delete(); }); it("{index: 'x'} with overlap", async function() { @@ -331,6 +390,8 @@ module.exports = perspective => { table.update(data_2); let result = await view.to_json(); expect(data.slice(0, 2).concat(data_2)).toEqual(result); + view.delete(); + table.delete(); }); it("update and index (int)", function(done) { @@ -341,6 +402,8 @@ module.exports = perspective => { expect(data_2).toEqual(new_data); let json = await view.to_json(); expect(data.slice(0, 2).concat(data_2)).toEqual(json); + view.delete(); + table.delete(); done(); }); table.update(data_2); @@ -354,6 +417,8 @@ module.exports = perspective => { expect(data_2).toEqual(new_data); let json = await view.to_json(); expect(data.slice(0, 2).concat(data_2)).toEqual(json); + view.delete(); + table.delete(); done(); }); table.update(data_2); @@ -379,6 +444,8 @@ module.exports = perspective => { {__ROW_PATH__: [true, "c"]} ]; expect(expected).toEqual(result); + view.delete(); + table.delete(); }); it("partial update", function(done) { @@ -391,6 +458,8 @@ module.exports = perspective => { expect(new_data).toEqual(expected.slice(0, 2)); let json = await view.to_json(); expect(json).toEqual(expected); + view.delete(); + table.delete(); done(); }); table.update(partial); @@ -411,6 +480,8 @@ module.exports = perspective => { expect(new_data).toEqual(expected.slice(0, 2)); let json = await view.to_json(); expect(json).toEqual(expected); + view.delete(); + table.delete(); done(); }); table.update(partial); @@ -430,6 +501,8 @@ module.exports = perspective => { expect(new_data).toEqual(expected.slice(0, 2)); let json = await view.to_json(); expect(json).toEqual(expected); + view.delete(); + table.delete(); done(); }); table.update(partial); @@ -446,6 +519,8 @@ module.exports = perspective => { }); let json = await view.to_json(); expect(json).toEqual([{__ROW_PATH__: [], y: 1}, {__ROW_PATH__: [1], y: 1}, {__ROW_PATH__: [2], y: 0}]); + view.delete(); + table.delete(); }); it("can be removed entirely", async function() { @@ -455,6 +530,8 @@ module.exports = perspective => { var view = table.view(); let json = await view.to_json(); expect(json).toEqual([{x: 1, y: 1}]); + view.delete(); + table.delete(); }); it("partial update with null unsets value", function(done) { @@ -466,6 +543,8 @@ module.exports = perspective => { table.update(partial); view.to_json().then(json => { expect(json).toEqual(expected); + view.delete(); + table.delete(); done(); }); }); @@ -479,6 +558,8 @@ module.exports = perspective => { table.update(update); view.to_json().then(json => { expect(json).toEqual(expected); + view.delete(); + table.delete(); done(); }); }); @@ -496,6 +577,8 @@ module.exports = perspective => { table.update(partial); view.to_json().then(json => { expect(json).toEqual(expected); + view.delete(); + table.delete(); done(); }); }); @@ -511,6 +594,8 @@ module.exports = perspective => { }); let result = await view.to_json(); expect(data.slice(0, 2)).toEqual(result); + view.delete(); + table.delete(); }); it("`top`", async function() { @@ -522,6 +607,8 @@ module.exports = perspective => { }); let result = await view.to_json(); expect(data.slice(2)).toEqual(result); + view.delete(); + table.delete(); }); it("`width`", async function() { @@ -534,6 +621,8 @@ module.exports = perspective => { var result2 = _.map(data, x => _.pick(x, "x", "y")); let result = await view.to_json(); expect(result2).toEqual(result); + view.delete(); + table.delete(); }); it("`left`", async function() { @@ -548,6 +637,8 @@ module.exports = perspective => { }); let result2 = await view.to_json(); expect(result).toEqual(result2); + view.delete(); + table.delete(); }); it("All", async function() { @@ -564,7 +655,9 @@ module.exports = perspective => { return _.pick(x, "y"); }); let result2 = await view.to_json(); - expect(result.slice(1, 3)).toEqual(result2); + expect(result2).toEqual(result.slice(1, 3)); + view.delete(); + table.delete(); }); }); }; diff --git a/packages/perspective/webpack-plugin.js b/packages/perspective/webpack-plugin.js index 1e4103231f..e4b189ef6d 100644 --- a/packages/perspective/webpack-plugin.js +++ b/packages/perspective/webpack-plugin.js @@ -48,7 +48,13 @@ class PerspectiveWebpackPlugin { rules.push({ test: /perspective\.(asmjs|wasm)\.js$/, include: load_path, - use: {loader: BLOB_LOADER_PATH, options: {name: "[name].worker.js"}} + use: [{ + loader: WORKER_LOADER_PATH, + options: {name: "[name].js", compiled: true} + }, { + loader: BLOB_LOADER_PATH, + options: {name: "[name].worker.js"} + }] }); } else { rules.push({ @@ -64,7 +70,7 @@ class PerspectiveWebpackPlugin { rules.push({ test: /\.js$/, include: load_path, - exclude: /node_modules[/\\](?!\@jpmorganchase)|psp\.(asmjs|async|sync)\.js/, + exclude: /node_modules[/\\](?!\@jpmorganchase)|psp\.(asmjs|async|sync)\.js|perspective\.(asmjs|wasm)\.worker\.js/, loader: "babel-loader", options: BABEL_CONFIG }); diff --git a/scripts/CMakeLists.txt b/scripts/CMakeLists.txt index 8ed90a8154..7cd8177dd6 100644 --- a/scripts/CMakeLists.txt +++ b/scripts/CMakeLists.txt @@ -22,11 +22,11 @@ if (PSP_WASM_BUILD) set(EXTENDED_FLAGS " \ --bind \ + --source-map-base ./build/ \ --memory-init-file 0 \ -s NO_EXIT_RUNTIME=1 \ -s NO_FILESYSTEM=1 \ -s ALLOW_MEMORY_GROWTH=1 \ - -s NO_DYNAMIC_EXECUTION=2 \ -s EXPORTED_FUNCTIONS=\"['_main']\" \ ") diff --git a/src/cpp/context_two.cpp b/src/cpp/context_two.cpp index fa9c1afa49..7cf183fe8a 100644 --- a/src/cpp/context_two.cpp +++ b/src/cpp/context_two.cpp @@ -264,6 +264,14 @@ t_ctx2::get_data(t_tvidx start_row, t_tvidx end_row, t_tvidx start_col, t_tvidx return retval; } + +void +t_ctx2::column_sort_by(const t_sortsvec& sortby) { + PSP_TRACE_SENTINEL(); + PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); + m_ctraversal->sort_by(m_config, sortby, *(ctree().get())); +} + void t_ctx2::sort_by(const t_sortsvec& sortby) { PSP_TRACE_SENTINEL(); diff --git a/src/cpp/main.cpp b/src/cpp/main.cpp index 3e73283ded..5bedee6323 100644 --- a/src/cpp/main.cpp +++ b/src/cpp/main.cpp @@ -614,12 +614,12 @@ _fill_col(val dcol, t_col_sptr col, t_bool is_arrow) { * */ void -_fill_data(t_table_sptr tbl, t_svec ocolnames, val j_data, std::vector odt, +_fill_data(t_table& tbl, t_svec ocolnames, val j_data, std::vector odt, t_uint32 offset, t_bool is_arrow) { std::vector data_cols = vecFromJSArray(j_data); for (auto cidx = 0; cidx < ocolnames.size(); ++cidx) { auto name = ocolnames[cidx]; - auto col = tbl->get_column(name); + auto col = tbl.get_column(name); auto col_type = odt[cidx]; auto dcol = data_cols[cidx]; @@ -676,59 +676,186 @@ _fill_data(t_table_sptr tbl, t_svec ocolnames, val j_data, std::vector * Public */ +void +set_column_nth(t_column* col, t_uindex idx, val value) { + + // Check if the value is a javascript null + if (value.isNull()) { + col->unset(idx); + return; + } + + switch (col->get_dtype()) { + case DTYPE_BOOL: { + col->set_nth(idx, value.as(), STATUS_VALID); + break; + } + case DTYPE_FLOAT64: { + col->set_nth(idx, value.as(), STATUS_VALID); + break; + } + case DTYPE_FLOAT32: { + col->set_nth(idx, value.as(), STATUS_VALID); + break; + } + case DTYPE_UINT32: { + col->set_nth(idx, value.as(), STATUS_VALID); + break; + } + case DTYPE_UINT64: { + col->set_nth(idx, value.as(), STATUS_VALID); + break; + } + case DTYPE_INT32: { + col->set_nth(idx, value.as(), STATUS_VALID); + break; + } + case DTYPE_INT64: { + col->set_nth(idx, value.as(), STATUS_VALID); + break; + } + case DTYPE_STR: { + std::wstring welem = value.as(); + + std::wstring_convert converter; + std::string elem = converter.to_bytes(welem); + col->set_nth(idx, elem, STATUS_VALID); + break; + } + case DTYPE_DATE: { + col->set_nth(idx, jsdate_to_t_date(value), STATUS_VALID); + break; + } + case DTYPE_TIME: { + col->set_nth( + idx, static_cast(value.as()), STATUS_VALID); + break; + } + case DTYPE_UINT8: + case DTYPE_UINT16: + case DTYPE_INT8: + case DTYPE_INT16: + default: { + // Other types not implemented + } + } +} + /** - * Create a populated table. + * Helper function for computed columns * * Params * ------ - * j_colnames - a JS Array of column names. - * j_dtypes - a JS Array of column types. - * j_data - a JS Array of JS Array columns. + * * * Returns * ------- - * a populated table. + * */ -t_table_sptr -make_table(t_uint32 size, val j_colnames, val j_dtypes, val j_data, t_uint32 offset, - t_uint32 limit, t_str index, t_bool is_arrow, t_bool is_delete) { - // Create the input and port schemas - t_svec colnames = vecFromJSArray(j_colnames); - t_dtypevec dtypes = vecFromJSArray(j_dtypes); +void +table_add_computed_column(t_table& table, val computed_defs) { + auto vcomputed_defs = vecFromJSArray(computed_defs); + for (auto i = 0; i < vcomputed_defs.size(); ++i) { + val coldef = vcomputed_defs[i]; + t_str name = coldef["column"].as(); + val inputs = coldef["inputs"]; + val func = coldef["func"]; + val type = coldef["type"]; + + t_str stype; + + if (type.isUndefined()) { + stype = "string"; + } else { + stype = type.as(); + } - // Create the table - // TODO assert size > 0 - auto tbl = std::make_shared(t_schema(colnames, dtypes)); - tbl->init(); - tbl->extend(size); + t_dtype dtype; + if (stype == "integer") { + dtype = DTYPE_INT32; + } else if (stype == "float") { + dtype = DTYPE_FLOAT64; + } else if (stype == "boolean") { + dtype = DTYPE_BOOL; + } else if (stype == "date") { + dtype = DTYPE_DATE; + } else if (stype == "datetime") { + dtype = DTYPE_TIME; + } else { + dtype = DTYPE_STR; + } - _fill_data(tbl, colnames, j_data, dtypes, offset, is_arrow); + // Get list of input column names + auto icol_names = vecFromJSArray(inputs); - // Set up pkey and op columns - if (is_delete) { - auto op_col = tbl->add_column("psp_op", DTYPE_UINT8, false); - op_col->raw_fill(OP_DELETE); - } else { - auto op_col = tbl->add_column("psp_op", DTYPE_UINT8, false); - op_col->raw_fill(OP_INSERT); - } + // Get t_column* for all input columns + t_colcptrvec icols; + for (const auto& cc : icol_names) { + icols.push_back(table._get_column(cc)); + } - if (index == "") { - // If user doesn't specify an column to use as the pkey index, just use - // row number - auto key_col = tbl->add_column("psp_pkey", DTYPE_INT32, true); - auto okey_col = tbl->add_column("psp_okey", DTYPE_INT32, true); + int arity = icols.size(); - for (auto ridx = 0; ridx < tbl->size(); ++ridx) { - key_col->set_nth(ridx, (ridx + offset) % limit); - okey_col->set_nth(ridx, (ridx + offset) % limit); + // Add new column + t_column* out = table.add_column(name, dtype, true); + + val i1 = val::undefined(), i2 = val::undefined(), i3 = val::undefined(), + i4 = val::undefined(); + + t_uindex size = table.size(); + for (t_uindex ridx = 0; ridx < size; ++ridx) { + val value = val::undefined(); + + switch (arity) { + case 0: { + value = func(); + break; + } + case 1: { + i1 = scalar_to_val(icols[0]->get_scalar(ridx)); + if (!i1.isNull()) { + value = func(i1); + } + break; + } + case 2: { + i1 = scalar_to_val(icols[0]->get_scalar(ridx)); + i2 = scalar_to_val(icols[1]->get_scalar(ridx)); + if (!i1.isNull() && !i2.isNull()) { + value = func(i1, i2); + } + break; + } + case 3: { + i1 = scalar_to_val(icols[0]->get_scalar(ridx)); + i2 = scalar_to_val(icols[1]->get_scalar(ridx)); + i3 = scalar_to_val(icols[2]->get_scalar(ridx)); + if (!i1.isNull() && !i2.isNull() && !i3.isNull()) { + value = func(i1, i2, i3); + } + break; + } + case 4: { + i1 = scalar_to_val(icols[0]->get_scalar(ridx)); + i2 = scalar_to_val(icols[1]->get_scalar(ridx)); + i3 = scalar_to_val(icols[2]->get_scalar(ridx)); + i4 = scalar_to_val(icols[3]->get_scalar(ridx)); + if (!i1.isNull() && !i2.isNull() && !i3.isNull() && !i4.isNull()) { + value = func(i1, i2, i3, i4); + } + break; + } + default: { + // Don't handle other arity values + break; + } + } + + if (!value.isUndefined()) { + set_column_nth(out, ridx, value); + } } - } else { - tbl->clone_column(index, "psp_pkey"); - tbl->clone_column(index, "psp_okey"); } - - return tbl; } /** @@ -744,8 +871,8 @@ make_table(t_uint32 size, val j_colnames, val j_dtypes, val j_data, t_uint32 off * A gnode. */ t_gnode_sptr -make_gnode(t_table_sptr table) { - auto iscm = table->get_schema(); +make_gnode(const t_table& table) { + auto iscm = table.get_schema(); t_svec ocolnames(iscm.columns()); t_dtypevec odt(iscm.types()); @@ -771,6 +898,88 @@ make_gnode(t_table_sptr table) { return gnode; } +/** + * Create a populated table. + * + * Params + * ------ + * chunk - a JS object containing parsed data and associated metadata + * offset + * limit + * index + * is_delete - sets the table operation + * + * Returns + * ------- + * a populated table. + */ +t_gnode_sptr +make_table(t_pool* pool, val gnode, val chunk, val computed, t_uint32 offset, t_uint32 limit, + t_str index, t_bool is_delete) { + t_uint32 size; + if (!chunk["row_count"].isUndefined()) { + size = chunk["row_count"].as(); + } else if (!chunk["cdata"][0]["length"].isUndefined()) { + size = chunk["cdata"][0]["length"].as(); + } else { + size = 0; + } + + // Create the input and port schemas + t_svec colnames = vecFromJSArray(chunk["names"]); + t_dtypevec dtypes = vecFromJSArray(chunk["types"]); + + // Create the table + // TODO assert size > 0 + t_table tbl(t_schema(colnames, dtypes)); + tbl.init(); + tbl.extend(size); + + _fill_data(tbl, colnames, chunk["cdata"], dtypes, offset, chunk["is_arrow"].as()); + + // Set up pkey and op columns + if (is_delete) { + auto op_col = tbl.add_column("psp_op", DTYPE_UINT8, false); + op_col->raw_fill(OP_DELETE); + } else { + auto op_col = tbl.add_column("psp_op", DTYPE_UINT8, false); + op_col->raw_fill(OP_INSERT); + } + + if (index == "") { + // If user doesn't specify an column to use as the pkey index, just use + // row number + auto key_col = tbl.add_column("psp_pkey", DTYPE_INT32, true); + auto okey_col = tbl.add_column("psp_okey", DTYPE_INT32, true); + + for (auto ridx = 0; ridx < tbl.size(); ++ridx) { + key_col->set_nth(ridx, (ridx + offset) % limit); + okey_col->set_nth(ridx, (ridx + offset) % limit); + } + } else { + tbl.clone_column(index, "psp_pkey"); + tbl.clone_column(index, "psp_okey"); + } + + t_gnode_sptr new_gnode; + + if (gnode.isUndefined()) { + new_gnode = make_gnode(tbl); + pool->register_gnode(new_gnode.get()); + } else { + new_gnode = gnode.as(); + } + + if (!computed.isUndefined()) { + table_add_computed_column(tbl, computed); + } + + pool->send(new_gnode->get_id(), 0, tbl); + pool->_process(); + + return new_gnode; +} + /** * Copies the internal table from a gnode * @@ -779,12 +988,17 @@ make_gnode(t_table_sptr table) { * * Returns * ------- - * A table. + * A gnode. */ -t_table_sptr -clone_gnode_table(t_gnode_sptr gnode) { - // This creates a copy of the table - return t_table_sptr(gnode->_get_pkeyed_table()); +t_gnode_sptr +clone_gnode_table(t_pool* pool, t_gnode_sptr gnode, val computed) { + t_table* tbl = gnode->_get_pkeyed_table(); + table_add_computed_column(*tbl, computed); + t_gnode_sptr new_gnode = make_gnode(*tbl); + pool->register_gnode(new_gnode.get()); + pool->send(new_gnode->get_id(), 0, *tbl); + pool->_process(); + return new_gnode; } /** @@ -852,30 +1066,28 @@ make_context_one(t_schema schema, val j_pivots, t_filter_op combiner, val j_filt */ t_ctx2_sptr make_context_two(t_schema schema, val j_rpivots, val j_cpivots, t_filter_op combiner, - val j_filters, val j_aggs, val j_sortby) { + val j_filters, val j_aggs, bool show_totals) { auto fvec = _get_fterms(schema, j_filters); auto aggspecs = _get_aggspecs(j_aggs); auto rpivots = vecFromJSArray(j_rpivots); auto cpivots = vecFromJSArray(j_cpivots); - auto svec = _get_sort(j_sortby); + t_totals total = show_totals ? TOTALS_BEFORE : TOTALS_HIDDEN; - auto cfg = t_config(rpivots, cpivots, aggspecs, TOTALS_HIDDEN, combiner, fvec); + auto cfg = t_config(rpivots, cpivots, aggspecs, total, combiner, fvec); auto ctx2 = std::make_shared(schema, cfg); ctx2->init(); ctx2->set_deltas_enabled(true); - if (svec.size() > 0) { - ctx2->sort_by(svec); - } return ctx2; } void -sort(t_ctx2_sptr ctx2, val j_sortby) { +sort(t_ctx2_sptr ctx2, val j_sortby, val j_column_sortby) { auto svec = _get_sort(j_sortby); if (svec.size() > 0) { ctx2->sort_by(svec); } + ctx2->column_sort_by(_get_sort(j_column_sortby)); } val @@ -888,158 +1100,6 @@ get_column_data(t_table_sptr table, t_str colname) { return arr; } -void -set_column_nth(t_column* col, t_uindex idx, val value) { - - // Check if the value is a javascript null - if (value.isNull()) { - col->unset(idx); - return; - } - - switch (col->get_dtype()) { - case DTYPE_BOOL: { - col->set_nth(idx, value.as(), STATUS_VALID); - break; - } - case DTYPE_FLOAT64: { - col->set_nth(idx, value.as(), STATUS_VALID); - break; - } - case DTYPE_FLOAT32: { - col->set_nth(idx, value.as(), STATUS_VALID); - break; - } - case DTYPE_UINT32: { - col->set_nth(idx, value.as(), STATUS_VALID); - break; - } - case DTYPE_UINT64: { - col->set_nth(idx, value.as(), STATUS_VALID); - break; - } - case DTYPE_INT32: { - col->set_nth(idx, value.as(), STATUS_VALID); - break; - } - case DTYPE_INT64: { - col->set_nth(idx, value.as(), STATUS_VALID); - break; - } - case DTYPE_STR: { - std::wstring welem = value.as(); - - std::wstring_convert converter; - std::string elem = converter.to_bytes(welem); - col->set_nth(idx, elem, STATUS_VALID); - break; - } - case DTYPE_DATE: { - col->set_nth(idx, jsdate_to_t_date(value), STATUS_VALID); - break; - } - case DTYPE_TIME: { - col->set_nth( - idx, static_cast(value.as()), STATUS_VALID); - break; - } - case DTYPE_UINT8: - case DTYPE_UINT16: - case DTYPE_INT8: - case DTYPE_INT16: - default: { - // Other types not implemented - } - } -} - -/** - * Helper function for computed columns - * - * Params - * ------ - * - * - * Returns - * ------- - * - */ - -void -table_add_computed_column(t_table_sptr table, t_str name, t_dtype dtype, val func, val inputs) { - - // Get list of input column names - auto icol_names = vecFromJSArray(inputs); - - // Get t_column* for all input columns - t_colcptrvec icols; - for (const auto& cc : icol_names) { - icols.push_back(table->_get_column(cc)); - } - - int arity = icols.size(); - - // Add new column - t_column* out = table->add_column(name, dtype, true); - - val i1 = val::undefined(), i2 = val::undefined(), i3 = val::undefined(), - i4 = val::undefined(); - - t_uindex size = table->size(); - for (t_uindex ridx = 0; ridx < size; ++ridx) { - val value = val::undefined(); - - switch (arity) { - case 0: { - value = func(); - break; - } - case 1: { - i1 = scalar_to_val(icols[0]->get_scalar(ridx)); - if (!i1.isNull()) { - value = func(i1); - } - break; - } - case 2: { - i1 = scalar_to_val(icols[0]->get_scalar(ridx)); - i2 = scalar_to_val(icols[1]->get_scalar(ridx)); - if (!i1.isNull() && !i2.isNull()) { - value = func(i1, i2); - } - break; - } - case 3: { - i1 = scalar_to_val(icols[0]->get_scalar(ridx)); - i2 = scalar_to_val(icols[1]->get_scalar(ridx)); - i3 = scalar_to_val(icols[2]->get_scalar(ridx)); - if (!i1.isNull() && !i2.isNull() && !i3.isNull()) { - value = func(i1, i2, i3); - } - break; - } - case 4: { - i1 = scalar_to_val(icols[0]->get_scalar(ridx)); - i2 = scalar_to_val(icols[1]->get_scalar(ridx)); - i3 = scalar_to_val(icols[2]->get_scalar(ridx)); - i4 = scalar_to_val(icols[3]->get_scalar(ridx)); - if (!i1.isNull() && !i2.isNull() && !i3.isNull() && !i4.isNull()) { - value = func(i1, i2, i3, i4); - } - break; - } - default: { - // Don't handle other arity values - break; - } - } - - if (!value.isUndefined()) { - set_column_nth(out, ridx, value); - } - } -} - /** * * @@ -1062,6 +1122,36 @@ get_data(T ctx, t_uint32 start_row, t_uint32 end_row, t_uint32 start_col, t_uint return arr; } +val +get_data_two_skip_headers(t_ctx2_sptr ctx, t_uint32 depth, t_uint32 start_row, t_uint32 end_row, + t_uint32 start_col, t_uint32 end_col) { + auto col_length = ctx->unity_get_column_count(); + std::vector col_nums; + col_nums.push_back(0); + for (t_uindex i = 0; i < col_length; ++i) { + if (ctx->unity_get_column_path(i + 1).size() == depth) { + col_nums.push_back(i + 1); + } + } + col_nums = std::vector(col_nums.begin() + start_col, + col_nums.begin() + std::min(end_col, (t_uint32)col_nums.size())); + auto slice = ctx->get_data(start_row, end_row, col_nums.front(), col_nums.back() + 1); + val arr = val::array(); + t_uindex i = 0; + auto iter = slice.begin(); + while (iter != slice.end()) { + t_uindex prev = col_nums.front(); + for (auto idx = col_nums.begin(); idx != col_nums.end(); idx++, i++) { + t_uindex col_num = *idx; + iter += col_num - prev; + prev = col_num; + arr.set(i, scalar_to_val(*iter)); + } + if (iter != slice.end()) + iter++; + } + return arr; +} /** * Main */ @@ -1222,7 +1312,7 @@ EMSCRIPTEN_BINDINGS(perspective) { .function("unity_init_load_step_end", &t_ctx2::unity_init_load_step_end); class_("t_pool") - .constructor() + .constructor<>() .smart_ptr>("shared_ptr") .function("register_gnode", &t_pool::register_gnode, allow_raw_pointers()) .function("process", &t_pool::_process) @@ -1358,9 +1448,9 @@ EMSCRIPTEN_BINDINGS(perspective) { .value("TOTALS_AFTER", TOTALS_AFTER); function("sort", &sort); - function("make_table", &make_table); + function("make_table", &make_table, allow_raw_pointers()); function("make_gnode", &make_gnode); - function("clone_gnode_table", &clone_gnode_table); + function("clone_gnode_table", &clone_gnode_table, allow_raw_pointers()); function("make_context_zero", &make_context_zero); function("make_context_one", &make_context_one); function("make_context_two", &make_context_two); @@ -1371,6 +1461,7 @@ EMSCRIPTEN_BINDINGS(perspective) { function("get_data_zero", &get_data); function("get_data_one", &get_data); function("get_data_two", &get_data); + function("get_data_two_skip_headers", &get_data_two_skip_headers); function("col_to_js_typed_array_zero", &col_to_js_typed_array); function("col_to_js_typed_array_one", &col_to_js_typed_array); function("col_to_js_typed_array_two", &col_to_js_typed_array); diff --git a/src/cpp/pool.cpp b/src/cpp/pool.cpp index 763f10d31b..72f4199fe5 100644 --- a/src/cpp/pool.cpp +++ b/src/cpp/pool.cpp @@ -25,9 +25,16 @@ t_updctx::t_updctx(t_uindex gnode_id, const t_str& ctx) , m_ctx(ctx) {} #ifdef PSP_ENABLE_WASM -t_pool::t_pool(emscripten::val update_delegate) +emscripten::val +empty_callback() { + emscripten::val callback = emscripten::val::global("Object").new_(); + callback.set("_update_callback", emscripten::val::global("Function").new_()); + return callback; +} + +t_pool::t_pool() : m_sleep(0) - , m_update_delegate(update_delegate) + , m_update_delegate(empty_callback()) , m_has_python_dep(false) { m_run.clear(); } diff --git a/src/cpp/sparse_tree.cpp b/src/cpp/sparse_tree.cpp index 74ea70603a..324b3505ca 100644 --- a/src/cpp/sparse_tree.cpp +++ b/src/cpp/sparse_tree.cpp @@ -909,29 +909,34 @@ t_stree::update_agg_table(t_uindex nidx, t_agg_update_info& info, t_uindex src_r case AGGTYPE_WEIGHTED_MEAN: { auto pkeys = get_pkeys(nidx); - t_f64vec values; - t_f64vec weights; + t_float64 nr = 0; + t_float64 dr = 0; + t_tscalvec values; + t_tscalvec weights; gstate.read_column(spec.get_dependencies()[0].name(), pkeys, values); - gstate.read_column(spec.get_dependencies()[1].name(), pkeys, weights); - t_float64 init_value = 0.0; + auto weights_it = weights.begin(); + auto values_it = values.begin(); - auto nr = std::inner_product( - weights.begin(), weights.end(), values.begin(), init_value); - - auto dr = std::accumulate(weights.begin(), weights.end(), t_float64(0)); + for (; weights_it != weights.end() && values_it != values.end(); + ++weights_it, ++values_it) { + if (weights_it->is_valid() && values_it->is_valid() && !weights_it->is_nan() + && !values_it->is_nan()) { + nr += weights_it->to_double() * values_it->to_double(); + dr += weights_it->to_double(); + } + } t_f64pair* dst_pair = dst->get_nth(dst_ridx); - old_value.set(dst_pair->first / dst_pair->second); dst_pair->first = nr; dst_pair->second = dr; - dst->set_valid(dst_ridx, true); - + bool valid = (dr != 0); + dst->set_valid(dst_ridx, valid); new_value.set(nr / dr); } break; case AGGTYPE_UNIQUE: { diff --git a/src/include/perspective/context_two.h b/src/include/perspective/context_two.h index 08d15ce79a..94f18f9e4a 100644 --- a/src/include/perspective/context_two.h +++ b/src/include/perspective/context_two.h @@ -44,6 +44,8 @@ class PERSPECTIVE_EXPORT t_ctx2 : public t_ctxbase { t_aggspecvec get_aggregates() const; + void column_sort_by(const t_sortsvec& sortby); + void set_depth(t_header header, t_depth depth); using t_ctxbase::get_data; diff --git a/src/include/perspective/pool.h b/src/include/perspective/pool.h index ae6a5242b2..8394434c1f 100644 --- a/src/include/perspective/pool.h +++ b/src/include/perspective/pool.h @@ -38,17 +38,14 @@ class PERSPECTIVE_EXPORT t_pool { typedef std::pair t_ctx_id; public: + t_pool(); + t_uindex register_gnode(t_gnode* node); #ifdef PSP_ENABLE_WASM - t_pool(emscripten::val update_delegate); void set_update_delegate(emscripten::val ud); - t_uindex register_gnode(t_gnode* node); void register_context(t_uindex gnode_id, const t_str& name, t_ctx_type type, t_int32 ptr); void py_notify_userspace(); #else - t_pool(); - t_uindex register_gnode(t_gnode* node); void register_context(t_uindex gnode_id, const t_str& name, t_ctx_type type, t_int64 ptr); - void set_update_delegate(); void py_notify_userspace(); #endif t_pool(const t_pool& p) = delete;