Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial support for loading Apache Arrow data #11

Merged
merged 16 commits into from
Jan 26, 2018
Merged

Initial support for loading Apache Arrow data #11

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
c16c1c9
Initial support for Arrow (https://arrow.apache.org/)
nmichaud Jan 12, 2018
fb989ca
In place loading of arrow arrays into perspective
nmichaud Jan 12, 2018
5faf2f2
Don't compile with dynamic executions.
nmichaud Jan 20, 2018
f3469fb
Arrow uses ES6's Symbol, need polyfill for IE
nmichaud Jan 21, 2018
5d43c4d
Add arrow example
nmichaud Jan 21, 2018
7999cdf
Fixed name mangling issue in arrow build.
texodus Jan 22, 2018
f5f3843
Make arrow detection implicit
nmichaud Jan 22, 2018
a8d5264
Clean up differentiation of arrow/non-arrow data
nmichaud Jan 22, 2018
9f36b2e
Fixed webworker bug. Added unit test for Arrow constructor.
texodus Jan 24, 2018
7376186
Convert additional types to js vals
nmichaud Jan 26, 2018
97f0c25
Better dispatch on arrow datatypes for numeric data
nmichaud Jan 26, 2018
d40f820
Unpack arrow bitmap column into psp bool
nmichaud Jan 26, 2018
3be64db
Don't copy more than col->size() data from arrow
nmichaud Jan 26, 2018
235352d
Add test case with all supported arrow datatypes
nmichaud Jan 26, 2018
c29aaed
Refactor to support updates using arrow data
nmichaud Jan 26, 2018
7abd9f0
Rearrange scalar to val output
nmichaud Jan 26, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Refactor to support updates using arrow data
  • Loading branch information
@nmichaud
nmichaud committed Jan 26, 2018
commit c29aaed44ea92403b5d8b880895b71c53d65156e
155 changes: 96 additions & 59 deletions packages/perspective/src/js/perspective.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -265,6 +265,81 @@ function parse_data(data, names, types) {
};
}

/**
* Converts arrow data into a canonical representation for
* interfacing with perspective.
*
* @private
* @param {object} data Array buffer
* @returns An object with 3 properties:
* names - the column names.
* types - the column t_dtypes.
*/
function load_arrow_buffer(data, names, types) {

// TODO Need to validate that the names/types passed in match those in the buffer

var arrow = Arrow.Table.from([new Uint8Array(data)]);

names = [];
types = [];
let cdata = [];
for (let column of arrow.columns) {
switch (column.type) {
case 'Utf8':
types.push(__MODULE__.t_dtype.DTYPE_STR);
break;
case 'FloatingPoint':
if (column instanceof Arrow.Float64Vector) {
types.push(__MODULE__.t_dtype.DTYPE_FLOAT64);
}
else if (column instanceof Arrow.Float32Vector) {
types.push(__MODULE__.t_dtype.DTYPE_FLOAT32);
}
break;
case 'Int':
if (column instanceof Arrow.Int64Vector) {
types.push(__MODULE__.t_dtype.DTYPE_INT64);
}
else if (column instanceof Arrow.Int32Vector) {
types.push(__MODULE__.t_dtype.DTYPE_INT32);
}
else if (column instanceof Arrow.Int16Vector) {
types.push(__MODULE__.t_dtype.DTYPE_INT16);
}
else if (column instanceof Arrow.Int8Vector) {
types.push(__MODULE__.t_dtype.DTYPE_INT8);
}
break;
case 'Bool':
types.push(__MODULE__.t_dtype.DTYPE_BOOL);
break;
case 'Timestamp':
types.push(__MODULE__.t_dtype.DTYPE_TIME);
break;
default:
continue;
break;
}
switch (column.type) {
case 'Utf8':
cdata.push(column);
break;
default:
cdata.push(column.slice());
break;
}
names.push(column.name);
}

return {
row_count: arrow.length,
names: names,
types: types,
cdata: cdata
};
}

/******************************************************************************
*
* View
Expand Down Expand Up @@ -843,11 +918,26 @@ table.prototype.view = function(config) {
* @see {@link table}
*/
table.prototype.update = function (data) {
let {row_count, names, types, cdata} = parse_data(data, this.columns(), this.gnode.get_tblschema().types());
this.initialized = true;
let tbl = __MODULE__.make_table(row_count || 0, names, types, cdata, this.gnode.get_table().size(), this.index, this.tindex);
__MODULE__.fill(this.id, tbl, this.gnode, this.pool);
tbl.delete();
let pdata;

if (data instanceof ArrayBuffer) {
pdata = load_arrow_buffer(data, this.columns(), this.gnode.get_tblschema().types());
}
else {
pdata = parse_data(data, this.columns(), this.gnode.get_tblschema().types());
}

let tbl;
try{
tbl = __MODULE__.make_table(pdata.row_count || 0, pdata.names, pdata.types, pdata.cdata, this.gnode.get_table().size(), this.index, this.tindex);
__MODULE__.fill(this.id, tbl, this.gnode, this.pool);
this.initialized = true;
} catch (e) {
} finally {
if (tbl) {
tbl.delete();
}
}
}

/**
Expand Down Expand Up @@ -1033,60 +1123,7 @@ const perspective = {

if (data instanceof ArrayBuffer) {
// Arrow data
var arr = new Uint8Array(data)
var arrow = Arrow.Table.from([arr]);

let names = [];
let types = [];
let cdata = [];
for (let column of arrow.columns) {
switch (column.type) {
case 'Utf8':
types.push(__MODULE__.t_dtype.DTYPE_STR);
break;
case 'FloatingPoint':
if (column instanceof Arrow.Float64Vector) {
types.push(__MODULE__.t_dtype.DTYPE_FLOAT64);
}
else if (column instanceof Arrow.Float32Vector) {
types.push(__MODULE__.t_dtype.DTYPE_FLOAT32);
}
break;
case 'Int':
if (column instanceof Arrow.Int64Vector) {
types.push(__MODULE__.t_dtype.DTYPE_INT64);
}
else if (column instanceof Arrow.Int32Vector) {
types.push(__MODULE__.t_dtype.DTYPE_INT32);
}
else if (column instanceof Arrow.Int16Vector) {
types.push(__MODULE__.t_dtype.DTYPE_INT16);
}
else if (column instanceof Arrow.Int8Vector) {
types.push(__MODULE__.t_dtype.DTYPE_INT8);
}
break;
case 'Bool':
types.push(__MODULE__.t_dtype.DTYPE_BOOL);
break;
case 'Timestamp':
types.push(__MODULE__.t_dtype.DTYPE_TIME);
break;
default:
continue;
break;
}
switch (column.type) {
case 'Utf8':
cdata.push(column);
break;
default:
cdata.push(column.slice());
break;
}
names.push(column.name);
}
pdata = {row_count: arrow.length, names: names, types: types, cdata: cdata};
pdata = load_arrow_buffer(data);
} else {
if (typeof data === "string") {
if (data[0] === ",") {
Expand Down
18 changes: 18 additions & 0 deletions packages/perspective/test/js/updates.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@
import _ from "underscore";
import papaparse from "papaparse";

import arrow from "../arrow/test.arrow";

var data = [
{'x': 1, 'y':'a', 'z': true},
{'x': 2, 'y':'b', 'z': false},
Expand All @@ -30,6 +32,14 @@ var data_2 = [
{'x': 6, 'y':'h', 'z': true},
];

var arrow_result = [
{"f32": 1.5, "f64": 1.5, "i64": 1, "i32": 1, "i16": 1, "i8": 1, "bool": true, "char": "a", "dict": "a", "datetime": +(new Date("2018-01-25"))},
{"f32": 2.5, "f64": 2.5, "i64": 2, "i32": 2, "i16": 2, "i8": 2, "bool": false, "char": "b", "dict": "b", "datetime": +(new Date("2018-01-26"))},
{"f32": 3.5, "f64": 3.5, "i64": 3, "i32": 3, "i16": 3, "i8": 3, "bool": true, "char": "c", "dict": "c", "datetime": +(new Date("2018-01-27"))},
{"f32": 4.5, "f64": 4.5, "i64": 4, "i32": 4, "i16": 4, "i8": 4, "bool": false, "char": "d", "dict": "d", "datetime": +(new Date("2018-01-28"))},
{"f32": 5.5, "f64": 5.5, "i64": 5, "i32": 5, "i16": 5, "i8": 5, "bool": true, "char": "d", "dict": "d", "datetime": +(new Date("2018-01-29"))}
];

module.exports = (perspective) => {

describe("Updates", function() {
Expand Down Expand Up @@ -59,6 +69,14 @@ module.exports = (perspective) => {
expect(data).toEqual(result);
});

it("Arrow `update()`s", async function () {
var table = perspective.table(arrow);
table.update(arrow);
var view = table.view();
let result = await view.to_json();
expect(arrow_result.concat(arrow_result)).toEqual(result);
});

});

describe("Notifications", function() {
Expand Down