Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Initial support for loading Apache Arrow data #11

Merged
merged 16 commits into from
Jan 26, 2018
Merged

Initial support for loading Apache Arrow data #11

Changes from 1 commit
Commits
Show all changes
16 commits
Select commit Hold shift + click to select a range
c16c1c9
Initial support for Arrow (https://arrow.apache.org/)
nmichaud Jan 12, 2018
fb989ca
In place loading of arrow arrays into perspective
nmichaud Jan 12, 2018
5faf2f2
Don't compile with dynamic executions.
nmichaud Jan 20, 2018
f3469fb
Arrow uses ES6's Symbol, need polyfill for IE
nmichaud Jan 21, 2018
5d43c4d
Add arrow example
nmichaud Jan 21, 2018
7999cdf
Fixed name mangling issue in arrow build.
texodus Jan 22, 2018
f5f3843
Make arrow detection implicit
nmichaud Jan 22, 2018
a8d5264
Clean up differentiation of arrow/non-arrow data
nmichaud Jan 22, 2018
9f36b2e
Fixed webworker bug. Added unit test for Arrow constructor.
texodus Jan 24, 2018
7376186
Convert additional types to js vals
nmichaud Jan 26, 2018
97f0c25
Better dispatch on arrow datatypes for numeric data
nmichaud Jan 26, 2018
d40f820
Unpack arrow bitmap column into psp bool
nmichaud Jan 26, 2018
3be64db
Don't copy more than col->size() data from arrow
nmichaud Jan 26, 2018
235352d
Add test case with all supported arrow datatypes
nmichaud Jan 26, 2018
c29aaed
Refactor to support updates using arrow data
nmichaud Jan 26, 2018
7abd9f0
Rearrange scalar to val output
nmichaud Jan 26, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Clean up differentiation of arrow/non-arrow data
  • Loading branch information
@nmichaud
nmichaud committed Jan 22, 2018
commit a8d5264307b5d2392da4880b4ee6fdc93875b3d5
23 changes: 11 additions & 12 deletions packages/perspective/src/cpp/main.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,7 +143,6 @@ _get_aggspecs(val j_aggs)
*/


template<typename T>
void
vecFromTypedArray(const val &typedArray, void* data, t_int32 length) {
val memory = val::module_property("buffer");
Expand All @@ -158,9 +157,9 @@ _fill_col(val dcol, t_col_sptr col, t_col_sptr key_col, bool fill_index)
{
t_int32 nrows = dcol["length"].as<t_int32>();

if (internal::typeSupportsMemoryView<T>()) {
if (!dcol["buffer"].isUndefined()) {
t_lstore* lstore = col->_get_data_lstore();
vecFromTypedArray<T>(dcol, lstore->get_ptr(0), nrows);
vecFromTypedArray(dcol, lstore->get_ptr(0), nrows);
col->valid_raw_fill(true);
} else {
for (auto i = 0; i < nrows; ++i)
Expand All @@ -181,9 +180,9 @@ _fill_col<t_int64>(val dcol, t_col_sptr col, t_col_sptr key_col, bool fill_index
{
t_int32 nrows = dcol["length"].as<t_int32>();

if (internal::typeSupportsMemoryView<t_int32>()) {
if (dcol["constructor"]["name"].as<t_str>() == "Int32Array") {
t_lstore* lstore = col->_get_data_lstore();
vecFromTypedArray<t_int64>(dcol, lstore->get_ptr(0), nrows);
vecFromTypedArray(dcol, lstore->get_ptr(0), nrows);
col->valid_raw_fill(true);
} else {
throw std::logic_error("Unreachable");
Expand All @@ -196,9 +195,9 @@ _fill_col<t_time>(val dcol, t_col_sptr col, t_col_sptr key_col, bool fill_index)
{
t_int32 nrows = dcol["length"].as<t_int32>();

if (internal::typeSupportsMemoryView<t_int32>()) {
if (dcol["constructor"]["name"].as<t_str>() == "Int32Array") {
t_lstore* lstore = col->_get_data_lstore();
vecFromTypedArray<t_int64>(dcol, lstore->get_ptr(0), nrows);
vecFromTypedArray(dcol, lstore->get_ptr(0), nrows);
col->valid_raw_fill(true);
} else {
for (auto i = 0; i < nrows; ++i)
Expand Down Expand Up @@ -237,22 +236,22 @@ _fill_col_dict(t_uint32 nrows, val dcol, val vkeys, t_col_sptr col)
std::vector<T> keys;
keys.reserve(ksize);
keys.resize(ksize);
vecFromTypedArray<T>(vkeys, keys.data(), ksize);
vecFromTypedArray(vkeys, keys.data(), ksize);

val values = dcol["data"]["values"];
val vdata = values["data"];
t_int32 vsize = vdata["length"].as<t_int32>();
std::vector<t_uchar> data;
data.reserve(vsize);
data.resize(vsize);
vecFromTypedArray<t_uchar>(vdata, data.data(), vsize);
vecFromTypedArray(vdata, data.data(), vsize);

val voffsets = values["offsets"];
t_int32 osize = voffsets["length"].as<t_int32>();
std::vector<t_int32> offsets;
offsets.reserve(osize);
offsets.resize(osize);
vecFromTypedArray<t_int32>(voffsets, offsets.data(), osize);
vecFromTypedArray(voffsets, offsets.data(), osize);

t_str elem;

Expand Down Expand Up @@ -303,14 +302,14 @@ _fill_col<std::string>(val dcol, t_col_sptr col, t_col_sptr key_col, bool fill_i
std::vector<t_uint8> data;
data.reserve(vsize);
data.resize(vsize);
vecFromTypedArray<t_uint8>(vdata, data.data(), vsize);
vecFromTypedArray(vdata, data.data(), vsize);

val voffsets = values["offsets"];
t_int32 osize = voffsets["length"].as<t_int32>();
std::vector<t_int32> offsets;
offsets.reserve(osize);
offsets.resize(osize);
vecFromTypedArray<t_int32>(voffsets, offsets.data(), osize);
vecFromTypedArray(voffsets, offsets.data(), osize);

t_str elem;

Expand Down