diff --git a/CHANGELOG.md b/CHANGELOG.md index 804d63cbde..03694b9eb3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,33 @@ # Changelog +## [v1.1.0](https://github.com/finos/perspective/tree/HEAD) + +[Full Changelog](https://github.com/finos/perspective/compare/v1.0.8...HEAD) + +**Breaking changes:** + +- CSV output rewrite \(Arrow/C++\) [\#1692](https://github.com/finos/perspective/pull/1692) ([texodus](https://github.com/texodus)) + +**Implemented enhancements:** + +- Column style menu for `string` type columns [\#1691](https://github.com/finos/perspective/pull/1691) ([texodus](https://github.com/texodus)) + +**Fixed bugs:** + +- String field filter in server mode not work [\#1690](https://github.com/finos/perspective/issues/1690) +- CSV output with `row_pivots` \> 1 don't render correctly [\#1666](https://github.com/finos/perspective/issues/1666) +- Fix auto-reset when `HTMLPerspectiveViewerElement.load()` called twice [\#1695](https://github.com/finos/perspective/pull/1695) ([texodus](https://github.com/texodus)) + +**Closed issues:** + +- SyntaxError: Unexpected token '.' when using webpack PerspectivePlugin [\#1687](https://github.com/finos/perspective/issues/1687) +- installation error Ubuntu 20.04.3 LTS \(GNU/Linux 5.11.0-1021-oracle aarch64\) [\#1686](https://github.com/finos/perspective/issues/1686) + +**Merged pull requests:** + +- purge six dependency [\#1689](https://github.com/finos/perspective/pull/1689) ([timkpaine](https://github.com/timkpaine)) +- Reduce CI: Turn off branch builds, only build on PRs to master [\#1688](https://github.com/finos/perspective/pull/1688) ([timkpaine](https://github.com/timkpaine)) + ## [v1.0.8](https://github.com/finos/perspective/tree/HEAD) [Full Changelog](https://github.com/finos/perspective/compare/v1.0.7...HEAD) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4e8e851e07..7d5db23dcf 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,15 +1,29 @@ trigger: + batch: true branches: include: - - "*" # must quote since "*" is a YAML reserved character; we want a string + - master tags: include: - "v*" pr: + autoCancel: true branches: include: - - "*" # must quote since "*" is a YAML reserved character; we want a string + - master + drafts: true + paths: + exclude: + - AUTHORS + - CHANGELOG.md + - CONTRIBUTING.md + - LICENSE + - README.md + - binder/ + - docs/ + - examples/ + - python/perspective/README.md schedules: - cron: "0 0 * * 6" diff --git a/cmake/arrow/CMakeLists.txt b/cmake/arrow/CMakeLists.txt index 0f0ae9364d..00de71ab41 100644 --- a/cmake/arrow/CMakeLists.txt +++ b/cmake/arrow/CMakeLists.txt @@ -18,8 +18,7 @@ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin") endif() set(ARROW_SRCS - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/builder.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/pretty_print.cc + # Base ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/array/array_base.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/array/array_binary.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/array/array_decimal.cc @@ -39,9 +38,11 @@ set(ARROW_SRCS ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/array/diff.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/array/util.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/array/validate.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/builder.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/buffer.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/chunked_array.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compare.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/config.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/datum.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/device.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/extension_type.cc @@ -55,48 +56,40 @@ set(ARROW_SRCS ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/table.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/table_builder.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/tensor.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/tensor/coo_converter.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/tensor/csf_converter.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/tensor/csx_converter.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/type.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/visitor.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/converter.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/chunker.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/column_builder.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/column_decoder.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/options.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/parser.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/reader.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/filesystem/filesystem.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/filesystem/localfs.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/filesystem/mockfs.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/filesystem/path_util.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/filesystem/util_internal.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/json/options.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/json/chunked_builder.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/json/chunker.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/json/converter.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/json/parser.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/json/reader.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/c/bridge.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/io/buffered.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/io/caching.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/io/compressed.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/io/file.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/io/interfaces.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/io/memory.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/testing/util.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/io/slow.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/io/stdio.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/io/transform.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/async_util.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/basic_decimal.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/bit_block_counter.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/bit_run_reader.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/bit_util.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/bitmap.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/bitmap_builders.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/bitmap_ops.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/bpacking.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/cancel.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/compression.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/counting_semaphore.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/cpu_info.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/decimal.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/future.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/formatting.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/delimiting.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/formatting.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/future.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/int_util.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/io_util.cc - # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/iterator.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/logging.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/key_value_metadata.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/memory.cc @@ -104,41 +97,114 @@ set(ARROW_SRCS ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/string.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/string_builder.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/task_group.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/tdigest.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/thread_pool.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/time.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/trie.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/unreachable.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/uri.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/utf8.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/value_parsing.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/base64.cpp + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/datetime/tz.cpp + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/bignum.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/double-conversion.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/bignum-dtoa.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/fast-dtoa.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/cached-powers.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/fixed-dtoa.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/diy-fp.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/bignum.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/strtod.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/datetime/tz.cpp + # CSV + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/converter.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/chunker.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/column_builder.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/column_decoder.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/options.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/parser.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/reader.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/csv/writer.cc + # IPC ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/dictionary.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/feather.cc - # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/json_integration.cc - # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/json_internal.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/json_simple.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/message.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/metadata_internal.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/reader.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/options.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/writer.cc) + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/reader.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/ipc/writer.cc + # Compute + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/api_aggregate.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/api_scalar.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/api_vector.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/cast.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/aggregate_node.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/exec_plan.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/expression.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/filter_node.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/project_node.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/source_node.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/sink_node.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/order_by_impl.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/function.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/function_internal.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernel.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/registry.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/aggregate_basic.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/aggregate_mode.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/aggregate_quantile.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/aggregate_tdigest.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/aggregate_var_std.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/codegen_internal.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/hash_aggregate.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_arithmetic.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_boolean.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_cast_boolean.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_cast_dictionary.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_cast_internal.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_cast_nested.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_cast_numeric.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_cast_string.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_cast_temporal.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_compare.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_nested.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_set_lookup.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_string.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_temporal_binary.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_temporal_unary.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_validity.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/scalar_if_else.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/util_internal.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/vector_array_sort.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/vector_hash.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/vector_nested.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/vector_replace.cc + ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/vector_selection.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/vector_sort.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/kernels/row_encoder.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/union_node.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/key_hash.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/key_map.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/key_compare.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/key_encode.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/util.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/hash_join_dict.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/hash_join.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/hash_join_node.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/compute/exec/task_util.cc + ) if (PSP_PYTHON_BUILD) set(ARROW_SRCS ${ARROW_SRCS} - # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/datum.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/io/file.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/tensor/coo_converter.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/tensor/csf_converter.cc ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/tensor/csx_converter.cc - # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/formatting.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/time.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/bignum-dtoa.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/fast-dtoa.cc - ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/fixed-dtoa.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/util/time.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/bignum-dtoa.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/fast-dtoa.cc + # ${CMAKE_BINARY_DIR}/arrow-src/cpp/src/arrow/vendored/double-conversion/fixed-dtoa.cc ) if(WIN32) diff --git a/cmake/arrow/config.h b/cmake/arrow/config.h index 5de6f288cf..1d16dd021e 100644 --- a/cmake/arrow/config.h +++ b/cmake/arrow/config.h @@ -22,3 +22,30 @@ /* #undef DOUBLE_CONVERSION_HAS_CASE_INSENSIBILITY */ /* #undef GRPCPP_PP_INCLUDE */ + +#define ARROW_VERSION_STRING "6000001" + +#define ARROW_SO_VERSION "" +#define ARROW_FULL_SO_VERSION "" + +#define ARROW_CXX_COMPILER_ID "" +#define ARROW_CXX_COMPILER_VERSION "" +#define ARROW_CXX_COMPILER_FLAGS "" + +#define ARROW_GIT_ID "" +#define ARROW_GIT_DESCRIPTION "" + +#define ARROW_PACKAGE_KIND "" + +// #cmakedefine ARROW_COMPUTE +// #cmakedefine ARROW_CSV +// #cmakedefine ARROW_DATASET +// #cmakedefine ARROW_FILESYSTEM +// #cmakedefine ARROW_FLIGHT +// #cmakedefine ARROW_IPC +// #cmakedefine ARROW_JSON + +// #cmakedefine ARROW_S3 +// #cmakedefine ARROW_USE_NATIVE_INT128 + +// #cmakedefine GRPCPP_PP_INCLUDE \ No newline at end of file diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt index 0eae29f083..278dea98c0 100644 --- a/cpp/perspective/CMakeLists.txt +++ b/cpp/perspective/CMakeLists.txt @@ -241,10 +241,10 @@ endif() ####################### include_directories("${CMAKE_SOURCE_DIR}/src/include") -if(NOT WIN32) - set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") - set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") -endif() +# if(NOT WIN32) +# set(CMAKE_CXX_FLAGS_RELEASE "-O3 -DNDEBUG") +# set(CMAKE_C_FLAGS_RELEASE "-O3 -DNDEBUG") +# endif() if (PSP_WASM_BUILD) #################### @@ -286,6 +286,7 @@ if (PSP_WASM_BUILD) set(OPT_FLAGS " \ -O3 \ -g0 \ + -flto \ ") # TODO: -flto @@ -465,7 +466,13 @@ endif() ##################### -set(CMAKE_C_FLAGS " \ +set(CMAKE_C_FLAGS_RELEASE " \ + ${CMAKE_C_FLAGS} \ + ${EXTENDED_FLAGS} \ + ${OPT_FLAGS} \ + ") + +set(CMAKE_CXX_FLAGS_RELEASE " \ ${CMAKE_C_FLAGS} \ ${EXTENDED_FLAGS} \ ${OPT_FLAGS} \ @@ -476,6 +483,7 @@ if(CMAKE_CXX_COMPILER_ID STREQUAL "GNU") endif() set (SOURCE_FILES + ${PSP_CPP_SRC}/src/cpp/vendor/arrow_compute_registry.cpp ${PSP_CPP_SRC}/src/cpp/aggregate.cpp ${PSP_CPP_SRC}/src/cpp/aggspec.cpp ${PSP_CPP_SRC}/src/cpp/arg_sort.cpp @@ -595,7 +603,7 @@ message("${BUILD_MESSAGE}\n") if (PSP_WASM_BUILD) set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} \ --bind \ - --source-map-base=\"\" \ + --source-map-base \"\" \ --memory-init-file 0 \ -s EXPORT_ES6=1 \ -s NO_EXIT_RUNTIME=1 \ @@ -606,6 +614,7 @@ if (PSP_WASM_BUILD) -s MAXIMUM_MEMORY=4gb \ -s USE_ES6_IMPORT_META=0 \ -s EXPORTED_FUNCTIONS=\"['_main']\" \ + -s ERROR_ON_UNDEFINED_SYMBOLS=1 \ ") add_library(psp ${WASM_SOURCE_FILES}) @@ -613,7 +622,7 @@ if (PSP_WASM_BUILD) set_target_properties(psp PROPERTIES COMPILE_FLAGS "") target_link_libraries(psp arrow re2) - # "esm/erspective.cpp.js" from CMAKE_EXECUTABLE_SYNTAX + # "esm/perspective.cpp.js" from CMAKE_EXECUTABLE_SYNTAX add_executable(perspective_esm src/cpp/emscripten.cpp) target_link_libraries(perspective_esm psp) target_link_options(perspective_esm PRIVATE -s ENVIRONMENT=worker) diff --git a/cpp/perspective/package.json b/cpp/perspective/package.json index e1a00565fb..6216dc79d6 100644 --- a/cpp/perspective/package.json +++ b/cpp/perspective/package.json @@ -3,7 +3,7 @@ "private": true, "author": "The Perspective Authors", "license": "Apache-2.0", - "version": "1.0.8", + "version": "1.1.0", "main": "./dist/esm/perspective.cpp.js", "files": [ "dist/esm/**/*", diff --git a/cpp/perspective/src/cpp/arrow_writer.cpp b/cpp/perspective/src/cpp/arrow_writer.cpp index a0813212b1..582190c759 100644 --- a/cpp/perspective/src/cpp/arrow_writer.cpp +++ b/cpp/perspective/src/cpp/arrow_writer.cpp @@ -75,204 +75,11 @@ namespace apachearrow { return t.to_string(); } - std::int32_t - get_idx(std::int32_t cidx, std::int32_t ridx, std::int32_t stride, - t_get_data_extents extents) { - return (ridx - extents.m_srow) * stride + (cidx - extents.m_scol); - } - - std::shared_ptr - boolean_col_to_array(const std::vector& data, std::int32_t cidx, - std::int32_t stride, t_get_data_extents extents) { - arrow::BooleanBuilder array_builder; - auto reserve_status - = array_builder.Reserve(extents.m_erow - extents.m_srow); - if (!reserve_status.ok()) { - std::stringstream ss; - ss << "Failed to allocate buffer for column: " - << reserve_status.message() << std::endl; - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - - for (int ridx = extents.m_srow; ridx < extents.m_erow; ++ridx) { - auto idx = get_idx(cidx, ridx, stride, extents); - t_tscalar scalar = data.operator[](idx); - arrow::Status s; - if (scalar.is_valid() && scalar.get_dtype() != DTYPE_NONE) { - array_builder.UnsafeAppend(get_scalar(scalar)); - } else { - array_builder.UnsafeAppendNull(); - } - } - - std::shared_ptr array; - arrow::Status status = array_builder.Finish(&array); - if (!status.ok()) { - PSP_COMPLAIN_AND_ABORT( - "Could not serialize boolean column: " + status.message()); - } - return array; - } - - std::shared_ptr - date_col_to_array(const std::vector& data, std::int32_t cidx, - std::int32_t stride, t_get_data_extents extents) { - arrow::Date32Builder array_builder; - auto reserve_status - = array_builder.Reserve(extents.m_erow - extents.m_srow); - if (!reserve_status.ok()) { - std::stringstream ss; - ss << "Failed to allocate buffer for column: " - << reserve_status.message() << std::endl; - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - - for (int ridx = extents.m_srow; ridx < extents.m_erow; ++ridx) { - auto idx = get_idx(cidx, ridx, stride, extents); - t_tscalar scalar = data.operator[](idx); - if (scalar.is_valid() && scalar.get_dtype() != DTYPE_NONE) { - t_date val = scalar.get(); - // years are signed, while month/days are unsigned - date::year year{val.year()}; - // Increment month by 1, as date::month is [1-12] but - // t_date::month() is [0-11] - date::month month{static_cast(val.month() + 1)}; - date::day day{static_cast(val.day())}; - date::year_month_day ymd(year, month, day); - date::sys_days days_since_epoch = ymd; - array_builder.UnsafeAppend(static_cast( - days_since_epoch.time_since_epoch().count())); - } else { - array_builder.UnsafeAppendNull(); - } - } - - std::shared_ptr array; - arrow::Status status = array_builder.Finish(&array); - if (!status.ok()) { - PSP_COMPLAIN_AND_ABORT( - "Could not serialize date column: " + status.message()); - } - return array; - } - - std::shared_ptr - timestamp_col_to_array(const std::vector& data, - std::int32_t cidx, std::int32_t stride, t_get_data_extents extents) { - // TimestampType requires parameters, so initialize them here - std::shared_ptr type - = arrow::timestamp(arrow::TimeUnit::MILLI); - arrow::TimestampBuilder array_builder( - type, arrow::default_memory_pool()); - auto reserve_status - = array_builder.Reserve(extents.m_erow - extents.m_srow); - if (!reserve_status.ok()) { - std::stringstream ss; - ss << "Failed to allocate buffer for column: " - << reserve_status.message() << std::endl; - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - - for (int ridx = extents.m_srow; ridx < extents.m_erow; ++ridx) { - auto idx = get_idx(cidx, ridx, stride, extents); - t_tscalar scalar = data.operator[](idx); - if (scalar.is_valid() && scalar.get_dtype() != DTYPE_NONE) { - array_builder.UnsafeAppend(get_scalar(scalar)); - } else { - array_builder.UnsafeAppendNull(); - } - } - - std::shared_ptr array; - arrow::Status status = array_builder.Finish(&array); - if (!status.ok()) { - PSP_COMPLAIN_AND_ABORT( - "Could not serialize timestamp column: " + status.message()); - } - return array; - } - - std::shared_ptr - string_col_to_dictionary_array(const std::vector& data, - std::int32_t cidx, std::int32_t stride, t_get_data_extents extents) { - t_vocab vocab; - vocab.init(false); - arrow::Int32Builder indices_builder; - arrow::StringBuilder values_builder; - auto reserve_status - = indices_builder.Reserve(extents.m_erow - extents.m_srow); - if (!reserve_status.ok()) { - std::stringstream ss; - ss << "Failed to allocate buffer for column: " - << reserve_status.message() << std::endl; - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - - for (int ridx = extents.m_srow; ridx < extents.m_erow; ++ridx) { - auto idx = get_idx(cidx, ridx, stride, extents); - t_tscalar scalar = data.operator[](idx); - if (scalar.is_valid() && scalar.get_dtype() != DTYPE_NONE) { - auto adx = vocab.get_interned(scalar.to_string()); - indices_builder.UnsafeAppend(adx); - } else { - indices_builder.UnsafeAppendNull(); - } - } - - // get str out of vocab - for (auto i = 0; i < vocab.get_vlenidx(); i++) { - const char* str = vocab.unintern_c(i); - arrow::Status s = values_builder.Append(str, strlen(str)); - if (!s.ok()) { - std::stringstream ss; - ss << "Could not append string to dictionary array: " - << s.message() << std::endl; - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - } - - // Write dictionary indices - std::shared_ptr indices_array; - arrow::Status indices_status = indices_builder.Finish(&indices_array); - if (!indices_status.ok()) { - std::stringstream ss; - ss << "Could not write indices for dictionary array: " - << indices_status.message() << std::endl; - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - - // Write dictionary values - std::shared_ptr values_array; - arrow::Status values_status = values_builder.Finish(&values_array); - if (!values_status.ok()) { - std::stringstream ss; - ss << "Could not write values for dictionary array: " - << values_status.message() << std::endl; - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - auto dictionary_type = arrow::dictionary(arrow::int32(), arrow::utf8()); - -#if ARROW_VERSION_MAJOR < 1 - std::shared_ptr dictionary_array; - PSP_CHECK_ARROW_STATUS(arrow::DictionaryArray::FromArrays( - dictionary_type, indices_array, values_array, &dictionary_array)); - - return dictionary_array; -#else - arrow::Result> result - = arrow::DictionaryArray::FromArrays( - dictionary_type, indices_array, values_array); - - if (!result.ok()) { - std::stringstream ss; - ss << "Could not write values for dictionary array: " - << result.status().message() << std::endl; - PSP_COMPLAIN_AND_ABORT(ss.str()); - } - - return *result; -#endif - } + // std::int32_t + // get_idx(std::int32_t cidx, std::int32_t ridx, std::int32_t stride, + // t_get_data_extents extents) { + // return (ridx - extents.m_srow) * stride + (cidx - extents.m_scol); + // } } // namespace apachearrow } // namespace perspective \ No newline at end of file diff --git a/cpp/perspective/src/cpp/computed_expression.cpp b/cpp/perspective/src/cpp/computed_expression.cpp index 86ab007e4b..70312c97e1 100644 --- a/cpp/perspective/src/cpp/computed_expression.cpp +++ b/cpp/perspective/src/cpp/computed_expression.cpp @@ -443,8 +443,8 @@ t_computed_function_store::t_computed_function_store(t_expression_vocab& vocab, , m_substring_fn(computed_function::substring(vocab, is_type_validator)) , m_replace_fn( computed_function::replace(vocab, regex_mapping, is_type_validator)) - , m_replace_all_fn( - computed_function::replace_all(vocab, regex_mapping, is_type_validator)) {} + , m_replace_all_fn(computed_function::replace_all( + vocab, regex_mapping, is_type_validator)) {} void t_computed_function_store::register_computed_functions( @@ -462,8 +462,7 @@ t_computed_function_store::register_computed_functions( sym_table.add_function("is_null", t_computed_expression_parser::IS_NULL_FN); sym_table.add_function( "is_not_null", t_computed_expression_parser::IS_NOT_NULL_FN); - sym_table.add_function( - "random", t_computed_expression_parser::RANDOM_FN); + sym_table.add_function("random", t_computed_expression_parser::RANDOM_FN); // Date/datetime functions sym_table.add_function( diff --git a/cpp/perspective/src/cpp/computed_function.cpp b/cpp/perspective/src/cpp/computed_function.cpp index c456c1a58f..7ca4de56ff 100644 --- a/cpp/perspective/src/cpp/computed_function.cpp +++ b/cpp/perspective/src/cpp/computed_function.cpp @@ -581,7 +581,6 @@ namespace computed_function { if (!str.is_valid()) return rval; - re2::StringPiece result; const std::string& match_string = str.to_string(); bool found @@ -598,7 +597,8 @@ namespace computed_function { std::size_t start_idx = result.data() - match_string.data(); std::size_t end_idx = start_idx + result.size() - 1; - if (start_idx < 0 || end_idx < 0 || end_idx >= match_string.size() || (start_idx > end_idx)) { + if (start_idx < 0 || end_idx < 0 || end_idx >= match_string.size() + || (start_idx > end_idx)) { rval.set(false); return rval; } @@ -616,7 +616,8 @@ namespace computed_function { return rval; } - substring::substring(t_expression_vocab& expression_vocab, bool is_type_validator) + substring::substring( + t_expression_vocab& expression_vocab, bool is_type_validator) : m_expression_vocab(expression_vocab) , m_is_type_validator(is_type_validator) {} @@ -652,7 +653,7 @@ namespace computed_function { if (gt.type == t_generic_type::e_scalar) { t_scalar_view temp_scalar_view(gt); t_tscalar temp_scalar = temp_scalar_view(); - + // type check - first param must be string, 2nd and 3rd param // must be numeric, all must be valid t_dtype dtype = temp_scalar.get_dtype(); @@ -684,7 +685,7 @@ namespace computed_function { return rval; } } - + // done type checking if (m_is_type_validator) { return rval; @@ -694,9 +695,8 @@ namespace computed_function { // Value check: strings cannot be 0 length, indices must be valid if (length == 0 || start_idx < 0 - || (num_params == 3 && substring_length < 0) - || start_idx >= length - || (substring_length != std::string::npos + || (num_params == 3 && substring_length < 0) || start_idx >= length + || (substring_length != std::string::npos && start_idx + substring_length > length)) { return rval; } @@ -728,8 +728,8 @@ namespace computed_function { // the replace pattern t_string_view pattern_view(parameters[1]); - std::string match_pattern = - std::string(pattern_view.begin(), pattern_view.end()); + std::string match_pattern + = std::string(pattern_view.begin(), pattern_view.end()); // replacer can be a string literal, for the string '' as intern does // not pick up on empty strings but we need to be able to replace @@ -742,8 +742,8 @@ namespace computed_function { replacer_scalar = replacer_view(); } else if (gt.type == t_generic_type::e_string) { t_string_view replacer_view(gt); - std::string replacer_str = - std::string(replacer_view.begin(), replacer_view.end()); + std::string replacer_str + = std::string(replacer_view.begin(), replacer_view.end()); // only the empty string should be passed in as a string literal, // all other strings must be interned first. @@ -761,11 +761,11 @@ namespace computed_function { if (string_scalar.m_type != DTYPE_STR || replacer_scalar.m_type != DTYPE_STR - || match_pattern.size() == 0) { + || match_pattern.size() == 0) { rval.m_status = STATUS_CLEAR; return rval; } - + // typecheck the regex RE2* compiled_pattern = m_regex_mapping.intern(match_pattern); @@ -775,20 +775,22 @@ namespace computed_function { } // done with type_checking - if (m_is_type_validator) return rval; + if (m_is_type_validator) + return rval; // make a copy of search_str, as replace() will mutate it and we // don't want to mutate the string in the vocab std::string search_string = string_scalar.to_string(); - if (search_string.size() == 0) return rval; + if (search_string.size() == 0) + return rval; // but we can take a reference to the replacer const std::string& replacer_string = replacer_scalar.to_string(); re2::StringPiece replacer(replacer_string); - bool replaced = RE2::Replace( - &(search_string), *(compiled_pattern), replacer); + bool replaced + = RE2::Replace(&(search_string), *(compiled_pattern), replacer); if (!replaced) { // Return the original result if the replacement didn't happen @@ -822,8 +824,8 @@ namespace computed_function { // the replace pattern t_string_view pattern_view(parameters[1]); - std::string match_pattern = - std::string(pattern_view.begin(), pattern_view.end()); + std::string match_pattern + = std::string(pattern_view.begin(), pattern_view.end()); // replacer can be a string literal, for the string '' as intern does // not pick up on empty strings but we need to be able to replace @@ -836,8 +838,8 @@ namespace computed_function { replacer_scalar = replacer_view(); } else if (gt.type == t_generic_type::e_string) { t_string_view replacer_view(gt); - std::string replacer_str = - std::string(replacer_view.begin(), replacer_view.end()); + std::string replacer_str + = std::string(replacer_view.begin(), replacer_view.end()); // only the empty string should be passed in as a string literal, // all other strings must be interned first. @@ -855,11 +857,11 @@ namespace computed_function { if (string_scalar.m_type != DTYPE_STR || replacer_scalar.m_type != DTYPE_STR - || match_pattern.size() == 0) { + || match_pattern.size() == 0) { rval.m_status = STATUS_CLEAR; return rval; } - + // typecheck the regex RE2* compiled_pattern = m_regex_mapping.intern(match_pattern); @@ -869,13 +871,15 @@ namespace computed_function { } // done with type_checking - if (m_is_type_validator) return rval; + if (m_is_type_validator) + return rval; // make a copy of search_str, as replace() will mutate it and we // don't want to mutate the string in the vocab std::string search_string = string_scalar.to_string(); - if (search_string.size() == 0) return rval; + if (search_string.size() == 0) + return rval; // but we can take a reference to the replacer const std::string& replacer_string = replacer_scalar.to_string(); diff --git a/cpp/perspective/src/cpp/emscripten.cpp b/cpp/perspective/src/cpp/emscripten.cpp index 8626239334..43fa2b312b 100644 --- a/cpp/perspective/src/cpp/emscripten.cpp +++ b/cpp/perspective/src/cpp/emscripten.cpp @@ -205,15 +205,24 @@ namespace binding { to_arrow(std::shared_ptr> view, std::int32_t start_row, std::int32_t end_row, std::int32_t start_col, std::int32_t end_col) { std::shared_ptr s - = view->to_arrow(start_row, end_row, start_col, end_col); + = view->to_arrow(start_row, end_row, start_col, end_col, true); return str_to_arraybuffer(s)["buffer"]; } + template + t_val + to_csv(std::shared_ptr> view, std::int32_t start_row, + std::int32_t end_row, std::int32_t start_col, std::int32_t end_col) { + std::shared_ptr s + = view->to_csv(start_row, end_row, start_col, end_col); + return t_val(*s); + } + template t_val get_row_delta(std::shared_ptr> view) { auto slice = view->get_row_delta(); - auto row_delta = view->data_slice_to_arrow(slice); + auto row_delta = view->data_slice_to_arrow(slice, false); return str_to_arraybuffer(row_delta)["buffer"]; } @@ -937,9 +946,8 @@ namespace binding { << "` is nan" << std::endl; tbl.promote_column(name, DTYPE_STR, i, false); col = tbl.get_column(name); - _fill_col_string( - accessor, col, name, cidx, DTYPE_STR, is_update, - is_limit); + _fill_col_string(accessor, col, name, cidx, DTYPE_STR, + is_update, is_limit); return; } else { col->set_nth(i, static_cast(fval)); @@ -1040,16 +1048,20 @@ namespace binding { accessor, tbl, col, name, cidx, type, is_update, is_limit); } break; case DTYPE_BOOL: { - _fill_col_bool(accessor, col, name, cidx, type, is_update, is_limit); + _fill_col_bool( + accessor, col, name, cidx, type, is_update, is_limit); } break; case DTYPE_DATE: { - _fill_col_date(accessor, col, name, cidx, type, is_update, is_limit); + _fill_col_date( + accessor, col, name, cidx, type, is_update, is_limit); } break; case DTYPE_TIME: { - _fill_col_time(accessor, col, name, cidx, type, is_update, is_limit); + _fill_col_time( + accessor, col, name, cidx, type, is_update, is_limit); } break; case DTYPE_STR: { - _fill_col_string(accessor, col, name, cidx, type, is_update, is_limit); + _fill_col_string( + accessor, col, name, cidx, type, is_update, is_limit); } break; case DTYPE_NONE: { break; @@ -1084,7 +1096,8 @@ namespace binding { } auto col = tbl.get_column(name); - _fill_data_helper(dcol, tbl, col, name, cidx, type, is_update, is_limit); + _fill_data_helper( + dcol, tbl, col, name, cidx, type, is_update, is_limit); } // Fill index column - recreated every time a `t_data_table` is created. @@ -2285,6 +2298,10 @@ EMSCRIPTEN_BINDINGS(perspective) { function("to_arrow_zero", &to_arrow); function("to_arrow_one", &to_arrow); function("to_arrow_two", &to_arrow); + function("to_csv_unit", &to_csv); + function("to_csv_zero", &to_csv); + function("to_csv_one", &to_csv); + function("to_csv_two", &to_csv); function("get_row_delta_unit", &get_row_delta); function("get_row_delta_zero", &get_row_delta); function("get_row_delta_one", &get_row_delta); diff --git a/cpp/perspective/src/cpp/vendor/arrow_compute_registry.cpp b/cpp/perspective/src/cpp/vendor/arrow_compute_registry.cpp new file mode 100644 index 0000000000..3c8219e3fc --- /dev/null +++ b/cpp/perspective/src/cpp/vendor/arrow_compute_registry.cpp @@ -0,0 +1,200 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "arrow/compute/registry.h" + +#include +#include +#include +#include +#include + +#include "arrow/compute/function.h" +#include "arrow/compute/function_internal.h" +#include "arrow/compute/registry_internal.h" +#include "arrow/status.h" +#include "arrow/util/logging.h" + +namespace arrow { +namespace compute { + +class FunctionRegistry::FunctionRegistryImpl { + public: + Status AddFunction(std::shared_ptr function, bool allow_overwrite) { + RETURN_NOT_OK(function->Validate()); + + std::lock_guard mutation_guard(lock_); + + const std::string& name = function->name(); + auto it = name_to_function_.find(name); + if (it != name_to_function_.end() && !allow_overwrite) { + return Status::KeyError("Already have a function registered with name: ", name); + } + name_to_function_[name] = std::move(function); + return Status::OK(); + } + + Status AddAlias(const std::string& target_name, const std::string& source_name) { + std::lock_guard mutation_guard(lock_); + + auto it = name_to_function_.find(source_name); + if (it == name_to_function_.end()) { + return Status::KeyError("No function registered with name: ", source_name); + } + name_to_function_[target_name] = it->second; + return Status::OK(); + } + + Status AddFunctionOptionsType(const FunctionOptionsType* options_type, + bool allow_overwrite = false) { + std::lock_guard mutation_guard(lock_); + + const std::string name = options_type->type_name(); + auto it = name_to_options_type_.find(name); + if (it != name_to_options_type_.end() && !allow_overwrite) { + return Status::KeyError( + "Already have a function options type registered with name: ", name); + } + name_to_options_type_[name] = options_type; + return Status::OK(); + } + + Result> GetFunction(const std::string& name) const { + auto it = name_to_function_.find(name); + if (it == name_to_function_.end()) { + return Status::KeyError("No function registered with name: ", name); + } + return it->second; + } + + std::vector GetFunctionNames() const { + std::vector results; + for (auto it : name_to_function_) { + results.push_back(it.first); + } + std::sort(results.begin(), results.end()); + return results; + } + + Result GetFunctionOptionsType( + const std::string& name) const { + auto it = name_to_options_type_.find(name); + if (it == name_to_options_type_.end()) { + return Status::KeyError("No function options type registered with name: ", name); + } + return it->second; + } + + int num_functions() const { return static_cast(name_to_function_.size()); } + + private: + std::mutex lock_; + std::unordered_map> name_to_function_; + std::unordered_map name_to_options_type_; +}; + +std::unique_ptr FunctionRegistry::Make() { + return std::unique_ptr(new FunctionRegistry()); +} + +FunctionRegistry::FunctionRegistry() { impl_.reset(new FunctionRegistryImpl()); } + +FunctionRegistry::~FunctionRegistry() {} + +Status FunctionRegistry::AddFunction(std::shared_ptr function, + bool allow_overwrite) { + return impl_->AddFunction(std::move(function), allow_overwrite); +} + +Status FunctionRegistry::AddAlias(const std::string& target_name, + const std::string& source_name) { + return impl_->AddAlias(target_name, source_name); +} + +Status FunctionRegistry::AddFunctionOptionsType(const FunctionOptionsType* options_type, + bool allow_overwrite) { + return impl_->AddFunctionOptionsType(options_type, allow_overwrite); +} + +Result> FunctionRegistry::GetFunction( + const std::string& name) const { + return impl_->GetFunction(name); +} + +std::vector FunctionRegistry::GetFunctionNames() const { + return impl_->GetFunctionNames(); +} + +Result FunctionRegistry::GetFunctionOptionsType( + const std::string& name) const { + return impl_->GetFunctionOptionsType(name); +} + +int FunctionRegistry::num_functions() const { return impl_->num_functions(); } + +namespace internal { + +static std::unique_ptr CreateBuiltInRegistry() { + auto registry = FunctionRegistry::Make(); + + // Scalar functions + // RegisterScalarArithmetic(registry.get()); + // RegisterScalarBoolean(registry.get()); + RegisterScalarCast(registry.get()); + // RegisterScalarComparison(registry.get()); + // RegisterScalarIfElse(registry.get()); + // RegisterScalarNested(registry.get()); + // RegisterScalarSetLookup(registry.get()); + // RegisterScalarStringAscii(registry.get()); + // RegisterScalarTemporalBinary(registry.get()); + // RegisterScalarTemporalUnary(registry.get()); + // RegisterScalarValidity(registry.get()); + + // RegisterScalarOptions(registry.get()); + + // // Vector functions + // RegisterVectorArraySort(registry.get()); + // RegisterVectorHash(registry.get()); + // RegisterVectorNested(registry.get()); + // RegisterVectorReplace(registry.get()); + RegisterVectorSelection(registry.get()); + // RegisterVectorSort(registry.get()); + + // RegisterVectorOptions(registry.get()); + + // // Aggregate functions + // RegisterHashAggregateBasic(registry.get()); + // RegisterScalarAggregateBasic(registry.get()); + // RegisterScalarAggregateMode(registry.get()); + // RegisterScalarAggregateQuantile(registry.get()); + // RegisterScalarAggregateTDigest(registry.get()); + // RegisterScalarAggregateVariance(registry.get()); + + // RegisterAggregateOptions(registry.get()); + + return registry; +} + +} // namespace internal + +FunctionRegistry* GetFunctionRegistry() { + static auto g_registry = internal::CreateBuiltInRegistry(); + return g_registry.get(); +} + +} // namespace compute +} // namespace arrow \ No newline at end of file diff --git a/cpp/perspective/src/cpp/view.cpp b/cpp/perspective/src/cpp/view.cpp index 4df880cfe0..d3675b75b9 100644 --- a/cpp/perspective/src/cpp/view.cpp +++ b/cpp/perspective/src/cpp/view.cpp @@ -12,6 +12,8 @@ #include #include +#include + namespace perspective { std::string @@ -529,16 +531,25 @@ View::get_data(t_uindex start_row, t_uindex end_row, t_uindex start_col, template std::shared_ptr View::to_arrow(std::int32_t start_row, std::int32_t end_row, - std::int32_t start_col, std::int32_t end_col) const { + std::int32_t start_col, std::int32_t end_col, bool emit_group_by) const { std::shared_ptr> data_slice = get_data(start_row, end_row, start_col, end_col); - return data_slice_to_arrow(data_slice); + return data_slice_to_arrow(data_slice, emit_group_by); }; template std::shared_ptr -View::data_slice_to_arrow( - std::shared_ptr> data_slice) const { +View::to_csv(std::int32_t start_row, std::int32_t end_row, + std::int32_t start_col, std::int32_t end_col) const { + std::shared_ptr> data_slice + = get_data(start_row, end_row, start_col, end_col); + return data_slice_to_csv(data_slice); +}; + +template +std::pair, std::shared_ptr> +View::data_slice_to_batches( + bool emit_group_by, std::shared_ptr> data_slice) const { // From the data slice, get all the metadata we need t_get_data_extents extents = data_slice->get_data_extents(); std::int32_t start_col = extents.m_scol; @@ -555,9 +566,259 @@ View::data_slice_to_arrow( std::int32_t num_columns = end_col - start_col; - if (num_columns > 0) { - fields.reserve(num_columns); - vectors.reserve(num_columns); + std::vector row_pivots = m_view_config->get_row_pivots(); + t_uindex num_row_paths = emit_group_by ? row_pivots.size() : 0; + if (num_columns + num_row_paths > 0) { + fields.reserve(num_columns + num_row_paths); + vectors.reserve(num_columns + num_row_paths); + } + + if (emit_group_by && num_row_paths > 0 && !is_column_only()) { + auto schema = m_table->get_schema(); + for (auto rpidx = 0; rpidx < num_row_paths; ++rpidx) { + std::string column_name = row_pivots.at(rpidx); + std::string row_path_name = column_name; + row_path_name += " (Group by "; + row_path_name += std::to_string(rpidx + 1); + row_path_name += ")"; + + // Get the "table" type for this column, as row_pivots are not in + // the view schema. + t_dtype dtype; + if (schema.has_column(column_name)) { + dtype = schema.get_dtype(column_name); + } else { + for (const auto& expr : m_expressions) { + std::string expression_alias = expr->get_expression_alias(); + if (expr->get_expression_alias() == column_name) { + dtype = expr->get_dtype(); + break; + } + } + } + + std::shared_ptr arr; + switch (dtype) { + case DTYPE_INT8: { + fields.push_back( + arrow::field(row_path_name, arrow::int8())); + arr = apachearrow::numeric_col_to_array(extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_UINT8: { + fields.push_back( + arrow::field(row_path_name, arrow::uint8())); + arr = apachearrow::numeric_col_to_array(extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_INT16: { + fields.push_back( + arrow::field(row_path_name, arrow::int16())); + arr = apachearrow::numeric_col_to_array(extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_UINT16: { + fields.push_back( + arrow::field(row_path_name, arrow::uint16())); + arr = apachearrow::numeric_col_to_array(extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_INT32: { + fields.push_back( + arrow::field(row_path_name, arrow::int32())); + arr = apachearrow::numeric_col_to_array(extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_UINT32: { + fields.push_back( + arrow::field(row_path_name, arrow::uint32())); + arr = apachearrow::numeric_col_to_array(extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_INT64: { + fields.push_back( + arrow::field(row_path_name, arrow::int64())); + arr = apachearrow::numeric_col_to_array(extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_UINT64: { + fields.push_back( + arrow::field(row_path_name, arrow::uint64())); + arr = apachearrow::numeric_col_to_array(extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_FLOAT32: { + fields.push_back( + arrow::field(row_path_name, arrow::float32())); + arr = apachearrow::numeric_col_to_array(extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_FLOAT64: { + fields.push_back( + arrow::field(row_path_name, arrow::float64())); + arr = apachearrow::numeric_col_to_array(extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_DATE: { + fields.push_back( + arrow::field(row_path_name, arrow::date32())); + arr = apachearrow::date_col_to_array( + extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_TIME: { + fields.push_back(arrow::field(row_path_name, + arrow::timestamp(arrow::TimeUnit::MILLI))); + arr = apachearrow::timestamp_col_to_array( + extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_BOOL: { + fields.push_back( + arrow::field(row_path_name, arrow::boolean())); + arr = apachearrow::boolean_col_to_array( + extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_STR: { + fields.push_back(arrow::field(row_path_name, + arrow::dictionary(arrow::int32(), arrow::utf8()))); + arr = apachearrow::string_col_to_dictionary_array( + extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + case DTYPE_OBJECT: { + fields.push_back( + arrow::field(row_path_name, arrow::uint64())); + arr = apachearrow::numeric_col_to_array(extents, [&, rpidx](t_uindex ridx) { + auto depth = m_ctx->unity_get_row_depth(ridx); + if (rpidx < depth) { + return m_ctx->unity_get_row_path(ridx).at( + (depth - 1) - rpidx); + } else { + return mknone(); + } + }); + } break; + default: { + std::stringstream ss; + ss << "Cannot serialize column `" << row_path_name + << "` of type `" << get_dtype_descr(dtype) + << "` to Arrow format." << std::endl; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + } + vectors.push_back(arr); + } } // calculate the number of columns (including __ROW_PATH__) minus @@ -583,11 +844,11 @@ View::data_slice_to_arrow( std::vector col_path = names.at(cidx); t_dtype dtype = get_column_dtype(cidx); - // mean and weighted mean uses DTYPE_F64PAIR on the aggtable, which is - // the dtype returned by get_column_dtype. However, in the output data - // slice they are DTYPE_FLOAT64 and the f64 pair is not exposed outside - // of the sparse tree. Thus, treat f64 pair as DTYPE_FLOAT64 for arrow - // serialization. + // mean and weighted mean uses DTYPE_F64PAIR on the aggtable, which + // is the dtype returned by get_column_dtype. However, in the output + // data slice they are DTYPE_FLOAT64 and the f64 pair is not exposed + // outside of the sparse tree. Thus, treat f64 pair as DTYPE_FLOAT64 + // for arrow serialization. if (dtype == DTYPE_F64PAIR) { dtype = DTYPE_FLOAT64; } @@ -605,79 +866,135 @@ View::data_slice_to_arrow( case DTYPE_INT8: { fields.push_back(arrow::field(name, arrow::int8())); arr = apachearrow::numeric_col_to_array(slice, cidx, stride, extents); + std::int8_t>( + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_UINT8: { fields.push_back(arrow::field(name, arrow::uint8())); arr = apachearrow::numeric_col_to_array(slice, cidx, stride, extents); + std::uint8_t>( + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_INT16: { fields.push_back(arrow::field(name, arrow::int16())); arr = apachearrow::numeric_col_to_array(slice, cidx, stride, extents); + std::int16_t>( + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_UINT16: { fields.push_back(arrow::field(name, arrow::uint16())); arr = apachearrow::numeric_col_to_array(slice, cidx, stride, extents); + std::uint16_t>( + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_INT32: { fields.push_back(arrow::field(name, arrow::int32())); arr = apachearrow::numeric_col_to_array(slice, cidx, stride, extents); + std::int32_t>( + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_UINT32: { fields.push_back(arrow::field(name, arrow::uint32())); arr = apachearrow::numeric_col_to_array(slice, cidx, stride, extents); + std::uint32_t>( + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_INT64: { fields.push_back(arrow::field(name, arrow::int64())); arr = apachearrow::numeric_col_to_array(slice, cidx, stride, extents); + std::int64_t>( + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_UINT64: { fields.push_back(arrow::field(name, arrow::uint64())); arr = apachearrow::numeric_col_to_array(slice, cidx, stride, extents); + std::uint64_t>( + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_FLOAT32: { fields.push_back(arrow::field(name, arrow::float32())); arr = apachearrow::numeric_col_to_array(slice, cidx, stride, extents); + float>( + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_FLOAT64: { fields.push_back(arrow::field(name, arrow::float64())); arr = apachearrow::numeric_col_to_array(slice, cidx, stride, extents); + double>( + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_DATE: { fields.push_back(arrow::field(name, arrow::date32())); arr = apachearrow::date_col_to_array( - slice, cidx, stride, extents); + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_TIME: { fields.push_back(arrow::field( name, arrow::timestamp(arrow::TimeUnit::MILLI))); arr = apachearrow::timestamp_col_to_array( - slice, cidx, stride, extents); + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_BOOL: { fields.push_back(arrow::field(name, arrow::boolean())); arr = apachearrow::boolean_col_to_array( - slice, cidx, stride, extents); + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_STR: { fields.push_back(arrow::field( name, arrow::dictionary(arrow::int32(), arrow::utf8()))); arr = apachearrow::string_col_to_dictionary_array( - slice, cidx, stride, extents); + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; case DTYPE_OBJECT: { fields.push_back(arrow::field(name, arrow::uint64())); arr = apachearrow::numeric_col_to_array(slice, cidx, stride, extents); + std::uint64_t>( + extents, [slice, cidx, stride, extents](t_uindex ridx) { + return slice[(ridx - extents.m_srow) * stride + + (cidx - extents.m_scol)]; + }); } break; default: { std::stringstream ss; @@ -701,21 +1018,48 @@ View::data_slice_to_arrow( PSP_COMPLAIN_AND_ABORT(ss.str()); } - std::shared_ptr buffer; + return std::make_pair(arrow_schema, batches); +} -#if ARROW_VERSION_MAJOR < 1 - auto allocated = arrow::AllocateResizableBuffer(0, &buffer); +template +std::shared_ptr +View::data_slice_to_arrow( + std::shared_ptr> data_slice, bool emit_group_by) const { + std::pair, + std::shared_ptr> + pairs = data_slice_to_batches(emit_group_by, data_slice); + std::shared_ptr batches = pairs.second; + std::shared_ptr arrow_schema = pairs.first; + arrow::Result> allocated + = arrow::AllocateResizableBuffer(0); if (!allocated.ok()) { std::stringstream ss; - ss << "Failed to allocate buffer: " << allocated.message() << std::endl; + ss << "Failed to allocate buffer: " << allocated.status().message() + << std::endl; PSP_COMPLAIN_AND_ABORT(ss.str()); } + std::shared_ptr buffer; + buffer = *allocated; arrow::io::BufferOutputStream sink(buffer); - auto options = arrow::ipc::IpcOptions::Defaults(); - auto res = arrow::ipc::RecordBatchStreamWriter::Open( - &sink, arrow_schema, options); -#else + auto options = arrow::ipc::IpcWriteOptions::Defaults(); + auto res = arrow::ipc::MakeStreamWriter(&sink, arrow_schema, options); + std::shared_ptr writer = *res; + PSP_CHECK_ARROW_STATUS(writer->WriteRecordBatch(*batches)); + PSP_CHECK_ARROW_STATUS(writer->Close()); + PSP_CHECK_ARROW_STATUS(sink.Close()); + return std::make_shared(buffer->ToString()); +} + +template +std::shared_ptr +View::data_slice_to_csv( + std::shared_ptr> data_slice) const { + std::pair, + std::shared_ptr> + pairs = data_slice_to_batches(true, data_slice); + std::shared_ptr batches = pairs.second; + std::shared_ptr arrow_schema = pairs.first; arrow::Result> allocated = arrow::AllocateResizableBuffer(0); if (!allocated.ok()) { @@ -725,15 +1069,16 @@ View::data_slice_to_arrow( PSP_COMPLAIN_AND_ABORT(ss.str()); } + std::shared_ptr buffer; buffer = *allocated; arrow::io::BufferOutputStream sink(buffer); - auto options = arrow::ipc::IpcWriteOptions::Defaults(); - auto res = arrow::ipc::MakeStreamWriter(&sink, arrow_schema, options); -#endif - - std::shared_ptr writer = *res; + auto write_options = arrow::csv::WriteOptions::Defaults(); + auto maybe_writer + = arrow::csv::MakeCSVWriter(&sink, arrow_schema, write_options); + std::shared_ptr writer = *maybe_writer; PSP_CHECK_ARROW_STATUS(writer->WriteRecordBatch(*batches)); PSP_CHECK_ARROW_STATUS(writer->Close()); + PSP_CHECK_ARROW_STATUS(sink.Close()); return std::make_shared(buffer->ToString()); } @@ -926,17 +1271,17 @@ View::get_row_delta() const { t_uindex num_sides = sides(); if (num_sides == 2 && m_sort.size() > 0) { - // Use column_names instead of column_paths, as column_names does not - // skip hidden sort columns whereas column_paths does, which causes - // issues later on. + // Use column_names instead of column_paths, as column_names does + // not skip hidden sort columns whereas column_paths does, which + // causes issues later on. paths = column_names(true, m_column_pivots.size()); } else { paths = column_paths(); } - // Add __ROW_PATH__ to the beginning for column only or for 2-sided sorted - // contexts where we used `column_names`, which does not add __ROW_PATH__ - // automatically. + // Add __ROW_PATH__ to the beginning for column only or for 2-sided + // sorted contexts where we used `column_names`, which does not add + // __ROW_PATH__ automatically. if (is_column_only() || (num_sides == 2 && m_sort.size() > 0)) { t_tscalar row_path; row_path.set("__ROW_PATH__"); diff --git a/cpp/perspective/src/include/perspective/arrow_writer.h b/cpp/perspective/src/include/perspective/arrow_writer.h index d05335e9cd..44256b90fd 100644 --- a/cpp/perspective/src/include/perspective/arrow_writer.h +++ b/cpp/perspective/src/include/perspective/arrow_writer.h @@ -102,9 +102,79 @@ namespace apachearrow { * @param stride * @return std::shared_ptr */ - std::shared_ptr string_col_to_dictionary_array( - const std::vector& data, std::int32_t cidx, - std::int32_t stride, t_get_data_extents extents); + template + std::shared_ptr + string_col_to_dictionary_array(t_get_data_extents extents, F f) { + t_vocab vocab; + vocab.init(false); + arrow::Int32Builder indices_builder; + arrow::StringBuilder values_builder; + auto reserve_status + = indices_builder.Reserve(extents.m_erow - extents.m_srow); + if (!reserve_status.ok()) { + std::stringstream ss; + ss << "Failed to allocate buffer for column: " + << reserve_status.message() << std::endl; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + + for (int ridx = extents.m_srow; ridx < extents.m_erow; ++ridx) { + // auto idx = get_idx(cidx, ridx, stride, extents); + t_tscalar scalar = f(ridx); + if (scalar.is_valid() && scalar.get_dtype() != DTYPE_NONE) { + auto adx = vocab.get_interned(scalar.to_string()); + indices_builder.UnsafeAppend(adx); + } else { + indices_builder.UnsafeAppendNull(); + } + } + + // get str out of vocab + for (auto i = 0; i < vocab.get_vlenidx(); i++) { + const char* str = vocab.unintern_c(i); + arrow::Status s = values_builder.Append(str, strlen(str)); + if (!s.ok()) { + std::stringstream ss; + ss << "Could not append string to dictionary array: " + << s.message() << std::endl; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + } + + // Write dictionary indices + std::shared_ptr indices_array; + arrow::Status indices_status = indices_builder.Finish(&indices_array); + if (!indices_status.ok()) { + std::stringstream ss; + ss << "Could not write indices for dictionary array: " + << indices_status.message() << std::endl; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + + // Write dictionary values + std::shared_ptr values_array; + arrow::Status values_status = values_builder.Finish(&values_array); + if (!values_status.ok()) { + std::stringstream ss; + ss << "Could not write values for dictionary array: " + << values_status.message() << std::endl; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + auto dictionary_type = arrow::dictionary(arrow::int32(), arrow::utf8()); + + arrow::Result> result + = arrow::DictionaryArray::FromArrays( + dictionary_type, indices_array, values_array); + + if (!result.ok()) { + std::stringstream ss; + ss << "Could not write values for dictionary array: " + << result.status().message() << std::endl; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + + return *result; + } /** * @brief Build an `arrow::Array` from a column contained in `data`. Column @@ -118,10 +188,9 @@ namespace apachearrow { * @param stride * @return std::shared_ptr */ - template + template std::shared_ptr - numeric_col_to_array(const std::vector& data, std::int32_t cidx, - std::int32_t stride, t_get_data_extents extents) { + numeric_col_to_array(t_get_data_extents extents, F f) { // NumericBuilder encompasses the most types (int/float/datetime) arrow::NumericBuilder array_builder; auto reserve_status @@ -134,8 +203,7 @@ namespace apachearrow { } for (int ridx = extents.m_srow; ridx < extents.m_erow; ++ridx) { - auto idx = get_idx(cidx, ridx, stride, extents); - t_tscalar scalar = data.operator[](idx); + t_tscalar scalar = f(ridx); if (scalar.is_valid() && scalar.get_dtype() != DTYPE_NONE) { ArrowValueType val = get_scalar(scalar); array_builder.UnsafeAppend(val); @@ -154,5 +222,113 @@ namespace apachearrow { return array; } + template + std::shared_ptr + boolean_col_to_array(t_get_data_extents extents, F f) { + arrow::BooleanBuilder array_builder; + auto reserve_status + = array_builder.Reserve(extents.m_erow - extents.m_srow); + if (!reserve_status.ok()) { + std::stringstream ss; + ss << "Failed to allocate buffer for column: " + << reserve_status.message() << std::endl; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + + for (int ridx = extents.m_srow; ridx < extents.m_erow; ++ridx) { + t_tscalar scalar = f(ridx); + arrow::Status s; + if (scalar.is_valid() && scalar.get_dtype() != DTYPE_NONE) { + array_builder.UnsafeAppend(get_scalar(scalar)); + } else { + array_builder.UnsafeAppendNull(); + } + } + + std::shared_ptr array; + arrow::Status status = array_builder.Finish(&array); + if (!status.ok()) { + PSP_COMPLAIN_AND_ABORT( + "Could not serialize boolean column: " + status.message()); + } + return array; + } + + template + std::shared_ptr + date_col_to_array(t_get_data_extents extents, F f) { + arrow::Date32Builder array_builder; + auto reserve_status + = array_builder.Reserve(extents.m_erow - extents.m_srow); + if (!reserve_status.ok()) { + std::stringstream ss; + ss << "Failed to allocate buffer for column: " + << reserve_status.message() << std::endl; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + + for (int ridx = extents.m_srow; ridx < extents.m_erow; ++ridx) { + t_tscalar scalar = f(ridx); + if (scalar.is_valid() && scalar.get_dtype() != DTYPE_NONE) { + t_date val = scalar.get(); + // years are signed, while month/days are unsigned + date::year year{val.year()}; + // Increment month by 1, as date::month is [1-12] but + // t_date::month() is [0-11] + date::month month{static_cast(val.month() + 1)}; + date::day day{static_cast(val.day())}; + date::year_month_day ymd(year, month, day); + date::sys_days days_since_epoch = ymd; + array_builder.UnsafeAppend(static_cast( + days_since_epoch.time_since_epoch().count())); + } else { + array_builder.UnsafeAppendNull(); + } + } + + std::shared_ptr array; + arrow::Status status = array_builder.Finish(&array); + if (!status.ok()) { + PSP_COMPLAIN_AND_ABORT( + "Could not serialize date column: " + status.message()); + } + return array; + } + + template + std::shared_ptr + timestamp_col_to_array(t_get_data_extents extents, F f) { + // TimestampType requires parameters, so initialize them here + std::shared_ptr type + = arrow::timestamp(arrow::TimeUnit::MILLI); + arrow::TimestampBuilder array_builder( + type, arrow::default_memory_pool()); + auto reserve_status + = array_builder.Reserve(extents.m_erow - extents.m_srow); + if (!reserve_status.ok()) { + std::stringstream ss; + ss << "Failed to allocate buffer for column: " + << reserve_status.message() << std::endl; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + + for (int ridx = extents.m_srow; ridx < extents.m_erow; ++ridx) { + t_tscalar scalar = f(ridx); + if (scalar.is_valid() && scalar.get_dtype() != DTYPE_NONE) { + array_builder.UnsafeAppend(get_scalar(scalar)); + } else { + array_builder.UnsafeAppendNull(); + } + } + + std::shared_ptr array; + arrow::Status status = array_builder.Finish(&array); + if (!status.ok()) { + PSP_COMPLAIN_AND_ABORT( + "Could not serialize timestamp column: " + status.message()); + } + return array; + } + } // namespace apachearrow } // namespace perspective \ No newline at end of file diff --git a/cpp/perspective/src/include/perspective/vendor/arrow_compute_registry.h b/cpp/perspective/src/include/perspective/vendor/arrow_compute_registry.h new file mode 100644 index 0000000000..e83036db6a --- /dev/null +++ b/cpp/perspective/src/include/perspective/vendor/arrow_compute_registry.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// NOTE: API is EXPERIMENTAL and will change without going through a +// deprecation cycle + +#pragma once + +#include +#include +#include + +#include "arrow/result.h" +#include "arrow/status.h" +#include "arrow/util/visibility.h" + +namespace arrow { +namespace compute { + +class Function; +class FunctionOptionsType; + +/// \brief A mutable central function registry for built-in functions as well +/// as user-defined functions. Functions are implementations of +/// arrow::compute::Function. +/// +/// Generally, each function contains kernels which are implementations of a +/// function for a specific argument signature. After looking up a function in +/// the registry, one can either execute it eagerly with Function::Execute or +/// use one of the function's dispatch methods to pick a suitable kernel for +/// lower-level function execution. +class ARROW_EXPORT FunctionRegistry { + public: + ~FunctionRegistry(); + + /// \brief Construct a new registry. Most users only need to use the global + /// registry + static std::unique_ptr Make(); + + /// \brief Add a new function to the registry. Returns Status::KeyError if a + /// function with the same name is already registered + Status AddFunction(std::shared_ptr function, bool allow_overwrite = false); + + /// \brief Add aliases for the given function name. Returns Status::KeyError if the + /// function with the given name is not registered + Status AddAlias(const std::string& target_name, const std::string& source_name); + + /// \brief Add a new function options type to the registry. Returns Status::KeyError if + /// a function options type with the same name is already registered + Status AddFunctionOptionsType(const FunctionOptionsType* options_type, + bool allow_overwrite = false); + + /// \brief Retrieve a function by name from the registry + Result> GetFunction(const std::string& name) const; + + /// \brief Return vector of all entry names in the registry. Helpful for + /// displaying a manifest of available functions + std::vector GetFunctionNames() const; + + /// \brief Retrieve a function options type by name from the registry + Result GetFunctionOptionsType( + const std::string& name) const; + + /// \brief The number of currently registered functions + int num_functions() const; + + private: + FunctionRegistry(); + + // Use PIMPL pattern to not have std::unordered_map here + class FunctionRegistryImpl; + std::unique_ptr impl_; +}; + +/// \brief Return the process-global function registry +ARROW_EXPORT FunctionRegistry* GetFunctionRegistry(); + +} // namespace compute +} // namespace arrow diff --git a/cpp/perspective/src/include/perspective/view.h b/cpp/perspective/src/include/perspective/view.h index 337894e317..8ba822a08d 100644 --- a/cpp/perspective/src/include/perspective/view.h +++ b/cpp/perspective/src/include/perspective/view.h @@ -22,6 +22,7 @@ #include #include #include +#include #ifdef PSP_ENABLE_PYTHON #include #endif @@ -147,9 +148,25 @@ class PERSPECTIVE_EXPORT View { * @param end_row * @param start_col * @param end_col + * @param emit_group_by * @return std::shared_ptr */ std::shared_ptr to_arrow(std::int32_t start_row, + std::int32_t end_row, std::int32_t start_col, std::int32_t end_col, + bool emit_group_by) const; + + /** + * @brief Serializes the `View`'s data into the Apache Arrow format + * as a bytestring. Using start/end row and column, retrieve a data + * slice from the view and serialize it using `to_arrow_helper`. + * + * @param start_row + * @param end_row + * @param start_col + * @param end_col + * @return std::shared_ptr + */ + std::shared_ptr to_csv(std::int32_t start_row, std::int32_t end_row, std::int32_t start_col, std::int32_t end_col) const; @@ -165,6 +182,21 @@ class PERSPECTIVE_EXPORT View { * @return std::shared_ptr */ std::shared_ptr data_slice_to_arrow( + std::shared_ptr> data_slice, + bool emit_group_by) const; + + /** + * @brief Serializes a given data slice into the Apache Arrow format. Can + * be directly called with a pointer to a data slice in order to serialize + * it to Arrow. + * + * @param start_row + * @param end_row + * @param start_col + * @param end_col + * @return std::shared_ptr + */ + std::shared_ptr data_slice_to_csv( std::shared_ptr> data_slice) const; // Delta calculation @@ -249,6 +281,22 @@ class PERSPECTIVE_EXPORT View { std::string _map_aggregate_types( const std::string& name, const std::string& typestring) const; + /** + * @brief Serializes a given data slice into the Apache Arrow format. Can + * be directly called with a pointer to a data slice in order to serialize + * it to Arrow. + * + * @param start_row + * @param end_row + * @param start_col + * @param end_col + * @return std::shared_ptr + */ + std::pair, + std::shared_ptr> + data_slice_to_batches(bool emit_group_by, + std::shared_ptr> data_slice) const; + void _find_hidden_sort(const std::vector& sort); std::shared_ptr m_table; diff --git a/docs/md/js.md b/docs/md/js.md index 5d3a1b23bc..bfcb44f450 100644 --- a/docs/md/js.md +++ b/docs/md/js.md @@ -44,7 +44,8 @@ should use `@finos/perspective-webpack-plugin` to manage the `.worker.js` and performance, the plugin-compiled version of Perspective: - Downloads `.wasm` and `.js` assets in parallel. -- Compiles `.wasm` incrementally via [streaming instantiation](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/instantiateStreaming). +- Compiles `.wasm` incrementally via + [streaming instantiation](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/WebAssembly/instantiateStreaming). - Lazily downloads large features only when used such as `monaco-editor`. - overall bundle size is ~20% smaller (due to bas64 encoding overhead). @@ -64,10 +65,11 @@ module.exports = { }; ``` -`@finos/perspective-viewer` has a dependence on [`monaco-editor`](https://microsoft.github.io/monaco-editor/), -which itself depends on several CSS assets. If your webpack config uses a -loader for `"*.css"` or similar, you may need to exclude `monaco-editor` from -this loader to prevent double-encoding: +`@finos/perspective-viewer` has a dependence on +[`monaco-editor`](https://microsoft.github.io/monaco-editor/), which itself +depends on several CSS assets. If your webpack config uses a loader for +`"*.css"` or similar, you may need to exclude `monaco-editor` from this loader +to prevent double-encoding: ```javascript module.exports = { @@ -78,13 +80,10 @@ module.exports = { { test: /\.css$/, exclude: [/monaco-editor/], // <- Exclude `monaco-editor` - use: [ - {loader: "style-loader"}, - {loader: "css-loader"}, - ], - } - ] - } + use: [{ loader: "style-loader" }, { loader: "css-loader" }], + }, + ], + }, }; ``` @@ -400,7 +399,7 @@ _*index.html*_ > ``` -If you choose not to bundle the themes yourself, they are available through +If you choose not to bundle the themes yourself, they are available through [unpkg.com](https://unpkg.com/@finos/perspective-viewer/dist/umd/). These can be directly linked in your HTML file: diff --git a/docs/md/python.md b/docs/md/python.md index f3b57ae952..949d9ac24e 100644 --- a/docs/md/python.md +++ b/docs/md/python.md @@ -591,7 +591,7 @@ restrictions on memory and CPU feature utilization, and the architecture in general suffers when the dataset itself is too large to download to the client in full. -The Python runtime does not suffer from memory limitations, utilizes -Apache Arrow internal threadpools for threading and parallel processing, and -generates architecture optimized code, which currently makes it more suitable as -a server-side runtime than `node.js`. +The Python runtime does not suffer from memory limitations, utilizes Apache +Arrow internal threadpools for threading and parallel processing, and generates +architecture optimized code, which currently makes it more suitable as a +server-side runtime than `node.js`. diff --git a/docs/package.json b/docs/package.json index 92ff09742f..044cd15c80 100644 --- a/docs/package.json +++ b/docs/package.json @@ -1,7 +1,7 @@ { "name": "@finos/docs", "private": true, - "version": "1.0.8", + "version": "1.1.0", "scripts": { "theme": "lessc static/css/material.dark.less > static/css/material.dark.css", "examples": "docusaurus-examples", @@ -15,11 +15,11 @@ "write-translations": "docusaurus-write-translations" }, "devDependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8", - "@finos/perspective-workspace": "^1.0.8", + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0", + "@finos/perspective-workspace": "^1.1.0", "docusaurus": "^1.8.0", "less": "^3.9.0" }, diff --git a/examples/blocks/package.json b/examples/blocks/package.json index 5541fbeec5..609def8667 100644 --- a/examples/blocks/package.json +++ b/examples/blocks/package.json @@ -1,7 +1,7 @@ { "name": "blocks", "private": true, - "version": "1.0.8", + "version": "1.1.0", "description": "A collection of simple client-side Perspective examples for `http://bl.ocks.org`.", "scripts": { "start": "mkdirp dist && node server.js", @@ -11,10 +11,10 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8", + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0", "superstore-arrow": "1.0.0" } } diff --git a/examples/blocks/src/movies/layout.json b/examples/blocks/src/movies/layout.json index c07228fea5..613df14195 100644 --- a/examples/blocks/src/movies/layout.json +++ b/examples/blocks/src/movies/layout.json @@ -42,13 +42,13 @@ "filter": null, "plugin_config": { "IMDB Rating": { - "color_mode": "gradient", + "number_color_mode": "gradient", "pos_color": "#b87ff0", "neg_color": "#FF9485", "gradient": 8.1 }, "US Gross": { - "color_mode": "gradient", + "number_color_mode": "gradient", "pos_color": "#126e3a", "neg_color": "#FF9485", "gradient": 19729862602 diff --git a/examples/blocks/src/streaming/streaming.js b/examples/blocks/src/streaming/streaming.js index 910d75391b..d2607fc714 100644 --- a/examples/blocks/src/streaming/streaming.js +++ b/examples/blocks/src/streaming/streaming.js @@ -71,15 +71,15 @@ window.addEventListener("DOMContentLoaded", async function () { sort: [["chg", "desc"]], plugin_config: { "(-)chg": { - color_mode: "bar", + number_color_mode: "bar", gradient: 10, }, "(+)chg": { - color_mode: "bar", + number_color_mode: "bar", gradient: 10, }, chg: { - color_mode: "gradient", + number_color_mode: "gradient", gradient: 10, }, }, diff --git a/examples/git-history/package.json b/examples/git-history/package.json index 54ebc698cc..72c732ace4 100644 --- a/examples/git-history/package.json +++ b/examples/git-history/package.json @@ -1,7 +1,7 @@ { "name": "git-history", "private": true, - "version": "1.0.8", + "version": "1.1.0", "description": "An example of Perspective's own GIT history rendered in Perspective.", "scripts": { "start": "node server.js" @@ -9,9 +9,9 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8" + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0" } } diff --git a/examples/promo/package.json b/examples/promo/package.json index 6d388e6886..b12685a63b 100644 --- a/examples/promo/package.json +++ b/examples/promo/package.json @@ -1,7 +1,7 @@ { "name": "promo", "private": true, - "version": "1.0.8", + "version": "1.1.0", "description": "An puppeteer-guided demo of Perspective's functionality, as seen on Github.", "scripts": { "dev": "webpack-dev-server --open", @@ -13,14 +13,14 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8", - "@finos/perspective-workspace": "^1.0.8" + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0", + "@finos/perspective-workspace": "^1.1.0" }, "devDependencies": { - "@finos/perspective-webpack-plugin": "^1.0.8", + "@finos/perspective-webpack-plugin": "^1.1.0", "http-server": "^0.11.1", "npm-run-all": "^4.1.3", "rimraf": "^2.5.2" diff --git a/examples/react-monaco/package.json b/examples/react-monaco/package.json index 2685c667ca..4c6d28e290 100644 --- a/examples/react-monaco/package.json +++ b/examples/react-monaco/package.json @@ -1,7 +1,7 @@ { "name": "react-monaco", "private": true, - "version": "1.0.8", + "version": "1.1.0", "description": "An example app built using `@finos/perspective-viewer`.", "scripts": { "start": "webpack serve --open", @@ -10,16 +10,16 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8", + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0", "react": "16.8.6", "react-dom": "16.8.6", "react-monaco-editor": "^0.46.0" }, "devDependencies": { - "@finos/perspective-webpack-plugin": "^1.0.8", + "@finos/perspective-webpack-plugin": "^1.1.0", "@types/react": "^16.8.6", "@types/react-dom": "^16.9.4", "source-map-loader": "^0.2.4", diff --git a/examples/react/package.json b/examples/react/package.json index 589cf81217..0c73be3a03 100644 --- a/examples/react/package.json +++ b/examples/react/package.json @@ -1,7 +1,7 @@ { "name": "react", "private": true, - "version": "1.0.8", + "version": "1.1.0", "description": "An example app built using `@finos/perspective-viewer`.", "scripts": { "start": "webpack serve --open", @@ -10,15 +10,15 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8", + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0", "react": "16.8.6", "react-dom": "16.8.6" }, "devDependencies": { - "@finos/perspective-webpack-plugin": "^1.0.8", + "@finos/perspective-webpack-plugin": "^1.1.0", "@types/react": "^16.8.6", "@types/react-dom": "^16.9.4", "source-map-loader": "^0.2.4", diff --git a/examples/remote-express/package.json b/examples/remote-express/package.json index 71593d7f21..34e9c741e3 100644 --- a/examples/remote-express/package.json +++ b/examples/remote-express/package.json @@ -1,7 +1,7 @@ { "name": "remote-express", "private": true, - "version": "1.0.8", + "version": "1.1.0", "description": "An example of 2 Perspectives, one client and one server, streaming via Apache Arrow.", "scripts": { "start": "tsc && node dist/server.js" @@ -9,10 +9,10 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8", + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0", "express": "^4.17.1", "express-ws": "^5.0.2" }, diff --git a/examples/remote-workspace/package.json b/examples/remote-workspace/package.json index 13e81c8335..c6b3ce3b4a 100644 --- a/examples/remote-workspace/package.json +++ b/examples/remote-workspace/package.json @@ -1,7 +1,7 @@ { "name": "remote-workspace", "private": true, - "version": "1.0.8", + "version": "1.1.0", "description": "An example app built using `@finos/perspective-workspace`.", "scripts": { "start:server": "webpack serve --open", @@ -11,14 +11,14 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8", - "@finos/perspective-workspace": "^1.0.8" + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0", + "@finos/perspective-workspace": "^1.1.0" }, "devDependencies": { - "@finos/perspective-webpack-plugin": "^1.0.8", + "@finos/perspective-webpack-plugin": "^1.1.0", "http-server": "^0.11.1", "npm-run-all": "^4.1.3", "rimraf": "^2.5.2" diff --git a/examples/tornado-python/client_server_editing.html b/examples/tornado-python/client_server_editing.html index 4490c48190..7dfe153c94 100644 --- a/examples/tornado-python/client_server_editing.html +++ b/examples/tornado-python/client_server_editing.html @@ -53,11 +53,11 @@ // Connect to the websocket and create a new webworker const websocket = perspective.websocket("ws://localhost:8080/websocket"); - const worker = perspective.shared_worker(); + const worker = perspective.worker(); // Get handles to the `Table` on the server, and create a // `view()` on the server. - const server_table = websocket.open_table("data_source_one"); + const server_table = await websocket.open_table("data_source_one"); const server_view = await server_table.view(); // Serialize the current state of the view to an arrow, and create @@ -69,7 +69,7 @@ const client_table = await worker.table(arrow, {index: "Row ID"}); const client_view = await client_table.view();// client -> server - await viewer.load(client_table); + await viewer.load(Promise.resolve(client_table)); // Create ports on both the client and the server - this allows // the client and server to track which updates are coming from diff --git a/examples/tornado-python/package.json b/examples/tornado-python/package.json index ff5639efd5..3fae73daec 100644 --- a/examples/tornado-python/package.json +++ b/examples/tornado-python/package.json @@ -1,7 +1,7 @@ { "name": "tornado-python", "private": true, - "version": "1.0.8", + "version": "1.1.0", "description": "An example of editing a `perspective-python` server from the browser.", "scripts": { "start": "PYTHONPATH=../../python/perspective python3 server.py", @@ -10,15 +10,15 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8", - "@finos/perspective-workspace": "^1.0.8", + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0", + "@finos/perspective-workspace": "^1.1.0", "superstore-arrow": "^1.0.0" }, "devDependencies": { - "@finos/perspective-webpack-plugin": "^1.0.8", + "@finos/perspective-webpack-plugin": "^1.1.0", "npm-run-all": "^4.1.3", "rimraf": "^2.5.2" } diff --git a/examples/tornado-streaming-python/package.json b/examples/tornado-streaming-python/package.json index 5130956f28..d0fc628d36 100644 --- a/examples/tornado-streaming-python/package.json +++ b/examples/tornado-streaming-python/package.json @@ -1,7 +1,7 @@ { "name": "tornado-streaming-python", "private": true, - "version": "1.0.8", + "version": "1.1.0", "description": "An example of streaming a `perspective-python` server to the browser.", "scripts": { "start": "PYTHONPATH=../../python/perspective python3 server.py", @@ -10,15 +10,15 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8", - "@finos/perspective-workspace": "^1.0.8", + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0", + "@finos/perspective-workspace": "^1.1.0", "superstore-arrow": "^1.0.0" }, "devDependencies": { - "@finos/perspective-webpack-plugin": "^1.0.8", + "@finos/perspective-webpack-plugin": "^1.1.0", "npm-run-all": "^4.1.3", "rimraf": "^2.5.2" } diff --git a/examples/webpack-cross-origin/package.json b/examples/webpack-cross-origin/package.json index 75fd2808f9..ed89ac778d 100644 --- a/examples/webpack-cross-origin/package.json +++ b/examples/webpack-cross-origin/package.json @@ -1,7 +1,7 @@ { "name": "webpack-cross-origin", "private": true, - "version": "1.0.8", + "version": "1.1.0", "description": "An example of using the Perspective Webpack plugin to build a JS file with Webpack.", "scripts": { "start": "npm-run-all -l -p webpack-watch host:app host:bundles", @@ -12,13 +12,13 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8" + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0" }, "devDependencies": { - "@finos/perspective-webpack-plugin": "^1.0.8", + "@finos/perspective-webpack-plugin": "^1.1.0", "http-server": "^0.11.1" } } diff --git a/examples/webpack-example/package.json b/examples/webpack-example/package.json index e49b0e4769..b7045fd6e6 100644 --- a/examples/webpack-example/package.json +++ b/examples/webpack-example/package.json @@ -1,7 +1,7 @@ { "name": "webpack-example", "private": true, - "version": "1.0.8", + "version": "1.1.0", "description": "An example app built using `@finos/perspective-viewer`.", "scripts": { "start": "webpack serve" @@ -9,13 +9,13 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8" + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0" }, "devDependencies": { - "@finos/perspective-webpack-plugin": "^1.0.8", + "@finos/perspective-webpack-plugin": "^1.1.0", "css-loader": "^0.28.7", "style-loader": "^0.18.2", "webpack": "^5.14.0", diff --git a/examples/workspace-editing-python/package.json b/examples/workspace-editing-python/package.json index 89364aadda..a06fb9be80 100644 --- a/examples/workspace-editing-python/package.json +++ b/examples/workspace-editing-python/package.json @@ -1,7 +1,7 @@ { "name": "workspace-editing-python", "private": true, - "version": "1.0.8", + "version": "1.1.0", "description": "An example app demonstrating client/server editing, built using `@finos/perspective-workspace` and `perspective-python`.", "scripts": { "start": "yarn webpack && yarn start:server", @@ -12,14 +12,14 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8", - "@finos/perspective-workspace": "^1.0.8" + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0", + "@finos/perspective-workspace": "^1.1.0" }, "devDependencies": { - "@finos/perspective-webpack-plugin": "^1.0.8", + "@finos/perspective-webpack-plugin": "^1.1.0", "http-server": "^0.11.1", "npm-run-all": "^4.1.3", "rimraf": "^2.5.2" diff --git a/examples/workspace/package.json b/examples/workspace/package.json index 08cc9626ce..0a073a9be6 100644 --- a/examples/workspace/package.json +++ b/examples/workspace/package.json @@ -1,7 +1,7 @@ { "name": "workspace", "private": true, - "version": "1.0.8", + "version": "1.1.0", "description": "An example app built using `@finos/perspective-workspace`.", "scripts": { "start": "webpack serve --open", @@ -10,14 +10,14 @@ "keywords": [], "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8", - "@finos/perspective-workspace": "^1.0.8" + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0", + "@finos/perspective-workspace": "^1.1.0" }, "devDependencies": { - "@finos/perspective-webpack-plugin": "^1.0.8", + "@finos/perspective-webpack-plugin": "^1.1.0", "http-server": "^0.11.1", "npm-run-all": "^4.1.3", "rimraf": "^2.5.2" diff --git a/lerna.json b/lerna.json index 14189be9f4..b01e28fd18 100644 --- a/lerna.json +++ b/lerna.json @@ -6,5 +6,5 @@ ], "npmClient": "yarn", "useWorkspaces": true, - "version": "1.0.8" + "version": "1.1.0" } diff --git a/packages/perspective-cli/package.json b/packages/perspective-cli/package.json index 5e1301cf20..662fb39707 100644 --- a/packages/perspective-cli/package.json +++ b/packages/perspective-cli/package.json @@ -1,6 +1,6 @@ { "name": "@finos/perspective-cli", - "version": "1.0.8", + "version": "1.1.0", "description": "Perspective.js CLI", "main": "src/js/index.js", "publishConfig": { @@ -25,10 +25,10 @@ "perspective": "perspective" }, "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8", + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0", "commander": "^2.19.0", "puppeteer": "^10.2.0" } diff --git a/packages/perspective-jupyterlab/package.json b/packages/perspective-jupyterlab/package.json index 1043a907c3..0c5b40fb68 100644 --- a/packages/perspective-jupyterlab/package.json +++ b/packages/perspective-jupyterlab/package.json @@ -1,6 +1,6 @@ { "name": "@finos/perspective-jupyterlab", - "version": "1.0.8", + "version": "1.1.0", "description": "A Jupyterlab extension for the Perspective library, designed to be used with perspective-python.", "files": [ "dist/**/*", @@ -31,18 +31,18 @@ "version": "yarn build" }, "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", - "@finos/perspective-viewer-d3fc": "^1.0.8", - "@finos/perspective-viewer-datagrid": "^1.0.8", + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", + "@finos/perspective-viewer-d3fc": "^1.1.0", + "@finos/perspective-viewer-datagrid": "^1.1.0", "@jupyter-widgets/base": "^3.0.0 || ^4.0.0", "@jupyterlab/application": "^3.0.0", "@lumino/application": "^1.7.3", "@lumino/widgets": "^1.9.3" }, "devDependencies": { - "@finos/perspective-build": "^1.0.8", - "@finos/perspective-test": "^1.0.8", + "@finos/perspective-build": "^1.1.0", + "@finos/perspective-test": "^1.1.0", "@jupyter-widgets/base-manager": "^1.0.0-alpha.0" }, "jupyterlab": { diff --git a/packages/perspective-jupyterlab/src/less/index.less b/packages/perspective-jupyterlab/src/less/index.less index 4d5cda7103..c78a1826f7 100644 --- a/packages/perspective-jupyterlab/src/less/index.less +++ b/packages/perspective-jupyterlab/src/less/index.less @@ -13,8 +13,8 @@ div.PSPContainer-dark { flex: 1; } -perspective-column-style.dark { - .perspective-column-style-material-dark(); +perspective-number-column-style.dark { + .perspective-number-column-style-material-dark(); } perspective-expression-editor.dark { diff --git a/packages/perspective-jupyterlab/test/results/results.json b/packages/perspective-jupyterlab/test/results/results.json index 1db19fb745..81f8c0a310 100644 --- a/packages/perspective-jupyterlab/test/results/results.json +++ b/packages/perspective-jupyterlab/test/results/results.json @@ -1,6 +1,6 @@ { - "__GIT_COMMIT__": "21eb6e6303888b41ec49891317c4302f93515335", - "resize_Config_should_show_by_default": "b8bc3ea0c100cbd12f2b6364d13ad2cc", - "resize_Resize_the_container_causes_the_widget_to_resize": "b8bc3ea0c100cbd12f2b6364d13ad2cc", - "resize_row_pivots_traitlet_works": "52d03e6b2e62254a8135c0450d6ef1f2" + "__GIT_COMMIT__": "8f265c2384ba042f164f1c4ec59689890e9878e4", + "resize_Config_should_show_by_default": "52615e755c3626470ba4109bfa5a2c1f", + "resize_Resize_the_container_causes_the_widget_to_resize": "52615e755c3626470ba4109bfa5a2c1f", + "resize_row_pivots_traitlet_works": "c58e40d46bc2161fabb49d68ab84f366" } \ No newline at end of file diff --git a/packages/perspective-viewer-d3fc/package.json b/packages/perspective-viewer-d3fc/package.json index 89826ef1a4..7fef230704 100644 --- a/packages/perspective-viewer-d3fc/package.json +++ b/packages/perspective-viewer-d3fc/package.json @@ -1,6 +1,6 @@ { "name": "@finos/perspective-viewer-d3fc", - "version": "1.0.8", + "version": "1.1.0", "description": "Perspective.js D3FC Plugin", "unpkg": "./dist/cdn/perspective-viewer-d3fc.js", "jsdelivr": "./dist/cdn/perspective-viewer-d3fc.js", @@ -52,8 +52,8 @@ "author": "", "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", "chroma-js": "^1.3.4", "d3": "^7.1.1", "d3-svg-legend": "^2.25.6", @@ -61,6 +61,6 @@ "gradient-parser": "1.0.2" }, "devDependencies": { - "@finos/perspective-test": "^1.0.8" + "@finos/perspective-test": "^1.1.0" } } diff --git a/packages/perspective-viewer-datagrid/package.json b/packages/perspective-viewer-datagrid/package.json index 6de601e6dc..2d0562b5a5 100644 --- a/packages/perspective-viewer-datagrid/package.json +++ b/packages/perspective-viewer-datagrid/package.json @@ -1,6 +1,6 @@ { "name": "@finos/perspective-viewer-datagrid", - "version": "1.0.8", + "version": "1.1.0", "description": "Perspective datagrid plugin based on `regular-table`", "unpkg": "dist/cdn/perspective-viewer-datagrid.js", "jsdelivr": "dist/cdn/perspective-viewer-datagrid.js", @@ -37,13 +37,13 @@ "author": "", "license": "Apache-2.0", "dependencies": { - "@finos/perspective": "^1.0.8", - "@finos/perspective-viewer": "^1.0.8", + "@finos/perspective": "^1.1.0", + "@finos/perspective-viewer": "^1.1.0", "chroma-js": "^1.3.4", "regular-table": "=0.4.2" }, "devDependencies": { - "@finos/perspective-build": "^1.0.8", - "@finos/perspective-test": "^1.0.8" + "@finos/perspective-build": "^1.1.0", + "@finos/perspective-test": "^1.1.0" } } diff --git a/packages/perspective-viewer-datagrid/src/js/editing.js b/packages/perspective-viewer-datagrid/src/js/editing.js index c7e3467eec..ed64f4c49e 100644 --- a/packages/perspective-viewer-datagrid/src/js/editing.js +++ b/packages/perspective-viewer-datagrid/src/js/editing.js @@ -8,7 +8,7 @@ */ const selected_position_map = new WeakMap(); - +import {PLUGIN_SYMBOL} from "./plugin_menu.js"; function lock(body) { let lock; return async function (...args) { @@ -122,12 +122,17 @@ function editableStyleListener(table, viewer) { if (!viewer.hasAttribute("editable")) { return; } + const plugins = table[PLUGIN_SYMBOL] || {}; const edit = isEditable.call(this, viewer); for (const td of table.querySelectorAll("td")) { const meta = table.getMeta(td); const type = this.get_psp_type(meta); if (this._is_editable[meta.x]) { - if (type === "boolean") { + const col_name = meta.column_header[meta.column_header.length - 1]; + if (type === "string" && plugins[col_name]?.format === "link") { + td.toggleAttribute("contenteditable", false); + td.classList.toggle("boolean-editable", false); + } else if (type === "boolean") { td.toggleAttribute("contenteditable", false); td.classList.toggle("boolean-editable", meta.user !== null); } else { diff --git a/packages/perspective-viewer-datagrid/src/js/plugin.js b/packages/perspective-viewer-datagrid/src/js/plugin.js index 2d2205554f..26e2e1d321 100644 --- a/packages/perspective-viewer-datagrid/src/js/plugin.js +++ b/packages/perspective-viewer-datagrid/src/js/plugin.js @@ -117,6 +117,11 @@ customElements.define( config.pos_color = config.pos_color[0]; config.neg_color = config.neg_color[0]; } + + if (config?.color) { + config.color = config.color[0]; + } + token[col] = config; } @@ -152,6 +157,10 @@ customElements.define( config.neg_color = create_color_record(config.neg_color); } + if (config?.color) { + config.color = create_color_record(config.color); + } + if (Object.keys(config).length === 0) { delete token[col]; } diff --git a/packages/perspective-viewer-datagrid/src/js/plugin_menu.js b/packages/perspective-viewer-datagrid/src/js/plugin_menu.js index c9195ffb34..a8c7545f75 100644 --- a/packages/perspective-viewer-datagrid/src/js/plugin_menu.js +++ b/packages/perspective-viewer-datagrid/src/js/plugin_menu.js @@ -11,8 +11,6 @@ import chroma from "chroma-js"; export const PLUGIN_SYMBOL = Symbol("Plugin Symbol"); -let MENU = undefined; - export function make_gradient(chromahex) { const [r, g, b] = chromahex.rgb(); const [r1, g1, b1] = chromahex @@ -25,25 +23,37 @@ export function make_gradient(chromahex) { } export function activate_plugin_menu(regularTable, target, column_max) { - MENU = MENU || document.createElement("perspective-column-style"); + const is_numeric = typeof column_max !== "undefined"; + const MENU = document.createElement( + `perspective-${is_numeric ? "number" : "string"}-column-style` + ); const target_meta = regularTable.getMeta(target); const column_name = target_meta.column_header[target_meta.column_header.length - 1]; const column_type = this._schema[column_name]; - const default_config = { - gradient: column_max, - pos_color: this._pos_color[0], - neg_color: this._neg_color[0], - color_mode: "foreground", - }; + let default_config; + if (is_numeric) { + default_config = { + gradient: column_max, + pos_color: this._pos_color[0], + neg_color: this._neg_color[0], + number_color_mode: "foreground", + }; + } else { + default_config = { + color: this._color[0], + }; + } - if (column_type === "float") { + if (column_type === "string") { + // do nothing + } else if (column_type === "float") { default_config.fixed = 2; } else if (column_type === "integer") { default_config.fixed = 0; } else { this._open_column_styles_menu.pop(); - regularTable.draw(); + regularTable.draw({preserve_width: true}); return; } @@ -63,6 +73,14 @@ export function activate_plugin_menu(regularTable, target, column_max) { ]; } + if (config.color) { + config.color = [ + config.color, + ...chroma(config.color).rgb(), + make_gradient(chroma(config.color)), + ]; + } + regularTable[PLUGIN_SYMBOL] = regularTable[PLUGIN_SYMBOL] || {}; regularTable[PLUGIN_SYMBOL][column_name] = config; regularTable.draw({preserve_width: true}); @@ -81,8 +99,9 @@ export function activate_plugin_menu(regularTable, target, column_max) { update_handler ); MENU.removeEventListener("blur", blur_handler); + MENU.destroy(); this._open_column_styles_menu.pop(); - await regularTable.draw(); + await regularTable.draw({preserve_width: true}); regularTable.parentElement.dispatchEvent( new Event("perspective-config-update") ); @@ -98,11 +117,15 @@ export function activate_plugin_menu(regularTable, target, column_max) { {}, (pset[column_name] = pset[column_name] || {}) ); + if (config.pos_color) { config.pos_color = config.pos_color[0]; config.neg_color = config.neg_color[0]; } - // open the menu + if (config.color) { + config.color = config.color[0]; + } + MENU.open(target, config, default_config); } diff --git a/packages/perspective-viewer-datagrid/src/js/regular_table_handlers.js b/packages/perspective-viewer-datagrid/src/js/regular_table_handlers.js index a7d1aa22a8..2a0d64fbbb 100644 --- a/packages/perspective-viewer-datagrid/src/js/regular_table_handlers.js +++ b/packages/perspective-viewer-datagrid/src/js/regular_table_handlers.js @@ -63,13 +63,17 @@ function styleListener(regularTable) { let type = get_psp_type.call(this, metadata); const is_numeric = type === "integer" || type === "float"; + const is_string = type === "string"; td.classList.toggle("psp-align-right", is_numeric); td.classList.toggle("psp-align-left", !is_numeric); td.classList.toggle( "psp-menu-open", this._open_column_styles_menu[0] === metadata._virtual_x ); - td.classList.toggle("psp-menu-enabled", is_numeric && !is_corner); + td.classList.toggle( + "psp-menu-enabled", + (is_string || is_numeric) && !is_corner + ); td.classList.toggle( "psp-is-width-override", regularTable._column_sizes?.override[metadata.size_key] !== @@ -130,9 +134,9 @@ function styleListener(regularTable) { const metadata = regularTable.getMeta(td); const column_name = metadata.column_header?.[metadata.column_header?.length - 1]; - const plugin = plugins[column_name]; let type = get_psp_type.call(this, metadata); + const plugin = plugins[column_name]; const is_numeric = type === "integer" || type === "float"; if (is_numeric) { @@ -155,14 +159,14 @@ function styleListener(regularTable) { })(); td.style.position = ""; - if (plugin?.color_mode === "background") { + if (plugin?.number_color_mode === "background") { const source = this._plugin_background; const foreground = infer_foreground_from_background( rgbaToRgb([r, g, b, 1], source) ); td.style.color = foreground; td.style.backgroundColor = hex; - } else if (plugin?.color_mode === "gradient") { + } else if (plugin?.number_color_mode === "gradient") { const a = Math.max( 0, Math.min(1, Math.abs(metadata.user / plugin.gradient)) @@ -171,12 +175,13 @@ function styleListener(regularTable) { const foreground = infer_foreground_from_background( rgbaToRgb([r, g, b, a], source) ); + td.style.color = foreground; td.style.backgroundColor = `rgba(${r},${g},${b},${a})`; - } else if (plugin?.color_mode === "disabled") { + } else if (plugin?.number_color_mode === "disabled") { td.style.backgroundColor = ""; td.style.color = ""; - } else if (plugin?.color_mode === "bar") { + } else if (plugin?.number_color_mode === "bar") { td.style.backgroundColor = ""; td.style.color = ""; td.style.position = "relative"; @@ -201,6 +206,66 @@ function styleListener(regularTable) { td.style.backgroundColor = ""; td.style.color = hex; + } else if (type === "string") { + const [hex, r, g, b, gradhex] = (() => { + if (plugin?.color !== undefined) { + return plugin.color; + } else { + return this._color; + } + })(); + + if ( + plugin?.string_color_mode === "foreground" && + metadata.user !== null + ) { + td.style.color = hex; + td.style.backgroundColor = ""; + if (plugin?.format === "link") { + td.children[0].style.color = hex; + } + } else if ( + plugin?.string_color_mode === "background" && + metadata.user !== null + ) { + const source = this._plugin_background; + const foreground = infer_foreground_from_background( + rgbaToRgb([r, g, b, 1], source) + ); + td.style.color = foreground; + td.style.backgroundColor = hex; + } else if ( + plugin?.string_color_mode === "series" && + metadata.user !== null + ) { + if (!this._series_color_map.has(column_name)) { + this._series_color_map.set(column_name, new Map()); + this._series_color_seed.set(column_name, 0); + } + + const series_map = this._series_color_map.get(column_name); + if (!series_map.has(metadata.user)) { + const seed = this._series_color_seed.get(column_name); + series_map.set(metadata.user, seed); + this._series_color_seed.set(column_name, seed + 1); + } + + const color_seed = series_map.get(metadata.user); + let [h, s, l] = chroma(hex).hsl(); + h = h + ((color_seed * 150) % 360); + const color2 = chroma(h, s, l, "hsl"); + const [r, g, b] = color2.rgb(); + const hex2 = color2.hex(); + const source = this._plugin_background; + const foreground = infer_foreground_from_background( + rgbaToRgb([r, g, b, 1], source) + ); + td.style.color = foreground; + td.style.backgroundColor = hex2; + } else { + td.style.backgroundColor = ""; + td.style.color = ""; + } } else { td.style.backgroundColor = ""; td.style.color = ""; @@ -244,7 +309,7 @@ function styleListener(regularTable) { td.classList.toggle("psp-align-left", is_th || !is_numeric); td.classList.toggle( "psp-color-mode-bar", - plugin?.color_mode === "bar" + plugin?.number_color_mode === "bar" && is_numeric ); } } @@ -335,6 +400,10 @@ async function mousedownListener(regularTable, event) { } let target = event.target; + if (target.tagName === "A") { + return; + } + while (target.tagName !== "TD" && target.tagName !== "TH") { target = target.parentElement; if (!regularTable.contains(target)) { @@ -356,10 +425,17 @@ async function mousedownListener(regularTable, event) { const meta = regularTable.getMeta(target); const column_name = meta.column_header?.[meta.column_header?.length - 1]; - const [, max] = await this._view.get_min_max(column_name); + const column_type = this._schema[column_name]; this._open_column_styles_menu.unshift(meta._virtual_x); - regularTable.draw(); - activate_plugin_menu.call(this, regularTable, target, max); + if (column_type === "string") { + regularTable.draw({preserve_width: true}); + activate_plugin_menu.call(this, regularTable, target); + } else { + const [, max] = await this._view.get_min_max(column_name); + regularTable.draw({preserve_width: true}); + activate_plugin_menu.call(this, regularTable, target, max); + } + event.preventDefault(); event.stopImmediatePropagation(); } else if ( @@ -416,13 +492,13 @@ function _format(parts, val, plugins = {}, use_table_schema = false) { } const title = parts[parts.length - 1]; - const plugin = plugins[title]; const type = (use_table_schema && this._table_schema[title]) || this._schema[title] || "string"; + const plugin = plugins[title]; const is_numeric = type === "integer" || type === "float"; - if (is_numeric && plugin?.color_mode === "bar") { + if (is_numeric && plugin?.number_color_mode === "bar") { const a = Math.max( 0, Math.min(0.95, Math.abs(val / plugin.gradient) * 0.95) @@ -436,6 +512,20 @@ function _format(parts, val, plugins = {}, use_table_schema = false) { )}%;position:absolute;${anchor}:0;height:80%;top:10%;pointer-events:none;` ); return div; + } else if (plugin?.format === "link" && type === "string") { + const anchor = document.createElement("a"); + anchor.setAttribute("href", val); + anchor.setAttribute("target", "_blank"); + anchor.textContent = val; + return anchor; + } else if (plugin?.format === "bold" && type === "string") { + const anchor = document.createElement("b"); + anchor.textContent = val; + return anchor; + } else if (plugin?.format === "italics" && type === "string") { + const anchor = document.createElement("i"); + anchor.textContent = val; + return anchor; } else { const is_plugin_override = is_numeric && plugin && plugin.fixed !== undefined; @@ -482,7 +572,13 @@ function* _tree_header(paths = [], row_headers, regularTable) { plugins, true ); - path = path.concat({toString: () => formatted}); + + if (formatted instanceof HTMLElement) { + path = path.concat(formatted); + } else { + path = path.concat({toString: () => formatted}); + } + path.length = row_headers.length + 1; yield path; } @@ -605,6 +701,9 @@ export async function createModel(regular, table, view, extend = {}) { const _neg_color = create_color_record( get_rule(regular, "--rt-neg-cell--color", "#FF5942") ); + const _color = create_color_record( + get_rule(regular, "--active--color", "#ff0000") + ); const _schema = {...schema, ...expression_schema}; const _table_schema = { ...table_schema, @@ -634,6 +733,7 @@ export async function createModel(regular, table, view, extend = {}) { _ids: [], _open_column_styles_menu: [], _plugin_background, + _color, _pos_color, _neg_color, _column_paths, @@ -642,6 +742,8 @@ export async function createModel(regular, table, view, extend = {}) { _row_header_types: config.row_pivots.map((column_path) => { return _table_schema[column_path]; }), + _series_color_map: new Map(), + _series_color_seed: new Map(), get_psp_type, }); diff --git a/packages/perspective-viewer-datagrid/src/js/row_selection.js b/packages/perspective-viewer-datagrid/src/js/row_selection.js index 3b0e28cd89..3a23390b1d 100644 --- a/packages/perspective-viewer-datagrid/src/js/row_selection.js +++ b/packages/perspective-viewer-datagrid/src/js/row_selection.js @@ -39,7 +39,7 @@ async function selectionListener(regularTable, viewer, event) { filter = filter.config.filter; } - await regularTable.draw(); + await regularTable.draw({preserve_width: true}); event.handled = true; viewer.dispatchEvent( new CustomEvent("perspective-select", { diff --git a/packages/perspective-viewer-datagrid/src/less/material.less b/packages/perspective-viewer-datagrid/src/less/material.less index d5324ce63e..e7a668cb0c 100644 --- a/packages/perspective-viewer-datagrid/src/less/material.less +++ b/packages/perspective-viewer-datagrid/src/less/material.less @@ -196,6 +196,14 @@ regular-table { cursor: col-resize; } + a { + color: var(--rt-pos-cell--color); + } + + a:visited { + color: var(--active--color); + } + // webkit (chrome, safari, etc) scrollbar styling &::-webkit-scrollbar, diff --git a/packages/perspective-viewer-datagrid/test/results/results.json b/packages/perspective-viewer-datagrid/test/results/results.json index 842929c601..2484ad8529 100644 --- a/packages/perspective-viewer-datagrid/test/results/results.json +++ b/packages/perspective-viewer-datagrid/test/results/results.json @@ -11,20 +11,20 @@ "superstore_filters_by_a_datetime_column_": "4f3ebe78d2092c8c6f74389778d764d5", "superstore_highlights_invalid_filter_": "d74bde78c8afbd835e451fd5afffc9fe", "superstore_sorts_by_an_alpha_column_": "e404bb87320c31ec31a54c8652ae439c", - "superstore_displays_visible_columns_": "d7cfabd1879d62859ce4fa057ed0a9d1", + "superstore_displays_visible_columns_": "0b95713a81bc8af9b2db97111cdba9fa", "superstore_resets_viewable_area_when_the_logical_size_expands_": "e9dd1d275f46f6a0857e0168835d0b38", "superstore_resets_viewable_area_when_the_physical_size_expands_": "e469597235cc032619bc095a76f9ad6f", - "__GIT_COMMIT__": "21eb6e6303888b41ec49891317c4302f93515335", - "superstore_shows_a_grid_without_any_settings_applied": "9ba7d100dc0f3e91a61668ac2f2ddabb", - "superstore_pivot_by_a_row": "46a0c0d3e44cbc2805758dbe40285112", - "superstore_pivot_by_two_rows": "929a59c221687110b49cfdf7df2c353c", - "superstore_pivot_by_a_column": "72448dcee7942e8a294db0b9465d4d2f", - "superstore_pivot_by_a_row_and_a_column": "9f018e67263ad41268987dcfcd751352", - "superstore_pivot_by_two_rows_and_two_columns": "cdaa134913bd6cae25d335ec477d5606", - "superstore_sort_by_a_hidden_column": "91456482bc01f14f0055a9a15e38fb37", - "superstore_sort_by_a_numeric_column": "623a3892f50b098071d7633a9e2ca1f6", - "superstore_sort_by_an_alpha_column": "c897199ae28dcec14c82e9f9e055f5a6", - "superstore_filters_filters_by_a_numeric_column": "1cc8b3b10e743899652717ec128dd156", - "superstore_filters_filters_by_an_alpha_column": "adef8a48d326237cb8af5d18a20f6ae7", - "superstore_filters_filters_with__in__comparator": "a013357125cb572b9181abb52dab0bb2" + "__GIT_COMMIT__": "6f5924eaf2b570a1d7d2f4883dc94f9a62462b9b", + "superstore_shows_a_grid_without_any_settings_applied": "2bf7d29b6483e16ad1ff65a1814f2230", + "superstore_pivot_by_a_row": "dac293d98f67757eb6f792f4d163c0d2", + "superstore_pivot_by_two_rows": "a5a7bb497fe76ae7c344192d498c64da", + "superstore_pivot_by_a_column": "13d1dbd9b8fa97a9053b111a93444d87", + "superstore_pivot_by_a_row_and_a_column": "f60df0f388a9840f63bbc17eea282b71", + "superstore_pivot_by_two_rows_and_two_columns": "11baf034141da27ef9038f35167bd1f5", + "superstore_sort_by_a_hidden_column": "a88f622d32cd13a6b1c330668eabd9ac", + "superstore_sort_by_a_numeric_column": "db67040f6d638a0c889c0cffb3832057", + "superstore_sort_by_an_alpha_column": "9e7a551308497ddca3e1b058cc69ba8c", + "superstore_filters_filters_by_a_numeric_column": "ce3bed9620c8fde9929d3a3ed548279c", + "superstore_filters_filters_by_an_alpha_column": "5cac36a354a18935179afe6b8c3c10da", + "superstore_filters_filters_with__in__comparator": "d7047b2da9afe8df7e7f8ceebea13f43" } \ No newline at end of file diff --git a/packages/perspective-webpack-plugin/package.json b/packages/perspective-webpack-plugin/package.json index 63c874d9db..d91a8a59c8 100644 --- a/packages/perspective-webpack-plugin/package.json +++ b/packages/perspective-webpack-plugin/package.json @@ -1,6 +1,6 @@ { "name": "@finos/perspective-webpack-plugin", - "version": "1.0.8", + "version": "1.1.0", "description": "Perspective.js Webpack Plugin", "main": "index.js", "publishConfig": { diff --git a/packages/perspective-workspace/package.json b/packages/perspective-workspace/package.json index 0ed1db4a91..12692b7261 100644 --- a/packages/perspective-workspace/package.json +++ b/packages/perspective-workspace/package.json @@ -1,6 +1,6 @@ { "name": "@finos/perspective-workspace", - "version": "1.0.8", + "version": "1.1.0", "description": "Perspective Workspace", "files": [ "dist/**/*", @@ -42,7 +42,7 @@ "author": "", "license": "Apache-2.0", "dependencies": { - "@finos/perspective-viewer": "^1.0.8", + "@finos/perspective-viewer": "^1.1.0", "@lumino/algorithm": "1.3.3", "@lumino/commands": "1.12.0", "@lumino/domutils": "1.2.3", @@ -52,7 +52,7 @@ "lodash": "^4.17.4" }, "devDependencies": { - "@finos/perspective-build": "^1.0.8", - "@finos/perspective-test": "^1.0.8" + "@finos/perspective-build": "^1.1.0", + "@finos/perspective-test": "^1.1.0" } } diff --git a/packages/perspective-workspace/src/themes/material.dark.less b/packages/perspective-workspace/src/themes/material.dark.less index 834bb33854..6202f3baf8 100644 --- a/packages/perspective-workspace/src/themes/material.dark.less +++ b/packages/perspective-workspace/src/themes/material.dark.less @@ -24,7 +24,7 @@ perspective-viewer.workspace-master-widget { --plugin--background: @grey800; } -perspective-column-style, +perspective-number-column-style, perspective-expression-editor { background-color: @grey700; color: #ffffff; diff --git a/packages/perspective-workspace/src/themes/material.less b/packages/perspective-workspace/src/themes/material.less index 11fb3ece9f..e6111951f1 100644 --- a/packages/perspective-workspace/src/themes/material.less +++ b/packages/perspective-workspace/src/themes/material.less @@ -24,7 +24,7 @@ perspective-viewer.workspace-master-widget { } } -perspective-column-style, +perspective-number-column-style, perspective-expression-editor { background-color: #ffffff; color: #161616; diff --git a/packages/perspective-workspace/test/js/integration/restore.spec.js b/packages/perspective-workspace/test/js/integration/restore.spec.js index 7499764e84..551120e996 100644 --- a/packages/perspective-workspace/test/js/integration/restore.spec.js +++ b/packages/perspective-workspace/test/js/integration/restore.spec.js @@ -74,46 +74,44 @@ function tests(extract) { return extract(page); }); - test.capture( - "restore workspace with viewers with generated slotids", - async (page) => { - const config = { - viewers: { - PERSPECTIVE_GENERATED_ID_0: { - table: "superstore", - name: "Test", - row_pivots: ["State"], - columns: ["Sales", "Profit"], - }, + // This test flaps constantly due to mis-ordered HTML attributes and I don't + // want to fix it for the value it provides. + test.skip("restore workspace with viewers with generated slotids", async (page) => { + const config = { + viewers: { + PERSPECTIVE_GENERATED_ID_0: { + table: "superstore", + name: "Test", + row_pivots: ["State"], + columns: ["Sales", "Profit"], }, - detail: { - main: { - currentIndex: 0, - type: "tab-area", - widgets: ["PERSPECTIVE_GENERATED_ID_0"], - }, + }, + detail: { + main: { + currentIndex: 0, + type: "tab-area", + widgets: ["PERSPECTIVE_GENERATED_ID_0"], }, - }; + }, + }; - await page.evaluate(async (config) => { - const workspace = document.getElementById("workspace"); - await workspace.restore(config); - }, config); + await page.evaluate(async (config) => { + const workspace = document.getElementById("workspace"); + await workspace.restore(config); + }, config); - await page.evaluate(async () => { - const workspace = - document.getElementById("workspace").workspace; - const widget = workspace.getAllWidgets()[0]; - await workspace.duplicate(widget); - }); + await page.evaluate(async () => { + const workspace = document.getElementById("workspace").workspace; + const widget = workspace.getAllWidgets()[0]; + await workspace.duplicate(widget); + }); - await page.evaluate(async () => { - await workspace.notifyResize(true); - }); + await page.evaluate(async () => { + await workspace.flush(); + }); - return extract(page); - } - ); + return extract(page); + }); } utils.with_server({paths: PATHS}, () => { diff --git a/packages/perspective-workspace/test/results/results.json b/packages/perspective-workspace/test/results/results.json index 5c09b4b084..dec0527a91 100644 --- a/packages/perspective-workspace/test/results/results.json +++ b/packages/perspective-workspace/test/results/results.json @@ -1,17 +1,17 @@ { - "__GIT_COMMIT__": "2c503d4283c893ff40e0ee3644a53df9b654f7ea", + "__GIT_COMMIT__": "dbab35e511c439fc33839744cf526c903256142d", "index_restore_workspace_with_detail_only": "d24f601369fbf86c853d4dd2894506e3", - "index_Light_DOM_restore_workspace_with_detail_only": "d24f601369fbf86c853d4dd2894506e3", + "index_Light_DOM_restore_workspace_with_detail_only": "935f2b58466497650703544fe11a0d55", "index_Shadow_DOM_restore_workspace_with_detail_only": "912377d649971a7d3d4d8444f559f8b6", - "index_Light_DOM_restore_workspace_with_master_and_detail": "9a38de209753293ccdf067ddc1cae12d", + "index_Light_DOM_restore_workspace_with_master_and_detail": "56b9c370f1031097414462c352d35bf0", "index_Shadow_DOM_restore_workspace_with_master_and_detail": "e91c85c79a6c2cbb4fd6051cf9af9a04", "index_Light_DOM_restore_workspace_with_viewers_with_generated_slotids": "0fad47e40121ecccf41c21bdfe2b7d56", "index_Shadow_DOM_restore_workspace_with_viewers_with_generated_slotids": "606bea2473f0e8b91756d5b69cd982be", - "index_Light_DOM_removeChild_Remove_One": "9efc06893b607cbd6a1e72642de195d6", + "index_Light_DOM_removeChild_Remove_One": "151bb1cea3ea5428feba5b7164cd7ab9", "index_Shadow_DOM_removeChild_Remove_One": "3a2af0dbfeb04e1e4f8e24acd2351fda", - "index_Light_DOM_Create_One": "6926536c63e772e6d339075156e0c358", - "index_Light_DOM_Create_Multiple": "36ba491911aa441350fb1e0b875c9d0c", - "index_Light_DOM_Create_multiple_with_names": "93a90058f59b1a330c393d6018024117", + "index_Light_DOM_Create_One": "f157c46964104ca82ab7f8928284b481", + "index_Light_DOM_Create_Multiple": "cf9b19a2f9edf70fd3cbe10d3b2953cf", + "index_Light_DOM_Create_multiple_with_names": "dcb85993a66f368f468500a879bba9df", "index_Shadow_DOM_Create_One": "02d56ecdcf18f8c44fe3d1c80749ce1a", "index_Shadow_DOM_Create_Multiple": "988e29fe4ebba32135338cebb1dcd698", "index_Shadow_DOM_Create_multiple_with_names": "9e6c683ce0a2bf9f0937f19aca244d90" diff --git a/packages/perspective/package.json b/packages/perspective/package.json index 69b605d21f..abe4652421 100644 --- a/packages/perspective/package.json +++ b/packages/perspective/package.json @@ -1,6 +1,6 @@ { "name": "@finos/perspective", - "version": "1.0.8", + "version": "1.1.0", "description": "Perspective.js", "repository": { "type": "git", @@ -41,7 +41,7 @@ "docs": "npm-run-all docs:jsdoc docs:deploy", "docs:jsdoc": "jsdoc2md src/js/perspective.js -p list --separators --no-gfm > README.md", "docs:deploy": "(echo \"---\nid: perspective\ntitle: perspective API\n---\n\n\"; cat README.md) > ../../docs/obj/perspective.md", - "test:run": "jest --color --ci --testPathIgnorePatterns='timezone'", + "test:run": "jest --color --ci --noStackTrace --testPathIgnorePatterns='timezone'", "test_timezone:run": "jest --color --ci --config=test/config/timezone/jest.config.js --rootDir=. timezone.spec.js", "test": "npm-run-all test:run", "clean": "rimraf dist" @@ -63,8 +63,8 @@ "ws": "^6.1.2" }, "devDependencies": { - "@finos/perspective-build": "^1.0.8", - "@finos/perspective-cpp": "^1.0.8", + "@finos/perspective-build": "^1.1.0", + "@finos/perspective-cpp": "^1.1.0", "jsverify": "^0.8.4", "lodash": "^4.17.4", "moment": "^2.19.1", diff --git a/packages/perspective/src/js/perspective.js b/packages/perspective/src/js/perspective.js index e304a24ee9..d7ea7d3215 100644 --- a/packages/perspective/src/js/perspective.js +++ b/packages/perspective/src/js/perspective.js @@ -799,7 +799,46 @@ export default function (Module) { * comma-separated column paths. */ view.prototype.to_csv = function (options) { - return to_format.call(this, options, formatters.csvFormatter); + _call_process(this.table.get_id()); + options = _parse_format_options.bind(this)(options); + const start_row = options.start_row; + const end_row = options.end_row; + const start_col = options.start_col; + const end_col = options.end_col; + const sides = this.sides(); + if (this.is_unit_context) { + return __MODULE__.to_csv_unit( + this._View, + start_row, + end_row, + start_col, + end_col + ); + } else if (sides === 0) { + return __MODULE__.to_csv_zero( + this._View, + start_row, + end_row, + start_col, + end_col + ); + } else if (sides === 1) { + return __MODULE__.to_csv_one( + this._View, + start_row, + end_row, + start_col, + end_col + ); + } else if (sides === 2) { + return __MODULE__.to_csv_two( + this._View, + start_row, + end_row, + start_col, + end_col + ); + } }; /** diff --git a/packages/perspective/src/js/view_formatters.js b/packages/perspective/src/js/view_formatters.js index 895f6d4584..7b95ddeaf7 100644 --- a/packages/perspective/src/js/view_formatters.js +++ b/packages/perspective/src/js/view_formatters.js @@ -18,50 +18,6 @@ const jsonFormatter = { slice: (data, start) => data.slice(start), }; -const csvFormatter = Object.assign({}, jsonFormatter, { - addColumnValue: (data, row, colName, value) => - row[colName.split("|").join(",")].unshift(value), - setColumnValue: (data, row, colName, value) => - (row[colName.split("|").join(",")] = value), - formatData: function (data, {delimiter = ","} = {}) { - if (data.length === 0) { - return ""; - } - - const format = function (x) { - if (x === null) { - return ""; - } - switch (typeof x) { - case "object": - case "string": - // CSV escapes with double double quotes, for real. - // Section 2.7 of the fake - // [CSV spec](https://tools.ietf.org/html/rfc4180) - return x.indexOf(delimiter) > -1 - ? `"${x.replace(/\"/g, '""')}"` - : x.toString(); - case "number": - return x; - case "boolean": - return x.toString(); - } - }; - - const columns = Object.keys(data[0]); - let csv = columns.map(format).join(delimiter); - for (let x = 0; x < data.length; x++) { - csv += - "\r\n" + - columns - .map((column) => format(data[x][column])) - .join(delimiter); - } - - return csv; - }, -}); - const jsonTableFormatter = { initDataValue: () => new Object(), initRowValue: () => {}, @@ -91,5 +47,4 @@ const jsonTableFormatter = { export default { jsonFormatter, jsonTableFormatter, - csvFormatter, }; diff --git a/packages/perspective/test/js/constructors.js b/packages/perspective/test/js/constructors.js index 326ef1fbab..3d4169c1c6 100644 --- a/packages/perspective/test/js/constructors.js +++ b/packages/perspective/test/js/constructors.js @@ -710,7 +710,7 @@ module.exports = (perspective) => { it("Serializes a simple view to CSV", async function () { var table = await perspective.table(data); var view = await table.view({}); - var answer = `x,y,z\r\n1,a,true\r\n2,b,false\r\n3,c,true\r\n4,d,false`; + var answer = `"x","y","z"\n1,"a",true\n2,"b",false\n3,"c",true\n4,"d",false\n`; let result = await view.to_csv(); expect(result).toEqual(answer); view.delete(); @@ -723,7 +723,7 @@ module.exports = (perspective) => { row_pivots: ["z"], columns: ["x"], }); - var answer = `__ROW_PATH__,x\r\n,10\r\nfalse,6\r\ntrue,4`; + var answer = `"z (Group by 1)","x"\n,10\nfalse,6\ntrue,4\n`; let result = await view.to_csv(); expect(result).toEqual(answer); view.delete(); @@ -737,7 +737,7 @@ module.exports = (perspective) => { column_pivots: ["y"], columns: ["x"], }); - var answer = `__ROW_PATH__,\"a,x\",\"b,x\",\"c,x\",\"d,x\"\r\n,1,2,3,4\r\nfalse,,2,,4\r\ntrue,1,,3,`; + var answer = `"z (Group by 1)","a|x","b|x","c|x","d|x"\n,1,2,3,4\nfalse,,2,,4\ntrue,1,,3,\n`; let result = await view.to_csv(); expect(result).toEqual(answer); view.delete(); @@ -775,7 +775,7 @@ module.exports = (perspective) => { let view = await table.view(); let result = await view.to_csv(); expect(result).toEqual( - `x,y\r\n"Test, hello!",1\r\nTest2",2\r\n"Test3, Hello!""",3` + `"x","y"\n"Test, hello!",1\n"Test2""",2\n"Test3, Hello!""",3\n` ); view.delete(); table.delete(); diff --git a/packages/perspective/test/js/to_format.js b/packages/perspective/test/js/to_format.js index ea36e92cb2..1dc9603cb3 100644 --- a/packages/perspective/test/js/to_format.js +++ b/packages/perspective/test/js/to_format.js @@ -651,9 +651,12 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; + expect(json).toEqual( + json2.map((x) => { + x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( + (x) => x + ); + delete x["string (Group by 1)"]; return x; }) ); @@ -679,6 +682,7 @@ module.exports = (perspective) => { let result = await view2.to_columns(); expect(result).toEqual({ + "string (Group by 1)": [null, "a", "b"], float: [2.75, 1.75, 3.75], string: [4, 2, 2], }); @@ -701,9 +705,12 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; + expect(json).toEqual( + json2.map((x) => { + x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( + (x) => x + ); + delete x["string (Group by 1)"]; return x; }) ); @@ -727,9 +734,12 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; + expect(json).toEqual( + json2.map((x) => { + x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( + (x) => x + ); + delete x["string (Group by 1)"]; return x; }) ); @@ -753,9 +763,12 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; + expect(json).toEqual( + json2.map((x) => { + x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( + (x) => x + ); + delete x["string (Group by 1)"]; return x; }) ); @@ -775,9 +788,12 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; + expect(json).toEqual( + json2.map((x) => { + x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( + (x) => x + ); + delete x["string (Group by 1)"]; return x; }) ); @@ -797,9 +813,12 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; + expect(json).toEqual( + json2.map((x) => { + x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( + (x) => x + ); + delete x["string (Group by 1)"]; return x; }) ); @@ -822,9 +841,12 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; + expect(json).toEqual( + json2.map((x) => { + x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( + (x) => x + ); + delete x["string (Group by 1)"]; return x; }) ); @@ -848,9 +870,12 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; + expect(json).toEqual( + json2.map((x) => { + x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( + (x) => x + ); + delete x["string (Group by 1)"]; return x; }) ); @@ -873,9 +898,12 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; + expect(json).toEqual( + json2.map((x) => { + x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( + (x) => x + ); + delete x["string (Group by 1)"]; return x; }) ); @@ -898,9 +926,12 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; + expect(json).toEqual( + json2.map((x) => { + x.__ROW_PATH__ = [x["string (Group by 1)"]].filter( + (x) => x + ); + delete x["string (Group by 1)"]; return x; }) ); @@ -920,12 +951,7 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; - return x; - }) - ); + expect(json).toEqual(json2); view2.delete(); table2.delete(); @@ -946,12 +972,7 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; - return x; - }) - ); + expect(json).toEqual(json2); view2.delete(); table2.delete(); @@ -971,12 +992,7 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; - return x; - }) - ); + expect(json).toEqual(json2); view2.delete(); table2.delete(); @@ -993,12 +1009,7 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; - return x; - }) - ); + expect(json).toEqual(json2); view2.delete(); table2.delete(); @@ -1015,12 +1026,7 @@ module.exports = (perspective) => { let view2 = await table2.view(); let json2 = await view2.to_json(); - expect(json2).toEqual( - json.map((x) => { - delete x["__ROW_PATH__"]; - return x; - }) - ); + expect(json).toEqual(json2); view2.delete(); table2.delete(); diff --git a/python/perspective/package.json b/python/perspective/package.json index 027c6795f1..ef8ff0580c 100644 --- a/python/perspective/package.json +++ b/python/perspective/package.json @@ -1,7 +1,7 @@ { "private": true, "name": "perspective-python-internal", - "version": "1.0.8", + "version": "1.1.0", "scripts": { "bench": "python3 bench/perspective_benchmark.py", "docs": "python3 docs/generate.py" diff --git a/python/perspective/perspective/core/_version.py b/python/perspective/perspective/core/_version.py index 06bb9f64ae..82aa893b08 100644 --- a/python/perspective/perspective/core/_version.py +++ b/python/perspective/perspective/core/_version.py @@ -1,2 +1,2 @@ -__version__ = "1.0.8" -major_minor_version = "1.0" +__version__ = "1.1.0" +major_minor_version = "1.1" diff --git a/python/perspective/perspective/core/data/np.py b/python/perspective/perspective/core/data/np.py index bf63688ae4..334776ef57 100644 --- a/python/perspective/perspective/core/data/np.py +++ b/python/perspective/perspective/core/data/np.py @@ -6,10 +6,10 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six -import numpy as np from datetime import datetime +import numpy as np + DATE_DTYPES = [ np.dtype("datetime64[D]"), np.dtype("datetime64[W]"), @@ -36,11 +36,6 @@ def make_null_mask(array): array.dtype, np.object_ ) - if six.PY2: - is_object_or_string_dtype = is_object_or_string_dtype or np.issubdtype( - array.dtype, np.unicode_ - ) - is_datetime_dtype = np.issubdtype(array.dtype, np.datetime64) or np.issubdtype( array.dtype, np.timedelta64 ) diff --git a/python/perspective/perspective/include/perspective/python.h b/python/perspective/perspective/include/perspective/python.h index d6b30d30ce..332a9fb770 100644 --- a/python/perspective/perspective/include/perspective/python.h +++ b/python/perspective/perspective/include/perspective/python.h @@ -19,7 +19,6 @@ #include #include - /****************************************************************************** * * Perspective includes @@ -41,11 +40,9 @@ #include #include - namespace perspective { -namespace binding { -} -} +namespace binding {} +} // namespace perspective /****************************************************************************** * @@ -53,15 +50,16 @@ namespace binding { */ using namespace perspective::binding; -PYBIND11_MODULE(libbinding, m) -{ +PYBIND11_MODULE(libbinding, m) { /****************************************************************************** * * PerspectiveCppError - * - * PerspectiveCppError is raised in Python when the C++ engine throws an exception. - * - * To catch all exceptions from Perspective, catch `PerspectiveError` and `PerspectiveCppError`. + * + * PerspectiveCppError is raised in Python when the C++ engine throws an + * exception. + * + * To catch all exceptions from Perspective, catch `PerspectiveError` and + * `PerspectiveCppError`. */ py::register_exception(m, "PerspectiveCppError"); @@ -70,8 +68,8 @@ PYBIND11_MODULE(libbinding, m) * Table */ py::class_>(m, "Table") - .def(py::init, std::vector, std::vector, - std::uint32_t, std::string>()) + .def(py::init, std::vector, + std::vector, std::uint32_t, std::string>()) .def("size", &Table::size) .def("get_schema", &Table::get_schema) .def("unregister_gnode", &Table::unregister_gnode) @@ -87,9 +85,10 @@ PYBIND11_MODULE(libbinding, m) * View */ // Bind a View for each context type - py::class_, std::shared_ptr>>(m, "View_ctxunit") - .def(py::init, std::shared_ptr, std::string, std::string, - std::shared_ptr>()) + py::class_, std::shared_ptr>>( + m, "View_ctxunit") + .def(py::init, std::shared_ptr, + std::string, std::string, std::shared_ptr>()) .def("sides", &View::sides) .def("num_rows", &View::num_rows) .def("num_columns", &View::num_columns) @@ -112,8 +111,8 @@ PYBIND11_MODULE(libbinding, m) .def("is_column_only", &View::is_column_only); py::class_, std::shared_ptr>>(m, "View_ctx0") - .def(py::init, std::shared_ptr, std::string, std::string, - std::shared_ptr>()) + .def(py::init, std::shared_ptr, + std::string, std::string, std::shared_ptr>()) .def("sides", &View::sides) .def("num_rows", &View::num_rows) .def("num_columns", &View::num_columns) @@ -136,8 +135,8 @@ PYBIND11_MODULE(libbinding, m) .def("is_column_only", &View::is_column_only); py::class_, std::shared_ptr>>(m, "View_ctx1") - .def(py::init, std::shared_ptr, std::string, std::string, - std::shared_ptr>()) + .def(py::init, std::shared_ptr, + std::string, std::string, std::shared_ptr>()) .def("sides", &View::sides) .def("num_rows", &View::num_rows) .def("num_columns", &View::num_columns) @@ -163,8 +162,8 @@ PYBIND11_MODULE(libbinding, m) .def("is_column_only", &View::is_column_only); py::class_, std::shared_ptr>>(m, "View_ctx2") - .def(py::init, std::shared_ptr, std::string, std::string, - std::shared_ptr>()) + .def(py::init, std::shared_ptr, + std::string, std::string, std::shared_ptr>()) .def("sides", &View::sides) .def("num_rows", &View::num_rows) .def("num_columns", &View::num_columns) @@ -194,17 +193,17 @@ PYBIND11_MODULE(libbinding, m) * * t_view_config */ - py::class_>(m, "t_view_config") - .def(py::init< - const std::vector&, + py::class_>( + m, "t_view_config") + .def(py::init&, const std::vector&, const tsl::ordered_map>&, const std::vector&, - const std::vector>>&, + const std::vector< + std::tuple>>&, const std::vector>&, const std::vector>&, - const std::string, - bool>()) + const std::string, bool>()) .def("add_filter_term", &t_view_config::add_filter_term); /****************************************************************************** @@ -212,7 +211,9 @@ PYBIND11_MODULE(libbinding, m) * t_data_table */ py::class_(m, "t_data_table") - .def("size", reinterpret_cast(&t_data_table::size)); + .def("size", + reinterpret_cast( + &t_data_table::size)); /****************************************************************************** * @@ -229,32 +230,37 @@ PYBIND11_MODULE(libbinding, m) * t_gnode */ py::class_>(m, "t_gnode") - .def("get_id", reinterpret_cast(&t_gnode::get_id)); + .def("get_id", + reinterpret_cast(&t_gnode::get_id)); /****************************************************************************** * * t_data_slice */ - py::class_, std::shared_ptr>>(m, "t_data_slice_ctxunit") + py::class_, + std::shared_ptr>>(m, "t_data_slice_ctxunit") .def("get_column_slice", &t_data_slice::get_column_slice) .def("get_slice", &t_data_slice::get_slice) .def("get_column_names", &t_data_slice::get_column_names) .def("get_pkeys", &t_data_slice::get_pkeys); - py::class_, std::shared_ptr>>(m, "t_data_slice_ctx0") + py::class_, std::shared_ptr>>( + m, "t_data_slice_ctx0") .def("get_column_slice", &t_data_slice::get_column_slice) .def("get_slice", &t_data_slice::get_slice) .def("get_column_names", &t_data_slice::get_column_names) .def("get_pkeys", &t_data_slice::get_pkeys); - py::class_, std::shared_ptr>>(m, "t_data_slice_ctx1") + py::class_, std::shared_ptr>>( + m, "t_data_slice_ctx1") .def("get_column_slice", &t_data_slice::get_column_slice) .def("get_slice", &t_data_slice::get_slice) .def("get_column_names", &t_data_slice::get_column_names) .def("get_row_path", &t_data_slice::get_row_path) .def("get_pkeys", &t_data_slice::get_pkeys); - py::class_, std::shared_ptr>>(m, "t_data_slice_ctx2") + py::class_, std::shared_ptr>>( + m, "t_data_slice_ctx2") .def("get_column_slice", &t_data_slice::get_column_slice) .def("get_slice", &t_data_slice::get_slice) .def("get_column_names", &t_data_slice::get_column_names) @@ -285,7 +291,6 @@ PYBIND11_MODULE(libbinding, m) */ py::class_(m, "t_ctx2"); - /****************************************************************************** * * t_pool @@ -303,8 +308,10 @@ PYBIND11_MODULE(libbinding, m) */ py::class_(m, "t_validated_expression_map") .def(py::init<>()) - .def("get_expression_schema", &t_validated_expression_map::get_expression_schema) - .def("get_expression_errors", &t_validated_expression_map::get_expression_errors); + .def("get_expression_schema", + &t_validated_expression_map::get_expression_schema) + .def("get_expression_errors", + &t_validated_expression_map::get_expression_errors); /****************************************************************************** * @@ -382,7 +389,7 @@ PYBIND11_MODULE(libbinding, m) .value("DTYPE_LAST_VLEN", DTYPE_LAST_VLEN) .value("DTYPE_LAST", DTYPE_LAST); - /****************************************************************************** + /****************************************************************************** * * t_filter_op */ @@ -438,6 +445,10 @@ PYBIND11_MODULE(libbinding, m) m.def("to_arrow_zero", &to_arrow_zero); m.def("to_arrow_one", &to_arrow_one); m.def("to_arrow_two", &to_arrow_two); + m.def("to_csv_unit", &to_csv_unit); + m.def("to_csv_zero", &to_csv_zero); + m.def("to_csv_one", &to_csv_one); + m.def("to_csv_two", &to_csv_two); m.def("get_row_delta_unit", &get_row_delta_unit); m.def("get_row_delta_zero", &get_row_delta_zero); m.def("get_row_delta_one", &get_row_delta_one); diff --git a/python/perspective/perspective/include/perspective/python/accessor.h b/python/perspective/perspective/include/perspective/python/accessor.h index 8d0a4bdb54..d9e7851832 100644 --- a/python/perspective/perspective/include/perspective/python/accessor.h +++ b/python/perspective/perspective/include/perspective/python/accessor.h @@ -17,17 +17,19 @@ namespace perspective { namespace binding { -/****************************************************************************** - * - * Data accessor API - */ + /****************************************************************************** + * + * Data accessor API + */ -std::vector get_column_names(t_val data, std::int32_t format); -t_dtype infer_type(t_val x, t_val date_validator); -t_dtype get_data_type(t_val data, std::int32_t format, py::str name, t_val date_validator); -std::vector get_data_types(t_val data, std::int32_t format, std::vector names, t_val date_validator); + std::vector get_column_names(t_val data, std::int32_t format); + t_dtype infer_type(t_val x, t_val date_validator); + t_dtype get_data_type( + t_val data, std::int32_t format, py::str name, t_val date_validator); + std::vector get_data_types(t_val data, std::int32_t format, + std::vector names, t_val date_validator); -} //namespace binding -} //namespace perspective +} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/include/perspective/python/column.h b/python/perspective/perspective/include/perspective/python/column.h index d2c6798d7f..6837e2ee19 100644 --- a/python/perspective/perspective/include/perspective/python/column.h +++ b/python/perspective/perspective/include/perspective/python/column.h @@ -16,12 +16,12 @@ namespace perspective { -//Specialize for PSP_OBJECT_TYPE +// Specialize for PSP_OBJECT_TYPE template <> void t_column::object_copied(t_uindex idx) const; template <> void t_column::object_cleared(t_uindex idx) const; -} +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/include/perspective/python/context.h b/python/perspective/perspective/include/perspective/python/context.h index b8574eb3f4..7de2437470 100644 --- a/python/perspective/perspective/include/perspective/python/context.h +++ b/python/perspective/perspective/include/perspective/python/context.h @@ -16,27 +16,31 @@ namespace perspective { namespace binding { -/****************************************************************************** - * - * Context API - */ -template <> -std::shared_ptr -make_context(std::shared_ptr
table, std::shared_ptr schema, std::shared_ptr view_config, const std::string& name); - -template <> -std::shared_ptr -make_context(std::shared_ptr
table, std::shared_ptr schema, std::shared_ptr view_config, const std::string& name); - -template <> -std::shared_ptr -make_context(std::shared_ptr
table, std::shared_ptr schema, std::shared_ptr view_config, const std::string& name); - -template <> -std::shared_ptr -make_context(std::shared_ptr
table, std::shared_ptr schema, std::shared_ptr view_config, const std::string& name); - -} //namespace binding -} //namespace perspective + /****************************************************************************** + * + * Context API + */ + template <> + std::shared_ptr make_context(std::shared_ptr
table, + std::shared_ptr schema, + std::shared_ptr view_config, const std::string& name); + + template <> + std::shared_ptr make_context(std::shared_ptr
table, + std::shared_ptr schema, + std::shared_ptr view_config, const std::string& name); + + template <> + std::shared_ptr make_context(std::shared_ptr
table, + std::shared_ptr schema, + std::shared_ptr view_config, const std::string& name); + + template <> + std::shared_ptr make_context(std::shared_ptr
table, + std::shared_ptr schema, + std::shared_ptr view_config, const std::string& name); + +} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/include/perspective/python/expressions.h b/python/perspective/perspective/include/perspective/python/expressions.h index c4eddfc673..f03a42db86 100644 --- a/python/perspective/perspective/include/perspective/python/expressions.h +++ b/python/perspective/perspective/include/perspective/python/expressions.h @@ -16,19 +16,18 @@ namespace perspective { namespace binding { -/** - * @brief Initialize the expressions parser. Must be called at module - * initialization before any interactions with the module. - * - */ -void init_expression_parser(); + /** + * @brief Initialize the expressions parser. Must be called at module + * initialization before any interactions with the module. + * + */ + void init_expression_parser(); -t_validated_expression_map -validate_expressions_py( - std::shared_ptr
table, - const std::vector>& p_expressions); + t_validated_expression_map validate_expressions_py( + std::shared_ptr
table, + const std::vector>& p_expressions); -} //namespace binding -} //namespace perspective +} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/include/perspective/python/fill.h b/python/perspective/perspective/include/perspective/python/fill.h index 322b158d3b..56949277e8 100644 --- a/python/perspective/perspective/include/perspective/python/fill.h +++ b/python/perspective/perspective/include/perspective/python/fill.h @@ -17,22 +17,25 @@ namespace perspective { namespace binding { -/****************************************************************************** - * - * Fill columns with data - */ - -template <> -void set_column_nth(std::shared_ptr col, t_uindex idx, t_val value); - -/****************************************************************************** - * - * Fill tables with data - */ - -void _fill_data(t_data_table& tbl, t_data_accessor accessor, const t_schema& input_schema, const std::string& index, std::uint32_t offset, std::uint32_t limit, bool is_update); - -} //namespace binding -} //namespace perspective + /****************************************************************************** + * + * Fill columns with data + */ + + template <> + void set_column_nth( + std::shared_ptr col, t_uindex idx, t_val value); + + /****************************************************************************** + * + * Fill tables with data + */ + + void _fill_data(t_data_table& tbl, t_data_accessor accessor, + const t_schema& input_schema, const std::string& index, + std::uint32_t offset, std::uint32_t limit, bool is_update); + +} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/include/perspective/python/numpy.h b/python/perspective/perspective/include/perspective/python/numpy.h index 034fd8bba5..dba1ce1922 100644 --- a/python/perspective/perspective/include/perspective/python/numpy.h +++ b/python/perspective/perspective/include/perspective/python/numpy.h @@ -30,140 +30,178 @@ namespace perspective { namespace numpy { - enum t_fill_status { - FILL_SUCCESS, - FILL_FAIL - }; + enum t_fill_status { FILL_SUCCESS, FILL_FAIL }; /** - * NumpyLoader fast-tracks the loading of Numpy arrays into Perspective, utilizing memcpy whenever possible. + * NumpyLoader fast-tracks the loading of Numpy arrays into Perspective, + * utilizing memcpy whenever possible. */ class PERSPECTIVE_BINDING_EXPORT NumpyLoader { - public: - NumpyLoader(t_val accessor); - ~NumpyLoader(); - - /** - * Initialize the Numpy loader by constructing the column names and data types arrays. - */ - void init(); - - /** - * Given `inferred_types` from Perspective, use the `m_types` array of numpy array dtypes and - * reconcile differences between numeric dtypes by *preferring the dtype of the numpy array* and - * returning a vector of the correct, reconciled types. - * - * This prevents the situation where Perspective infers an int column to `DTYPE_INT32` but the - * numpy array dtype is actually "int64". - * - * Marked const as this method does not mutate the internal `m_types` property. - */ - std::vector reconcile_dtypes(const std::vector& inferred_types) const; - - /** - * Fill a `t_data_table` with numpy array-backed data. - */ - void fill_table(t_data_table& tbl, const t_schema& input_schema, const std::string& index, - std::uint32_t offset, std::uint32_t limit, bool is_update); - - /** - * Fill a column with a Numpy array by copying it wholesale into the column without iteration. - * - * If the copy operation fails, fill the column iteratively. - * - * @param tbl - * @param col - * @param length - * @param type - * @param is_update - */ - void fill_column(t_data_table& tbl, std::shared_ptr col, const std::string& name, t_dtype type, std::uint32_t cidx, bool is_update); - - std::vector names() const; - std::vector types() const; - std::uint32_t row_count() const; - - /** - * Keep a list of numpy datetime64 units that we should treat as dates and not datetimes. - */ - static const std::vector DATE_UNITS; - private: - /** - * When memory cannot be copied for dtype=object arrays, for example), fill the column through iteration. - */ - void fill_column_iter(const py::array& array, t_data_table& tbl, std::shared_ptr col, const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update); - - /** - * Fill arrays with dtype=object using the data accessor's marshal method. - * - * Because we don't iterate through the array directly, don't pass the array into this method/any others that call `marshal`. - * - * If filling a column of `DTYPE_TIME`, is always `std::int64_t`. - */ - template - void fill_object_iter(t_data_table& tbl, std::shared_ptr col, const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update); - - // Fill dates that might be `datetime.date` or strings - void fill_date_iter(std::shared_ptr col, const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update); - - // Fill using numpy arrays with defined numpy dtypes that are not `object` - - /** - * Given a numpy array containing a numeric type, and a determination that `type` is numeric (int/float), fill it iteratively. - * - * Iterating through the array and filling the underlying column allows us to cast the array's values to the `t_dtype` of the table, - * which may be of a higher or a lower bit width (i.e. filling a table that was inferred as `DTYPE_INT32` with `DTYPE_INT64`, - * which is more commonly used in numpy arrays.) - */ - void fill_numeric_iter(const py::array& array, t_data_table& tbl, std::shared_ptr col, const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update); - - void fill_datetime_iter(const py::array& array, t_data_table& tbl, std::shared_ptr col, const std::string& name, - t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update); - - void fill_bool_iter(const py::array& array, t_data_table& tbl, std::shared_ptr col, const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update); - - /** - * Extract a numpy array from src and copy it into dest. - * - * If `np_dtype` and `type` mismatch in the following cases, then fill iteratively: - * - * - when `np_dtype` is int64 and `t_dtype` is `DTYPE_INT32` or `DTYPE_FLOAT64` - * - when `np_dtype` is float64 and `t_dtype` is `DTYPE_INT32` or `DTYPE_INT64` - * - * These errors occur frqeuently when a Table is created from non-numpy data, then updated with a numpy array. - * The `t_dtype` of the Table always supercedes the array dtype, as the table is immutable after creation. - * - * Returns a `t_fill_status` enum indicating success or failure of the copy operation. - */ - t_fill_status try_copy_array(const py::array& src, std::shared_ptr dest, t_dtype np_dtype, t_dtype type, const std::uint64_t offset); - - void fill_validity_map(std::shared_ptr col, std::uint64_t* mask_ptr, std::size_t mask_size, bool is_update); - - // Return the column names from the Python data accessor - std::vector make_names(); - - // Map the dtype of each numpy array into Perspective `t_dtype`s. - std::vector make_types(); - - bool m_init; - - /** - * A flag to determine whether to reconcile numpy array dtype with perspective inferred types. - * - * Defaults to false - is true when any array dtype is of int/float/bool. - */ - bool m_has_numeric_dtype; - t_val m_accessor; - std::vector m_names; - std::vector m_types; + public: + NumpyLoader(t_val accessor); + ~NumpyLoader(); + + /** + * Initialize the Numpy loader by constructing the column names and data + * types arrays. + */ + void init(); + + /** + * Given `inferred_types` from Perspective, use the `m_types` array of + * numpy array dtypes and reconcile differences between numeric dtypes + * by *preferring the dtype of the numpy array* and returning a vector + * of the correct, reconciled types. + * + * This prevents the situation where Perspective infers an int column to + * `DTYPE_INT32` but the numpy array dtype is actually "int64". + * + * Marked const as this method does not mutate the internal `m_types` + * property. + */ + std::vector reconcile_dtypes( + const std::vector& inferred_types) const; + + /** + * Fill a `t_data_table` with numpy array-backed data. + */ + void fill_table(t_data_table& tbl, const t_schema& input_schema, + const std::string& index, std::uint32_t offset, std::uint32_t limit, + bool is_update); + + /** + * Fill a column with a Numpy array by copying it wholesale into the + * column without iteration. + * + * If the copy operation fails, fill the column iteratively. + * + * @param tbl + * @param col + * @param length + * @param type + * @param is_update + */ + void fill_column(t_data_table& tbl, std::shared_ptr col, + const std::string& name, t_dtype type, std::uint32_t cidx, + bool is_update); + + std::vector names() const; + std::vector types() const; + std::uint32_t row_count() const; + + /** + * Keep a list of numpy datetime64 units that we should treat as dates + * and not datetimes. + */ + static const std::vector DATE_UNITS; + + private: + /** + * When memory cannot be copied for dtype=object arrays, for example), + * fill the column through iteration. + */ + void fill_column_iter(const py::array& array, t_data_table& tbl, + std::shared_ptr col, const std::string& name, + t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update); + + /** + * Fill arrays with dtype=object using the data accessor's marshal + * method. + * + * Because we don't iterate through the array directly, don't pass the + * array into this method/any others that call `marshal`. + * + * If filling a column of `DTYPE_TIME`, is always `std::int64_t`. + */ + template + void fill_object_iter(t_data_table& tbl, std::shared_ptr col, + const std::string& name, t_dtype np_dtype, t_dtype type, + std::uint32_t cidx, bool is_update); + + // Fill dates that might be `datetime.date` or strings + void fill_date_iter(std::shared_ptr col, + const std::string& name, t_dtype np_dtype, t_dtype type, + std::uint32_t cidx, bool is_update); + + // Fill using numpy arrays with defined numpy dtypes that are not + // `object` + + /** + * Given a numpy array containing a numeric type, and a determination + * that `type` is numeric (int/float), fill it iteratively. + * + * Iterating through the array and filling the underlying column allows + * us to cast the array's values to the `t_dtype` of the table, which + * may be of a higher or a lower bit width (i.e. filling a table that + * was inferred as `DTYPE_INT32` with `DTYPE_INT64`, which is more + * commonly used in numpy arrays.) + */ + void fill_numeric_iter(const py::array& array, t_data_table& tbl, + std::shared_ptr col, const std::string& name, + t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update); + + void fill_datetime_iter(const py::array& array, t_data_table& tbl, + std::shared_ptr col, const std::string& name, + t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update); + + void fill_bool_iter(const py::array& array, t_data_table& tbl, + std::shared_ptr col, const std::string& name, + t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update); + + /** + * Extract a numpy array from src and copy it into dest. + * + * If `np_dtype` and `type` mismatch in the following cases, then fill + * iteratively: + * + * - when `np_dtype` is int64 and `t_dtype` is `DTYPE_INT32` or + * `DTYPE_FLOAT64` + * - when `np_dtype` is float64 and `t_dtype` is `DTYPE_INT32` or + * `DTYPE_INT64` + * + * These errors occur frqeuently when a Table is created from non-numpy + * data, then updated with a numpy array. The `t_dtype` of the Table + * always supercedes the array dtype, as the table is immutable after + * creation. + * + * Returns a `t_fill_status` enum indicating success or failure of the + * copy operation. + */ + t_fill_status try_copy_array(const py::array& src, + std::shared_ptr dest, t_dtype np_dtype, t_dtype type, + const std::uint64_t offset); + + void fill_validity_map(std::shared_ptr col, + std::uint64_t* mask_ptr, std::size_t mask_size, bool is_update); + + // Return the column names from the Python data accessor + std::vector make_names(); + + // Map the dtype of each numpy array into Perspective `t_dtype`s. + std::vector make_types(); + + bool m_init; + + /** + * A flag to determine whether to reconcile numpy array dtype with + * perspective inferred types. + * + * Defaults to false - is true when any array dtype is of + * int/float/bool. + */ + bool m_has_numeric_dtype; + t_val m_accessor; + std::vector m_names; + std::vector m_types; }; /** * Copy the data of a numpy array into a `t_column`. */ template - void copy_array_helper(const void* src, std::shared_ptr dest, const std::uint64_t offset); + void copy_array_helper(const void* src, std::shared_ptr dest, + const std::uint64_t offset); } // namespace numpy -} // numpy perspective +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/include/perspective/python/serialization.h b/python/perspective/perspective/include/perspective/python/serialization.h index d71854c63e..517c39eaa1 100644 --- a/python/perspective/perspective/include/perspective/python/serialization.h +++ b/python/perspective/perspective/include/perspective/python/serialization.h @@ -17,43 +17,70 @@ namespace perspective { namespace binding { -/****************************************************************************** - * - * Data serialization - */ + /****************************************************************************** + * + * Data serialization + */ -template <> -t_val get_column_data(std::shared_ptr table, const std::string& colname); + template <> + t_val get_column_data( + std::shared_ptr table, const std::string& colname); -template -std::shared_ptr> get_data_slice(std::shared_ptr> view, std::uint32_t start_row, std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col); + template + std::shared_ptr> get_data_slice( + std::shared_ptr> view, std::uint32_t start_row, + std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col); -std::shared_ptr> -get_data_slice_unit(std::shared_ptr> view, std::uint32_t start_row, std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col); + std::shared_ptr> get_data_slice_unit( + std::shared_ptr> view, std::uint32_t start_row, + std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col); -std::shared_ptr> -get_data_slice_ctx0(std::shared_ptr> view, std::uint32_t start_row, std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col); + std::shared_ptr> get_data_slice_ctx0( + std::shared_ptr> view, std::uint32_t start_row, + std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col); -std::shared_ptr> -get_data_slice_ctx1(std::shared_ptr> view, std::uint32_t start_row, std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col); + std::shared_ptr> get_data_slice_ctx1( + std::shared_ptr> view, std::uint32_t start_row, + std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col); -std::shared_ptr> -get_data_slice_ctx2(std::shared_ptr> view, std::uint32_t start_row, std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col); + std::shared_ptr> get_data_slice_ctx2( + std::shared_ptr> view, std::uint32_t start_row, + std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col); -template -t_val get_from_data_slice(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx); -t_val get_from_data_slice_unit(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx); -t_val get_from_data_slice_ctx0(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx); -t_val get_from_data_slice_ctx1(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx); -t_val get_from_data_slice_ctx2(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx); + template + t_val get_from_data_slice(std::shared_ptr> data_slice, + t_uindex ridx, t_uindex cidx); + t_val get_from_data_slice_unit( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx); + t_val get_from_data_slice_ctx0( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx); + t_val get_from_data_slice_ctx1( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx); + t_val get_from_data_slice_ctx2( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx); -// wrap `get_pkeys` in order to convert t_scalar to t_val entirely within c++ -template -std::vector get_pkeys_from_data_slice(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx); -std::vector get_pkeys_from_data_slice_unit(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx); -std::vector get_pkeys_from_data_slice_ctx0(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx); -std::vector get_pkeys_from_data_slice_ctx1(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx); -std::vector get_pkeys_from_data_slice_ctx2(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx); + // wrap `get_pkeys` in order to convert t_scalar to t_val entirely within + // c++ + template + std::vector get_pkeys_from_data_slice( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx); + std::vector get_pkeys_from_data_slice_unit( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx); + std::vector get_pkeys_from_data_slice_ctx0( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx); + std::vector get_pkeys_from_data_slice_ctx1( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx); + std::vector get_pkeys_from_data_slice_ctx2( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx); } // end namespace binding } // end namespace perspective diff --git a/python/perspective/perspective/include/perspective/python/table.h b/python/perspective/perspective/include/perspective/python/table.h index 91be5a483e..f304f9c724 100644 --- a/python/perspective/perspective/include/perspective/python/table.h +++ b/python/perspective/perspective/include/perspective/python/table.h @@ -17,23 +17,15 @@ namespace perspective { namespace binding { -/****************************************************************************** - * - * Table API - */ -std::shared_ptr
make_table_py( - t_val table, - t_data_accessor accessor, - std::uint32_t limit, - py::str index, - t_op op, - bool is_update, - bool is_arrow, - bool is_csv, - t_uindex port_id -); + /****************************************************************************** + * + * Table API + */ + std::shared_ptr
make_table_py(t_val table, t_data_accessor accessor, + std::uint32_t limit, py::str index, t_op op, bool is_update, + bool is_arrow, bool is_csv, t_uindex port_id); -} //namespace binding -} //namespace perspective +} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/include/perspective/python/utils.h b/python/perspective/perspective/include/perspective/python/utils.h index 7049d0dc2f..2dc00a28c3 100644 --- a/python/perspective/perspective/include/perspective/python/utils.h +++ b/python/perspective/perspective/include/perspective/python/utils.h @@ -19,33 +19,55 @@ namespace perspective { namespace binding { -/****************************************************************************** - * - * Helper functions - */ -template -static void WARN(Args&&... args) { py::module::import("logging").attr("warning")(args...);}; + /****************************************************************************** + * + * Helper functions + */ + template + static void + WARN(Args&&... args) { + py::module::import("logging").attr("warning")(args...); + }; -template -static void CRITICAL(Args&&... args) { py::module::import("logging").attr("critical")(args...);}; + template + static void + CRITICAL(Args&&... args) { + py::module::import("logging").attr("critical")(args...); + }; -static bool IS_BOOL(t_val&& type_instance) { return type_instance.is(py::module::import("builtins").attr("bool")); }; -static bool IS_INT(t_val&& type_instance) { return type_instance.is(py::module::import("builtins").attr("int")); }; -static bool IS_FLOAT(t_val&& type_instance) { return type_instance.is(py::module::import("builtins").attr("float")); }; -static bool IS_STR(t_val&& type_instance) { return type_instance.is(py::module::import("builtins").attr("str")); }; -static bool IS_BYTES(t_val&& type_instance) { return type_instance.is(py::module::import("builtins").attr("bytes")); }; + static bool + IS_BOOL(t_val&& type_instance) { + return type_instance.is(py::module::import("builtins").attr("bool")); + }; + static bool + IS_INT(t_val&& type_instance) { + return type_instance.is(py::module::import("builtins").attr("int")); + }; + static bool + IS_FLOAT(t_val&& type_instance) { + return type_instance.is(py::module::import("builtins").attr("float")); + }; + static bool + IS_STR(t_val&& type_instance) { + return type_instance.is(py::module::import("builtins").attr("str")); + }; + static bool + IS_BYTES(t_val&& type_instance) { + return type_instance.is(py::module::import("builtins").attr("bytes")); + }; -/****************************************************************************** - * - * Date Parsing - */ + /****************************************************************************** + * + * Date Parsing + */ -t_dtype type_string_to_t_dtype(std::string type, std::string name = ""); -t_dtype type_string_to_t_dtype(py::str type, py::str name = ""); + t_dtype type_string_to_t_dtype(std::string type, std::string name = ""); + t_dtype type_string_to_t_dtype(py::str type, py::str name = ""); -t_val scalar_to_py(const t_tscalar& scalar, bool cast_double = false, bool cast_string = false); + t_val scalar_to_py(const t_tscalar& scalar, bool cast_double = false, + bool cast_string = false); -} //namespace binding -} //namespace perspective +} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/include/perspective/python/view.h b/python/perspective/perspective/include/perspective/python/view.h index 99a88fc4b3..a56ea054a5 100644 --- a/python/perspective/perspective/include/perspective/python/view.h +++ b/python/perspective/perspective/include/perspective/python/view.h @@ -18,82 +18,89 @@ #include #include - namespace perspective { namespace binding { -/****************************************************************************** - * - * View API - */ - -template <> -bool is_valid_filter(t_dtype type, t_val date_parser, t_filter_op comp, t_val filter_term); - -template <> -std::tuple> -make_filter_term(t_dtype column_type, t_val date_parser, const std::string& column_name, const std::string& filter_op_str, t_val filter_term); - -template <> -std::shared_ptr make_view_config( - const t_gnode& gnode, - std::shared_ptr schema, t_val date_parser, t_val config); - -template -std::shared_ptr> make_view(std::shared_ptr
table, const std::string& name, const std::string& separator, t_val view_config, t_val date_parser); - -/** - * Unlike Emscripten, where we can define templated headers in the Embind - * declaration, we need to explicitly specify all templated functions - * before they are used by Pybind. - */ -std::shared_ptr> make_view_unit(std::shared_ptr
table, std::string name, std::string separator, t_val view_config, t_val date_parser); -std::shared_ptr> make_view_ctx0(std::shared_ptr
table, std::string name, std::string separator, t_val view_config, t_val date_parser); -std::shared_ptr> make_view_ctx1(std::shared_ptr
table, std::string name, std::string separator, t_val view_config, t_val date_parser); -std::shared_ptr> make_view_ctx2(std::shared_ptr
table, std::string name, std::string separator, t_val view_config, t_val date_parser); - -py::bytes to_arrow_unit( - std::shared_ptr> view, - std::int32_t start_row, - std::int32_t end_row, - std::int32_t start_col, - std::int32_t end_col); - -py::bytes to_arrow_zero( - std::shared_ptr> view, - std::int32_t start_row, - std::int32_t end_row, - std::int32_t start_col, - std::int32_t end_col); - -py::bytes to_arrow_zero( - std::shared_ptr> view, - std::int32_t start_row, - std::int32_t end_row, - std::int32_t start_col, - std::int32_t end_col); - -py::bytes to_arrow_one( - std::shared_ptr> view, - std::int32_t start_row, - std::int32_t end_row, - std::int32_t start_col, - std::int32_t end_col); - -py::bytes to_arrow_two( - std::shared_ptr> view, - std::int32_t start_row, - std::int32_t end_row, - std::int32_t start_col, - std::int32_t end_col); - -py::bytes get_row_delta_unit(std::shared_ptr> view); -py::bytes get_row_delta_zero(std::shared_ptr> view); -py::bytes get_row_delta_one(std::shared_ptr> view); -py::bytes get_row_delta_two(std::shared_ptr> view); - - -} //namespace binding -} //namespace perspective + /****************************************************************************** + * + * View API + */ + + template <> + bool is_valid_filter( + t_dtype type, t_val date_parser, t_filter_op comp, t_val filter_term); + + template <> + std::tuple> + make_filter_term(t_dtype column_type, t_val date_parser, + const std::string& column_name, const std::string& filter_op_str, + t_val filter_term); + + template <> + std::shared_ptr make_view_config(const t_gnode& gnode, + std::shared_ptr schema, t_val date_parser, t_val config); + + template + std::shared_ptr> make_view(std::shared_ptr
table, + const std::string& name, const std::string& separator, + t_val view_config, t_val date_parser); + + /** + * Unlike Emscripten, where we can define templated headers in the Embind + * declaration, we need to explicitly specify all templated functions + * before they are used by Pybind. + */ + std::shared_ptr> make_view_unit( + std::shared_ptr
table, std::string name, std::string separator, + t_val view_config, t_val date_parser); + std::shared_ptr> make_view_ctx0(std::shared_ptr
table, + std::string name, std::string separator, t_val view_config, + t_val date_parser); + std::shared_ptr> make_view_ctx1(std::shared_ptr
table, + std::string name, std::string separator, t_val view_config, + t_val date_parser); + std::shared_ptr> make_view_ctx2(std::shared_ptr
table, + std::string name, std::string separator, t_val view_config, + t_val date_parser); + + py::bytes to_arrow_unit(std::shared_ptr> view, + std::int32_t start_row, std::int32_t end_row, std::int32_t start_col, + std::int32_t end_col); + + py::bytes to_arrow_zero(std::shared_ptr> view, + std::int32_t start_row, std::int32_t end_row, std::int32_t start_col, + std::int32_t end_col); + + py::bytes to_arrow_one(std::shared_ptr> view, + std::int32_t start_row, std::int32_t end_row, std::int32_t start_col, + std::int32_t end_col); + + py::bytes to_arrow_two(std::shared_ptr> view, + std::int32_t start_row, std::int32_t end_row, std::int32_t start_col, + std::int32_t end_col); + + std::string to_csv_unit(std::shared_ptr> view, + std::int32_t start_row, std::int32_t end_row, std::int32_t start_col, + std::int32_t end_col); + + std::string to_csv_zero(std::shared_ptr> view, + std::int32_t start_row, std::int32_t end_row, std::int32_t start_col, + std::int32_t end_col); + + std::string to_csv_one(std::shared_ptr> view, + std::int32_t start_row, std::int32_t end_row, std::int32_t start_col, + std::int32_t end_col); + + std::string to_csv_two(std::shared_ptr> view, + std::int32_t start_row, std::int32_t end_row, std::int32_t start_col, + std::int32_t end_col); + + py::bytes get_row_delta_unit(std::shared_ptr> view); + py::bytes get_row_delta_zero(std::shared_ptr> view); + py::bytes get_row_delta_one(std::shared_ptr> view); + py::bytes get_row_delta_two(std::shared_ptr> view); + +} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/manager/manager.py b/python/perspective/perspective/manager/manager.py index 00bbdbaaf6..d96a523f22 100644 --- a/python/perspective/perspective/manager/manager.py +++ b/python/perspective/perspective/manager/manager.py @@ -6,14 +6,14 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six import random import string from functools import partial + from ..core.exception import PerspectiveError from ..table import Table -from .session import PerspectiveSession from .manager_internal import _PerspectiveManagerInternal +from .session import PerspectiveSession def gen_name(size=10, chars=string.ascii_uppercase + string.digits): @@ -106,7 +106,7 @@ def get_table(self, name): def get_table_names(self): """Return the tables that are hosted with this manager by name.""" - return list(six.iterkeys(self._tables)) + return list(self._tables.keys()) def new_session(self): return PerspectiveSession(self) diff --git a/python/perspective/perspective/manager/manager_internal.py b/python/perspective/perspective/manager/manager_internal.py index 2685ea7ac1..f80681ab27 100644 --- a/python/perspective/perspective/manager/manager_internal.py +++ b/python/perspective/perspective/manager/manager_internal.py @@ -6,13 +6,13 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -from six import string_types import datetime -import logging import json +import logging from functools import partial + from ..core.exception import PerspectiveError -from ..table import Table, PerspectiveCppError +from ..table import PerspectiveCppError, Table from ..table._callback_cache import _PerspectiveCallBackCache from ..table._date_validator import _PerspectiveDateValidator @@ -94,7 +94,7 @@ def __process(self, msg, post_callback, client_id=None): self._pending_binary = None - if isinstance(msg, string_types): + if isinstance(msg, str): msg = json.loads(msg) if not isinstance(msg, dict): diff --git a/python/perspective/perspective/src/accessor.cpp b/python/perspective/perspective/src/accessor.cpp index 5fab2b9b15..fb595a6ab4 100644 --- a/python/perspective/perspective/src/accessor.cpp +++ b/python/perspective/perspective/src/accessor.cpp @@ -17,201 +17,218 @@ namespace perspective { namespace binding { -/****************************************************************************** - * - * Data accessor API - */ - -std::vector -get_column_names(t_val data, std::int32_t format) { - std::vector names; - if (format == 0) { - // record - py::list data_list = data.cast(); - std::int32_t max_check = 50; - - if(data_list.size()){ - for(auto tup: data_list[0].cast()){ - names.push_back(tup.first.cast()); + /****************************************************************************** + * + * Data accessor API + */ + + std::vector + get_column_names(t_val data, std::int32_t format) { + std::vector names; + if (format == 0) { + // record + py::list data_list = data.cast(); + std::int32_t max_check = 50; + + if (data_list.size()) { + for (auto tup : data_list[0].cast()) { + names.push_back(tup.first.cast()); + } } - } - std::int32_t check_index = std::min(max_check, int32_t(data_list.size())); + std::int32_t check_index + = std::min(max_check, int32_t(data_list.size())); - for (auto ix = 0; ix < check_index; ix++) { - py::dict next_dict = data_list[ix].cast(); - auto old_size = names.size(); + for (auto ix = 0; ix < check_index; ix++) { + py::dict next_dict = data_list[ix].cast(); + auto old_size = names.size(); - for (auto tup: next_dict) { - if (std::find(names.begin(), names.end(), tup.first.cast()) == names.end()) { - names.push_back(tup.first.cast()); + for (auto tup : next_dict) { + if (std::find(names.begin(), names.end(), + tup.first.cast()) + == names.end()) { + names.push_back(tup.first.cast()); + } } - } - if (old_size != names.size()){ - if (max_check == 50) { - WARN("Data parse warning: Array data has inconsistent rows"); + if (old_size != names.size()) { + if (max_check == 50) { + WARN("Data parse warning: Array data has inconsistent " + "rows"); + } + WARN("Extended from %d to %d", old_size, names.size()); + max_check *= 2; } - WARN("Extended from %d to %d", old_size, names.size()); - max_check *= 2; + } + } else if (format == 1 || format == 2) { + py::dict data_dict = data.cast(); + for (auto tup : data_dict) { + names.push_back(tup.first.cast()); } } - } else if (format == 1 || format == 2) { - py::dict data_dict = data.cast(); - for(auto tup: data_dict){ - names.push_back(tup.first.cast()); - } + return names; } - return names; -} - -t_dtype -infer_type(t_val x, t_val date_validator) { - std::string type_string = x.get_type().attr("__name__").cast(); - t_dtype t = t_dtype::DTYPE_STR; - // If object provides its own type, use that - if (py::hasattr(x, "_psp_dtype_" )) { - auto new_type = x.attr("_psp_dtype_")(); + t_dtype + infer_type(t_val x, t_val date_validator) { + std::string type_string + = x.get_type().attr("__name__").cast(); + t_dtype t = t_dtype::DTYPE_STR; - if (py::hasattr(new_type, "__name__")){ - // If type is a class, get its name - type_string = new_type.attr("__name__").cast(); + // If object provides its own type, use that + if (py::hasattr(x, "_psp_dtype_")) { + auto new_type = x.attr("_psp_dtype_")(); - } else { - // Assume that the string is the type - type_string = new_type.cast(); - } + if (py::hasattr(new_type, "__name__")) { + // If type is a class, get its name + type_string = new_type.attr("__name__").cast(); - // Extract representation if not storing as object - if (type_string != "object") { - if (py::hasattr(x, "_psp_repr_")) { - x = x.attr("_psp_repr_")(); } else { - x = x.cast(); + // Assume that the string is the type + type_string = new_type.cast(); + } + + // Extract representation if not storing as object + if (type_string != "object") { + if (py::hasattr(x, "_psp_repr_")) { + x = x.attr("_psp_repr_")(); + } else { + x = x.cast(); + } } } - } - if (x.is_none()) { - t = t_dtype::DTYPE_NONE; - } else if (py::isinstance(x) || type_string == "bool") { - // booleans are both instances of bool_ and int_ - check for bool first - t = t_dtype::DTYPE_BOOL; - } else if (type_string == "long") { - t = t_dtype::DTYPE_INT64; - } else if (py::isinstance(x) || type_string == "float") { - t = t_dtype::DTYPE_FLOAT64; - } else if (py::isinstance(x) || type_string == "int"){ - if (PY_MAJOR_VERSION < 3) { - t = t_dtype::DTYPE_INT32; - } else { + if (x.is_none()) { + t = t_dtype::DTYPE_NONE; + } else if (py::isinstance(x) || type_string == "bool") { + // booleans are both instances of bool_ and int_ - check for bool + // first + t = t_dtype::DTYPE_BOOL; + } else if (type_string == "long") { t = t_dtype::DTYPE_INT64; - } - } else if (py::isinstance(x) || type_string == "str") { - t_dtype parsed_type = date_validator.attr("format")(x).cast(); - if (parsed_type == t_dtype::DTYPE_DATE || parsed_type == t_dtype::DTYPE_TIME) { - t = parsed_type; - } else { - std::string lower = x.attr("lower")().cast(); - if (lower == "true" || lower == "false") { - t = t_dtype::DTYPE_BOOL; + } else if (py::isinstance(x) || type_string == "float") { + t = t_dtype::DTYPE_FLOAT64; + } else if (py::isinstance(x) || type_string == "int") { + if (PY_MAJOR_VERSION < 3) { + t = t_dtype::DTYPE_INT32; } else { - t = t_dtype::DTYPE_STR; + t = t_dtype::DTYPE_INT64; } + } else if (py::isinstance(x) || type_string == "str") { + t_dtype parsed_type + = date_validator.attr("format")(x).cast(); + if (parsed_type == t_dtype::DTYPE_DATE + || parsed_type == t_dtype::DTYPE_TIME) { + t = parsed_type; + } else { + std::string lower = x.attr("lower")().cast(); + if (lower == "true" || lower == "false") { + t = t_dtype::DTYPE_BOOL; + } else { + t = t_dtype::DTYPE_STR; + } + } + } else { + t = type_string_to_t_dtype(type_string); } - } else { - t = type_string_to_t_dtype(type_string); - } - - return t; -} -t_dtype -get_data_type( - t_val data, std::int32_t format, py::str name, t_val date_validator) { - std::int32_t i = 0; - boost::optional inferredType; - - if (format == 0) { - py::list data_list = data.cast(); + return t; + } - // loop parameters differ slightly so rewrite the loop - while (!inferredType.is_initialized() && i < 100 - && i < data_list.size()) { - if (!data_list.is_none()) { - if (!data_list[i].cast()[name].is_none()) { - inferredType = infer_type(data_list[i].cast()[name].cast(), date_validator); + t_dtype + get_data_type( + t_val data, std::int32_t format, py::str name, t_val date_validator) { + std::int32_t i = 0; + boost::optional inferredType; + + if (format == 0) { + py::list data_list = data.cast(); + + // loop parameters differ slightly so rewrite the loop + while (!inferredType.is_initialized() && i < 100 + && i < data_list.size()) { + if (!data_list.is_none()) { + if (!data_list[i].cast()[name].is_none()) { + inferredType = infer_type( + data_list[i].cast()[name].cast(), + date_validator); + } } + i++; } - i++; - } - } else if (format == 1) { - py::dict data_dict = data.cast(); - - while (!inferredType.is_initialized() && i < 100 - && i < data_dict[name].cast().size()) { - if (!data_dict[name].cast()[i].is_none()) { - inferredType = infer_type(data_dict[name].cast()[i].cast(), date_validator); + } else if (format == 1) { + py::dict data_dict = data.cast(); + + while (!inferredType.is_initialized() && i < 100 + && i < data_dict[name].cast().size()) { + if (!data_dict[name].cast()[i].is_none()) { + inferredType = infer_type( + data_dict[name].cast()[i].cast(), + date_validator); + } + i++; } - i++; } - } - if (!inferredType.is_initialized()) { - return t_dtype::DTYPE_STR; - } else { - return inferredType.get(); + if (!inferredType.is_initialized()) { + return t_dtype::DTYPE_STR; + } else { + return inferredType.get(); + } } -} - -std::vector -get_data_types(t_val data, std::int32_t format, std::vector names, - t_val date_validator) { - std::vector types; - if (names.size() == 0) { - WARN("Cannot determine data types without column names!"); - return types; - } + std::vector + get_data_types(t_val data, std::int32_t format, + std::vector names, t_val date_validator) { + std::vector types; + if (names.size() == 0) { + WARN("Cannot determine data types without column names!"); + return types; + } - if (format == 2) { - py::dict data_dict = data.cast(); + if (format == 2) { + py::dict data_dict = data.cast(); + + for (auto tup : data_dict) { + auto name = tup.first.cast(); + auto data_type = tup.second.get_type() + .attr("__name__") + .cast(); + std::string value; + + if (data_type == "type") { + value = py::str( + tup.second.cast().attr("__name__")) + .cast(); + } else { + value = tup.second.cast(); + } - for (auto tup : data_dict) { - auto name = tup.first.cast(); - auto data_type = tup.second.get_type().attr("__name__").cast(); - std::string value; + t_dtype type; - if (data_type == "type") { - value = py::str(tup.second.cast().attr("__name__")).cast(); - } else { - value = tup.second.cast(); + if (name == "__INDEX__") { + WARN("Warning: __INDEX__ column should not be in the Table " + "schema."); + continue; + } + type = type_string_to_t_dtype(value, name); + types.push_back(type); } - - t_dtype type; - if (name == "__INDEX__") { - WARN("Warning: __INDEX__ column should not be in the Table schema."); - continue; + return types; + } else { + types.resize(names.size()); + for (auto i = 0; i < names.size(); ++i) { + t_dtype type = get_data_type( + data, format, py::str(names[i]), date_validator); + types[i] = type; } - type = type_string_to_t_dtype(value, name); - types.push_back(type); } return types; - } else { - types.resize(names.size()); - for (auto i = 0; i < names.size(); ++i) { - t_dtype type = get_data_type(data, format, py::str(names[i]), date_validator); - types[i] = type; - } } - return types; -} - -} //namespace binding -} //namespace perspective +} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/src/column.cpp b/python/perspective/perspective/src/column.cpp index aef7140141..728ee59dc9 100644 --- a/python/perspective/perspective/src/column.cpp +++ b/python/perspective/perspective/src/column.cpp @@ -17,24 +17,25 @@ namespace perspective { template <> -void t_column::object_copied(std::uint64_t ptr) const { +void +t_column::object_copied(std::uint64_t ptr) const { // get what was there and incref if can - if (ptr){ + if (ptr) { py::handle handle = reinterpret_cast(ptr); handle.inc_ref(); } } template <> -void t_column::object_cleared(std::uint64_t ptr) const { +void +t_column::object_cleared(std::uint64_t ptr) const { // get what was there and decref if can - if (ptr){ + if (ptr) { py::handle handle = reinterpret_cast(ptr); handle.dec_ref(); } } - -} +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/src/context.cpp b/python/perspective/perspective/src/context.cpp index aff630c47d..759350b64a 100644 --- a/python/perspective/perspective/src/context.cpp +++ b/python/perspective/perspective/src/context.cpp @@ -16,142 +16,139 @@ namespace perspective { namespace binding { -/****************************************************************************** - * - * Context API - */ + /****************************************************************************** + * + * Context API + */ -template <> -std::shared_ptr -make_context(std::shared_ptr
table, std::shared_ptr schema, - std::shared_ptr view_config, const std::string& name) { - auto columns = view_config->get_columns(); - - auto cfg = t_config(columns); - auto ctx_unit = std::make_shared(*(schema.get()), cfg); - ctx_unit->init(); - - auto pool = table->get_pool(); - auto gnode = table->get_gnode(); - - pool->register_context( - gnode->get_id(), - name, - UNIT_CONTEXT, - reinterpret_cast(ctx_unit.get())); - - return ctx_unit; -} - - -template <> -std::shared_ptr -make_context(std::shared_ptr
table, std::shared_ptr schema, - std::shared_ptr view_config, const std::string& name) { - auto columns = view_config->get_columns(); - auto filter_op = view_config->get_filter_op(); - auto fterm = view_config->get_fterm(); - auto sortspec = view_config->get_sortspec(); - auto expressions = view_config->get_expressions(); - - auto cfg = t_config(columns, fterm, filter_op, expressions); - auto ctx0 = std::make_shared(*(schema.get()), cfg); - ctx0->init(); - ctx0->sort_by(sortspec); - - auto pool = table->get_pool(); - auto gnode = table->get_gnode(); - pool->register_context(gnode->get_id(), name, ZERO_SIDED_CONTEXT, - reinterpret_cast(ctx0.get())); - - return ctx0; -} - -template <> -std::shared_ptr -make_context(std::shared_ptr
table, std::shared_ptr schema, - std::shared_ptr view_config, const std::string& name) { - auto row_pivots = view_config->get_row_pivots(); - auto aggspecs = view_config->get_aggspecs(); - auto filter_op = view_config->get_filter_op(); - auto fterm = view_config->get_fterm(); - auto sortspec = view_config->get_sortspec(); - auto row_pivot_depth = view_config->get_row_pivot_depth(); - auto expressions = view_config->get_expressions(); - - auto cfg = t_config(row_pivots, aggspecs, fterm, filter_op, expressions); - auto ctx1 = std::make_shared(*(schema.get()), cfg); - - ctx1->init(); - ctx1->sort_by(sortspec); - - auto pool = table->get_pool(); - auto gnode = table->get_gnode(); - pool->register_context(gnode->get_id(), name, ONE_SIDED_CONTEXT, - reinterpret_cast(ctx1.get())); - - if (row_pivot_depth > -1) { - ctx1->set_depth(row_pivot_depth - 1); - } else { - ctx1->set_depth(row_pivots.size()); - } + template <> + std::shared_ptr + make_context(std::shared_ptr
table, std::shared_ptr schema, + std::shared_ptr view_config, const std::string& name) { + auto columns = view_config->get_columns(); - return ctx1; -} - -template <> -std::shared_ptr -make_context(std::shared_ptr
table, std::shared_ptr schema, - std::shared_ptr view_config, const std::string& name) { - bool column_only = view_config->is_column_only(); - auto row_pivots = view_config->get_row_pivots(); - auto column_pivots = view_config->get_column_pivots(); - auto aggspecs = view_config->get_aggspecs(); - auto filter_op = view_config->get_filter_op(); - auto fterm = view_config->get_fterm(); - auto sortspec = view_config->get_sortspec(); - auto col_sortspec = view_config->get_col_sortspec(); - auto row_pivot_depth = view_config->get_row_pivot_depth(); - auto column_pivot_depth = view_config->get_column_pivot_depth(); - auto expressions = view_config->get_expressions(); - - t_totals total = sortspec.size() > 0 ? TOTALS_BEFORE : TOTALS_HIDDEN; - - auto cfg = t_config( - row_pivots, column_pivots, aggspecs, total, fterm, filter_op, expressions, column_only); - auto ctx2 = std::make_shared(*(schema.get()), cfg); - - ctx2->init(); - - auto pool = table->get_pool(); - auto gnode = table->get_gnode(); - pool->register_context(gnode->get_id(), name, TWO_SIDED_CONTEXT, - reinterpret_cast(ctx2.get())); - - if (row_pivot_depth > -1) { - ctx2->set_depth(t_header::HEADER_ROW, row_pivot_depth - 1); - } else { - ctx2->set_depth(t_header::HEADER_ROW, row_pivots.size()); - } + auto cfg = t_config(columns); + auto ctx_unit = std::make_shared(*(schema.get()), cfg); + ctx_unit->init(); + + auto pool = table->get_pool(); + auto gnode = table->get_gnode(); - if (column_pivot_depth > -1) { - ctx2->set_depth(t_header::HEADER_COLUMN, column_pivot_depth - 1); - } else { - ctx2->set_depth(t_header::HEADER_COLUMN, column_pivots.size()); + pool->register_context(gnode->get_id(), name, UNIT_CONTEXT, + reinterpret_cast(ctx_unit.get())); + + return ctx_unit; } - if (sortspec.size() > 0) { - ctx2->sort_by(sortspec); + template <> + std::shared_ptr + make_context(std::shared_ptr
table, std::shared_ptr schema, + std::shared_ptr view_config, const std::string& name) { + auto columns = view_config->get_columns(); + auto filter_op = view_config->get_filter_op(); + auto fterm = view_config->get_fterm(); + auto sortspec = view_config->get_sortspec(); + auto expressions = view_config->get_expressions(); + + auto cfg = t_config(columns, fterm, filter_op, expressions); + auto ctx0 = std::make_shared(*(schema.get()), cfg); + ctx0->init(); + ctx0->sort_by(sortspec); + + auto pool = table->get_pool(); + auto gnode = table->get_gnode(); + pool->register_context(gnode->get_id(), name, ZERO_SIDED_CONTEXT, + reinterpret_cast(ctx0.get())); + + return ctx0; } - if (col_sortspec.size() > 0) { - ctx2->column_sort_by(col_sortspec); + template <> + std::shared_ptr + make_context(std::shared_ptr
table, std::shared_ptr schema, + std::shared_ptr view_config, const std::string& name) { + auto row_pivots = view_config->get_row_pivots(); + auto aggspecs = view_config->get_aggspecs(); + auto filter_op = view_config->get_filter_op(); + auto fterm = view_config->get_fterm(); + auto sortspec = view_config->get_sortspec(); + auto row_pivot_depth = view_config->get_row_pivot_depth(); + auto expressions = view_config->get_expressions(); + + auto cfg + = t_config(row_pivots, aggspecs, fterm, filter_op, expressions); + auto ctx1 = std::make_shared(*(schema.get()), cfg); + + ctx1->init(); + ctx1->sort_by(sortspec); + + auto pool = table->get_pool(); + auto gnode = table->get_gnode(); + pool->register_context(gnode->get_id(), name, ONE_SIDED_CONTEXT, + reinterpret_cast(ctx1.get())); + + if (row_pivot_depth > -1) { + ctx1->set_depth(row_pivot_depth - 1); + } else { + ctx1->set_depth(row_pivots.size()); + } + + return ctx1; } - return ctx2; -} + template <> + std::shared_ptr + make_context(std::shared_ptr
table, std::shared_ptr schema, + std::shared_ptr view_config, const std::string& name) { + bool column_only = view_config->is_column_only(); + auto row_pivots = view_config->get_row_pivots(); + auto column_pivots = view_config->get_column_pivots(); + auto aggspecs = view_config->get_aggspecs(); + auto filter_op = view_config->get_filter_op(); + auto fterm = view_config->get_fterm(); + auto sortspec = view_config->get_sortspec(); + auto col_sortspec = view_config->get_col_sortspec(); + auto row_pivot_depth = view_config->get_row_pivot_depth(); + auto column_pivot_depth = view_config->get_column_pivot_depth(); + auto expressions = view_config->get_expressions(); + + t_totals total = sortspec.size() > 0 ? TOTALS_BEFORE : TOTALS_HIDDEN; + + auto cfg = t_config(row_pivots, column_pivots, aggspecs, total, fterm, + filter_op, expressions, column_only); + auto ctx2 = std::make_shared(*(schema.get()), cfg); + + ctx2->init(); + + auto pool = table->get_pool(); + auto gnode = table->get_gnode(); + pool->register_context(gnode->get_id(), name, TWO_SIDED_CONTEXT, + reinterpret_cast(ctx2.get())); + + if (row_pivot_depth > -1) { + ctx2->set_depth(t_header::HEADER_ROW, row_pivot_depth - 1); + } else { + ctx2->set_depth(t_header::HEADER_ROW, row_pivots.size()); + } + + if (column_pivot_depth > -1) { + ctx2->set_depth(t_header::HEADER_COLUMN, column_pivot_depth - 1); + } else { + ctx2->set_depth(t_header::HEADER_COLUMN, column_pivots.size()); + } + + if (sortspec.size() > 0) { + ctx2->sort_by(sortspec); + } + + if (col_sortspec.size() > 0) { + ctx2->column_sort_by(col_sortspec); + } + + return ctx2; + } -} //namespace binding -} //namespace perspective +} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/src/expressions.cpp b/python/perspective/perspective/src/expressions.cpp index 9b720121a4..3ae7e9ed0c 100644 --- a/python/perspective/perspective/src/expressions.cpp +++ b/python/perspective/perspective/src/expressions.cpp @@ -13,49 +13,47 @@ namespace perspective { namespace binding { -void -init_expression_parser() { - t_computed_expression_parser::init(); -} - -t_validated_expression_map -validate_expressions_py( - std::shared_ptr
table, - const std::vector>& p_expressions) { - std::vector>>> expressions; - expressions.resize(p_expressions.size()); - - // Convert from vector of t_val into vector of tuples - for (t_uindex idx = 0; idx < p_expressions.size(); ++idx) { - const auto& expr = p_expressions[idx]; - std::string expression_alias = expr[0].cast(); - std::string expression_string = expr[1].cast(); - std::string parsed_expression_string = expr[2].cast(); - - auto p_column_ids = py::dict(expr[3]); - std::vector> column_ids; - column_ids.resize(p_column_ids.size()); - t_uindex cidx = 0; - - for (const auto& item : p_column_ids) { - column_ids[cidx] = std::pair( - item.first.cast(), - item.second.cast()); - ++cidx; - } + void + init_expression_parser() { + t_computed_expression_parser::init(); + } - auto tp = std::make_tuple( - expression_alias, - expression_string, - parsed_expression_string, - column_ids); + t_validated_expression_map + validate_expressions_py(std::shared_ptr
table, + const std::vector>& p_expressions) { + std::vector>>> + expressions; + expressions.resize(p_expressions.size()); + + // Convert from vector of t_val into vector of tuples + for (t_uindex idx = 0; idx < p_expressions.size(); ++idx) { + const auto& expr = p_expressions[idx]; + std::string expression_alias = expr[0].cast(); + std::string expression_string = expr[1].cast(); + std::string parsed_expression_string = expr[2].cast(); + + auto p_column_ids = py::dict(expr[3]); + std::vector> column_ids; + column_ids.resize(p_column_ids.size()); + t_uindex cidx = 0; + + for (const auto& item : p_column_ids) { + column_ids[cidx] = std::pair( + item.first.cast(), + item.second.cast()); + ++cidx; + } + + auto tp = std::make_tuple(expression_alias, expression_string, + parsed_expression_string, column_ids); + + expressions[idx] = tp; + } - expressions[idx] = tp; + return table->validate_expressions(expressions); } - return table->validate_expressions(expressions); -} - } // end namespace binding } // end namespace perspective diff --git a/python/perspective/perspective/src/fill.cpp b/python/perspective/perspective/src/fill.cpp index b559dcd4eb..25ab2689cb 100644 --- a/python/perspective/perspective/src/fill.cpp +++ b/python/perspective/perspective/src/fill.cpp @@ -17,398 +17,441 @@ namespace perspective { namespace binding { -/****************************************************************************** - * - * Fill columns with data - */ - -void -_fill_col_time(t_data_accessor accessor, std::shared_ptr col, std::string name, - std::int32_t cidx, t_dtype type, bool is_update, bool is_limit) { - t_uindex nrows = col->size(); - - for (auto i = 0; i < nrows; ++i) { - if (!accessor.attr("_has_column")(i, name).cast() && !is_limit) { - continue; - } + /****************************************************************************** + * + * Fill columns with data + */ + + void + _fill_col_time(t_data_accessor accessor, std::shared_ptr col, + std::string name, std::int32_t cidx, t_dtype type, bool is_update, + bool is_limit) { + t_uindex nrows = col->size(); + + for (auto i = 0; i < nrows; ++i) { + if (!accessor.attr("_has_column")(i, name).cast() + && !is_limit) { + continue; + } - t_val item = accessor.attr("marshal")(cidx, i, type); + t_val item = accessor.attr("marshal")(cidx, i, type); - if (item.is_none()) { - if (is_update) { - col->unset(i); - } else { - col->clear(i); + if (item.is_none()) { + if (is_update) { + col->unset(i); + } else { + col->clear(i); + } + continue; } - continue; - } - col->set_nth(i, item.cast()); + col->set_nth(i, item.cast()); + } } -} -void -_fill_col_date(t_data_accessor accessor, std::shared_ptr col, std::string name, - std::int32_t cidx, t_dtype type, bool is_update, bool is_limit) { - t_uindex nrows = col->size(); + void + _fill_col_date(t_data_accessor accessor, std::shared_ptr col, + std::string name, std::int32_t cidx, t_dtype type, bool is_update, + bool is_limit) { + t_uindex nrows = col->size(); - for (auto i = 0; i < nrows; ++i) { - if (!accessor.attr("_has_column")(i, name).cast() && !is_limit) { - continue; - } + for (auto i = 0; i < nrows; ++i) { + if (!accessor.attr("_has_column")(i, name).cast() + && !is_limit) { + continue; + } - t_val item = accessor.attr("marshal")(cidx, i, type); + t_val item = accessor.attr("marshal")(cidx, i, type); - if (item.is_none()) { - if (is_update) { - col->unset(i); - } else { - col->clear(i); + if (item.is_none()) { + if (is_update) { + col->unset(i); + } else { + col->clear(i); + } + continue; } - continue; - } - - auto date_components = item.cast>(); - t_date dt = t_date(date_components["year"], date_components["month"], date_components["day"]); - col->set_nth(i, dt); + auto date_components + = item.cast>(); + t_date dt = t_date(date_components["year"], + date_components["month"], date_components["day"]); + col->set_nth(i, dt); + } } -} -void -_fill_col_bool(t_data_accessor accessor, std::shared_ptr col, std::string name, - std::int32_t cidx, t_dtype type, bool is_update, bool is_limit) { - t_uindex nrows = col->size(); + void + _fill_col_bool(t_data_accessor accessor, std::shared_ptr col, + std::string name, std::int32_t cidx, t_dtype type, bool is_update, + bool is_limit) { + t_uindex nrows = col->size(); - for (auto i = 0; i < nrows; ++i) { - if (!accessor.attr("_has_column")(i, name).cast() && !is_limit) { - continue; - } + for (auto i = 0; i < nrows; ++i) { + if (!accessor.attr("_has_column")(i, name).cast() + && !is_limit) { + continue; + } - t_val item = accessor.attr("marshal")(cidx, i, type); + t_val item = accessor.attr("marshal")(cidx, i, type); - if (item.is_none()) { - if (is_update) { - col->unset(i); - } else { - col->clear(i); + if (item.is_none()) { + if (is_update) { + col->unset(i); + } else { + col->clear(i); + } + continue; } - continue; - } - auto elem = item.cast(); - col->set_nth(i, elem); + auto elem = item.cast(); + col->set_nth(i, elem); + } } -} -void -_fill_col_string(t_data_accessor accessor, std::shared_ptr col, std::string name, - std::int32_t cidx, t_dtype type, bool is_update, bool is_limit) { + void + _fill_col_string(t_data_accessor accessor, std::shared_ptr col, + std::string name, std::int32_t cidx, t_dtype type, bool is_update, + bool is_limit) { - t_uindex nrows = col->size(); + t_uindex nrows = col->size(); - for (auto i = 0; i < nrows; ++i) { - if (!accessor.attr("_has_column")(i, name).cast() && !is_limit) { - continue; - } + for (auto i = 0; i < nrows; ++i) { + if (!accessor.attr("_has_column")(i, name).cast() + && !is_limit) { + continue; + } - t_val item = accessor.attr("marshal")(cidx, i, type); + t_val item = accessor.attr("marshal")(cidx, i, type); - if (item.is_none()) { - if (is_update) { - col->unset(i); - } else { - col->clear(i); + if (item.is_none()) { + if (is_update) { + col->unset(i); + } else { + col->clear(i); + } + continue; } - continue; - } - col->set_nth(i, item.cast()); - } -} - -template <> -void -set_column_nth(std::shared_ptr col, t_uindex idx, t_val value) { - if (value.is_none()) { - col->unset(idx); - return; + col->set_nth(i, item.cast()); + } } - switch (col->get_dtype()) { - case DTYPE_BOOL: { - col->set_nth(idx, value.cast(), STATUS_VALID); - break; + template <> + void + set_column_nth(std::shared_ptr col, t_uindex idx, t_val value) { + if (value.is_none()) { + col->unset(idx); + return; } - case DTYPE_FLOAT64: { - col->set_nth(idx, value.cast(), STATUS_VALID); - break; - } - case DTYPE_FLOAT32: { - col->set_nth(idx, value.cast(), STATUS_VALID); - break; - } - case DTYPE_UINT32: { - col->set_nth(idx, value.cast(), STATUS_VALID); - break; - } - case DTYPE_UINT64: { - col->set_nth(idx, value.cast(), STATUS_VALID); - break; - } - case DTYPE_INT32: { - col->set_nth(idx, value.cast(), STATUS_VALID); - break; - } - case DTYPE_INT64: { - col->set_nth(idx, value.cast(), STATUS_VALID); - break; - } - case DTYPE_STR: { - col->set_nth(idx, value.cast(), STATUS_VALID); - break; - } - case DTYPE_DATE: { - t_date dt = t_date(value.attr("year").cast(), - value.attr("month").cast(), - value.attr("day").cast()); - col->set_nth(idx, dt, STATUS_VALID); - break; - } - case DTYPE_TIME: { - col->set_nth( - idx, static_cast(value.cast()), STATUS_VALID); - break; - } - case DTYPE_UINT8: - case DTYPE_UINT16: - case DTYPE_INT8: - case DTYPE_INT16: - default: { - // Other types not implemented + + switch (col->get_dtype()) { + case DTYPE_BOOL: { + col->set_nth(idx, value.cast(), STATUS_VALID); + break; + } + case DTYPE_FLOAT64: { + col->set_nth(idx, value.cast(), STATUS_VALID); + break; + } + case DTYPE_FLOAT32: { + col->set_nth(idx, value.cast(), STATUS_VALID); + break; + } + case DTYPE_UINT32: { + col->set_nth( + idx, value.cast(), STATUS_VALID); + break; + } + case DTYPE_UINT64: { + col->set_nth( + idx, value.cast(), STATUS_VALID); + break; + } + case DTYPE_INT32: { + col->set_nth( + idx, value.cast(), STATUS_VALID); + break; + } + case DTYPE_INT64: { + col->set_nth( + idx, value.cast(), STATUS_VALID); + break; + } + case DTYPE_STR: { + col->set_nth(idx, value.cast(), STATUS_VALID); + break; + } + case DTYPE_DATE: { + t_date dt = t_date(value.attr("year").cast(), + value.attr("month").cast(), + value.attr("day").cast()); + col->set_nth(idx, dt, STATUS_VALID); + break; + } + case DTYPE_TIME: { + col->set_nth(idx, + static_cast(value.cast()), + STATUS_VALID); + break; + } + case DTYPE_UINT8: + case DTYPE_UINT16: + case DTYPE_INT8: + case DTYPE_INT16: + default: { + // Other types not implemented + } } } -} -void -_fill_col_numeric(t_data_accessor accessor, t_data_table& tbl, - std::shared_ptr col, std::string name, std::int32_t cidx, t_dtype type, bool is_update, bool is_limit) { - t_uindex nrows = col->size(); + void + _fill_col_numeric(t_data_accessor accessor, t_data_table& tbl, + std::shared_ptr col, std::string name, std::int32_t cidx, + t_dtype type, bool is_update, bool is_limit) { + t_uindex nrows = col->size(); - for (auto i = 0; i < nrows; ++i) { - if (!accessor.attr("_has_column")(i, name).cast() && !is_limit) { - continue; - } + for (auto i = 0; i < nrows; ++i) { + if (!accessor.attr("_has_column")(i, name).cast() + && !is_limit) { + continue; + } - t_val item = accessor.attr("marshal")(cidx, i, type); + t_val item = accessor.attr("marshal")(cidx, i, type); - if (item.is_none()) { - if (is_update) { - col->unset(i); - } else { - col->clear(i); + if (item.is_none()) { + if (is_update) { + col->unset(i); + } else { + col->clear(i); + } + continue; } - continue; - } - switch (type) { - case DTYPE_UINT8: { - col->set_nth(i, item.cast()); - } break; - case DTYPE_UINT16: { - col->set_nth(i, item.cast()); - } break; - case DTYPE_UINT32: { - col->set_nth(i, item.cast()); - } break; - case DTYPE_UINT64: { - col->set_nth(i, item.cast()); - } break; - case DTYPE_INT8: { - col->set_nth(i, item.cast()); - } break; - case DTYPE_INT16: { - col->set_nth(i, item.cast()); - } break; - case DTYPE_INT32: { - // This handles cases where a long sequence of e.g. 0 precedes a clearly - // float value in an inferred column. Would not be needed if the type - // inference checked the entire column/we could reset parsing. - - // First we need to see if we can cast to double - double fval; - if (!py::hasattr(item, "__float__")) { - if (py::hasattr(item, "__int__")) { - // promote from int - fval = static_cast(item.cast()); + switch (type) { + case DTYPE_UINT8: { + col->set_nth(i, item.cast()); + } break; + case DTYPE_UINT16: { + col->set_nth(i, item.cast()); + } break; + case DTYPE_UINT32: { + col->set_nth(i, item.cast()); + } break; + case DTYPE_UINT64: { + col->set_nth(i, item.cast()); + } break; + case DTYPE_INT8: { + col->set_nth(i, item.cast()); + } break; + case DTYPE_INT16: { + col->set_nth(i, item.cast()); + } break; + case DTYPE_INT32: { + // This handles cases where a long sequence of e.g. 0 + // precedes a clearly float value in an inferred column. + // Would not be needed if the type inference checked the + // entire column/we could reset parsing. + + // First we need to see if we can cast to double + double fval; + if (!py::hasattr(item, "__float__")) { + if (py::hasattr(item, "__int__")) { + // promote from int + fval = static_cast(item.cast()); + } else { + // not __float__ and no __int__ defined, set to NaN + fval = std::nan(""); + } } else { - // not __float__ and no __int__ defined, set to NaN - fval = std::nan(""); + fval = item.cast(); } - } else { - fval = item.cast(); - } - if (!is_update && (fval > 2147483647 || fval < -2147483648)) { - WARN("Promoting column `%s` to float from int32", name); - tbl.promote_column(name, DTYPE_FLOAT64, i, true); - col = tbl.get_column(name); - type = DTYPE_FLOAT64; - col->set_nth(i, fval); - } else if (!is_update && isnan(fval)) { - WARN("Promoting column `%s` to string from int32", name); - tbl.promote_column(name, DTYPE_STR, i, false); - col = tbl.get_column(name); - _fill_col_string( - accessor, col, name, cidx, DTYPE_STR, is_update, is_limit); - return; - } else { - col->set_nth(i, static_cast(fval)); - } - } break; - case DTYPE_INT64: { - // First we need to see if we can cast to double - double fval; - if (!py::hasattr(item, "__float__")) { - if (py::hasattr(item, "__int__")) { - // promote from int - fval = static_cast(item.cast()); + if (!is_update + && (fval > 2147483647 || fval < -2147483648)) { + WARN("Promoting column `%s` to float from int32", name); + tbl.promote_column(name, DTYPE_FLOAT64, i, true); + col = tbl.get_column(name); + type = DTYPE_FLOAT64; + col->set_nth(i, fval); + } else if (!is_update && isnan(fval)) { + WARN( + "Promoting column `%s` to string from int32", name); + tbl.promote_column(name, DTYPE_STR, i, false); + col = tbl.get_column(name); + _fill_col_string(accessor, col, name, cidx, DTYPE_STR, + is_update, is_limit); + return; } else { - // not __float__ and no __int__ defined, set to NaN - fval = std::nan(""); + col->set_nth(i, static_cast(fval)); + } + } break; + case DTYPE_INT64: { + // First we need to see if we can cast to double + double fval; + if (!py::hasattr(item, "__float__")) { + if (py::hasattr(item, "__int__")) { + // promote from int + fval = static_cast(item.cast()); + } else { + // not __float__ and no __int__ defined, set to NaN + fval = std::nan(""); + } + } else { + fval = item.cast(); } - } else { - fval = item.cast(); - } - if (!is_update && isnan(fval)) { - WARN("Promoting column `%s` to string from int64", name); - tbl.promote_column(name, DTYPE_STR, i, false); - col = tbl.get_column(name); - _fill_col_string( - accessor, col, name, cidx, DTYPE_STR, is_update, is_limit); - return; - } else { - col->set_nth(i, static_cast(fval)); - } - } break; - case DTYPE_FLOAT32: { - col->set_nth(i, item.cast()); - } break; - case DTYPE_FLOAT64: { - bool is_float = py::isinstance(item) || py::hasattr(item, "__float__") || py::hasattr(item, "__int__"); + if (!is_update && isnan(fval)) { + WARN( + "Promoting column `%s` to string from int64", name); + tbl.promote_column(name, DTYPE_STR, i, false); + col = tbl.get_column(name); + _fill_col_string(accessor, col, name, cidx, DTYPE_STR, + is_update, is_limit); + return; + } else { + col->set_nth(i, static_cast(fval)); + } + } break; + case DTYPE_FLOAT32: { + col->set_nth(i, item.cast()); + } break; + case DTYPE_FLOAT64: { + bool is_float = py::isinstance(item) + || py::hasattr(item, "__float__") + || py::hasattr(item, "__int__"); + + bool is_numpy_nan = false; + if (py::isinstance(item) + || py::hasattr(item, "__float__")) { + is_numpy_nan = npy_isnan(item.cast()); + } - bool is_numpy_nan = false; - if (py::isinstance(item) || py::hasattr(item, "__float__")) { - is_numpy_nan = npy_isnan(item.cast()); - } + if (!is_update && (!is_float || is_numpy_nan)) { + WARN("Promoting column `%s` to string from float64", + name); + tbl.promote_column(name, DTYPE_STR, i, false); + col = tbl.get_column(name); + _fill_col_string(accessor, col, name, cidx, DTYPE_STR, + is_update, is_limit); + return; + } - if (!is_update && (!is_float || is_numpy_nan)) { - WARN("Promoting column `%s` to string from float64", name); - tbl.promote_column(name, DTYPE_STR, i, false); - col = tbl.get_column(name); - _fill_col_string( - accessor, col, name, cidx, DTYPE_STR, is_update, is_limit); - return; - } + // If not a float directly and doesn't have __float__, must + // promote with __int__ + if (!py::isinstance(item) + && !py::hasattr(item, "__float__")) { + col->set_nth( + i, static_cast(item.cast())); + } else { + col->set_nth(i, item.cast()); + } + } break; + case DTYPE_OBJECT: { + // Store pointer as uint64 (in 32-bit will promote to + // 64bits, should be ok) + std::uint64_t store = item.is_none() + ? 0 + : reinterpret_cast(item.ptr()); + + // Increment the reference count to account for internal + // storage of the raw pointer (don't actually do this as + // _process_column will handle it) + item.inc_ref(); // don't uncomment + + // Store the pointer a uint64 + col->set_nth(i, store); + col->set_valid(i, store != 0); + } break; + default: + break; + } + } + } - // If not a float directly and doesn't have __float__, must promote with __int__ - if (!py::isinstance(item) && !py::hasattr(item, "__float__")) { - col->set_nth(i, static_cast(item.cast())); - } else { - col->set_nth(i, item.cast()); - } + void + _fill_data_helper(t_data_accessor accessor, t_data_table& tbl, + std::shared_ptr col, std::string name, std::int32_t cidx, + t_dtype type, bool is_update, bool is_limit) { + switch (type) { + case DTYPE_BOOL: { + _fill_col_bool( + accessor, col, name, cidx, type, is_update, is_limit); } break; - case DTYPE_OBJECT: { - // Store pointer as uint64 (in 32-bit will promote to 64bits, should be ok) - std::uint64_t store = item.is_none() ? 0: reinterpret_cast(item.ptr()); - - // Increment the reference count to account for internal storage of the raw pointer - // (don't actually do this as _process_column will handle it) - item.inc_ref(); // don't uncomment - - // Store the pointer a uint64 - col->set_nth(i, store); - col->set_valid(i, store!=0); + case DTYPE_DATE: { + _fill_col_date( + accessor, col, name, cidx, type, is_update, is_limit); } break; - default: + case DTYPE_TIME: { + _fill_col_time( + accessor, col, name, cidx, type, is_update, is_limit); + } break; + case DTYPE_STR: { + _fill_col_string( + accessor, col, name, cidx, type, is_update, is_limit); + } break; + case DTYPE_NONE: { break; + } + default: + _fill_col_numeric( + accessor, tbl, col, name, cidx, type, is_update, is_limit); } } -} - -void -_fill_data_helper(t_data_accessor accessor, t_data_table& tbl, - std::shared_ptr col, std::string name, std::int32_t cidx, t_dtype type, bool is_update, bool is_limit) { - switch (type) { - case DTYPE_BOOL: { - _fill_col_bool(accessor, col, name, cidx, type, is_update, is_limit); - } break; - case DTYPE_DATE: { - _fill_col_date(accessor, col, name, cidx, type, is_update, is_limit); - } break; - case DTYPE_TIME: { - _fill_col_time(accessor, col, name, cidx, type, is_update, is_limit); - } break; - case DTYPE_STR: { - _fill_col_string(accessor, col, name, cidx, type, is_update, is_limit); - } break; - case DTYPE_NONE: { - break; - } - default: - _fill_col_numeric( - accessor, tbl, col, name, cidx, type, is_update, is_limit); - } -} -/****************************************************************************** - * - * Fill tables with data - */ + /****************************************************************************** + * + * Fill tables with data + */ + + void + _fill_data(t_data_table& tbl, t_data_accessor accessor, + const t_schema& input_schema, const std::string& index, + std::uint32_t offset, std::uint32_t limit, bool is_update) { + bool implicit_index = false; + bool is_limit = limit != UINT32_MAX; + std::vector col_names(input_schema.columns()); + std::vector data_types(input_schema.types()); + + for (auto cidx = 0; cidx < col_names.size(); ++cidx) { + auto name = col_names[cidx]; + auto type = data_types[cidx]; + + if (name == "__INDEX__") { + implicit_index = true; + std::shared_ptr pkey_col_sptr + = tbl.add_column_sptr("psp_pkey", type, true); + _fill_data_helper(accessor, tbl, pkey_col_sptr, "psp_pkey", + cidx, type, is_update, is_limit); + tbl.clone_column("psp_pkey", "psp_okey"); + continue; + } -void -_fill_data(t_data_table& tbl, t_data_accessor accessor, const t_schema& input_schema, - const std::string& index, std::uint32_t offset, std::uint32_t limit, bool is_update) { - bool implicit_index = false; - bool is_limit = limit != UINT32_MAX; - std::vector col_names(input_schema.columns()); - std::vector data_types(input_schema.types()); - - for (auto cidx = 0; cidx < col_names.size(); ++cidx) { - auto name = col_names[cidx]; - auto type = data_types[cidx]; - - if (name == "__INDEX__") { - implicit_index = true; - std::shared_ptr pkey_col_sptr = tbl.add_column_sptr("psp_pkey", type, true); - _fill_data_helper(accessor, tbl, pkey_col_sptr, "psp_pkey", cidx, type, is_update, is_limit); - tbl.clone_column("psp_pkey", "psp_okey"); - continue; - } - - auto col = tbl.get_column(name); - _fill_data_helper(accessor, tbl, col, name, cidx, type, is_update, is_limit); - } - // Fill index column - recreated every time a `t_data_table` is created. - if (!implicit_index) { - if (index == "") { - // Use row number as index if not explicitly provided or provided with `__INDEX__` - auto key_col = tbl.add_column("psp_pkey", DTYPE_INT32, true); - auto okey_col = tbl.add_column("psp_okey", DTYPE_INT32, true); - - for (std::uint32_t ridx = 0; ridx < tbl.size(); ++ridx) { - key_col->set_nth(ridx, (ridx + offset) % limit); - okey_col->set_nth(ridx, (ridx + offset) % limit); + auto col = tbl.get_column(name); + _fill_data_helper( + accessor, tbl, col, name, cidx, type, is_update, is_limit); + } + // Fill index column - recreated every time a `t_data_table` is created. + if (!implicit_index) { + if (index == "") { + // Use row number as index if not explicitly provided or + // provided with `__INDEX__` + auto key_col = tbl.add_column("psp_pkey", DTYPE_INT32, true); + auto okey_col = tbl.add_column("psp_okey", DTYPE_INT32, true); + + for (std::uint32_t ridx = 0; ridx < tbl.size(); ++ridx) { + key_col->set_nth( + ridx, (ridx + offset) % limit); + okey_col->set_nth( + ridx, (ridx + offset) % limit); + } + } else { + tbl.clone_column(index, "psp_pkey"); + tbl.clone_column(index, "psp_okey"); } - } else { - tbl.clone_column(index, "psp_pkey"); - tbl.clone_column(index, "psp_okey"); } } -} -} //namespace binding -} //namespace perspective +} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/src/numpy.cpp b/python/perspective/perspective/src/numpy.cpp index acd447f363..542e98b5fc 100644 --- a/python/perspective/perspective/src/numpy.cpp +++ b/python/perspective/perspective/src/numpy.cpp @@ -15,7 +15,8 @@ using namespace perspective; namespace perspective { namespace numpy { - const std::vector NumpyLoader::DATE_UNITS = {"[D]", "[W]", "[M]", "[Y]"}; + const std::vector NumpyLoader::DATE_UNITS + = {"[D]", "[W]", "[M]", "[Y]"}; NumpyLoader::NumpyLoader(t_val accessor) : m_init(false) @@ -30,14 +31,17 @@ namespace numpy { m_init = true; } - std::vector - NumpyLoader::reconcile_dtypes(const std::vector& inferred_types) const { + std::vector + NumpyLoader::reconcile_dtypes( + const std::vector& inferred_types) const { PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); std::uint32_t num_columns = m_names.size(); std::vector reconciled_types(num_columns); - // Get numpy dtypes as string so we can tell the difference between dates and datetimes - std::vector str_dtypes = m_accessor.attr("types")().cast>(); + // Get numpy dtypes as string so we can tell the difference between + // dates and datetimes + std::vector str_dtypes + = m_accessor.attr("types")().cast>(); for (auto i = 0; i < num_columns; ++i) { std::string numpy_type_as_string = str_dtypes[i]; @@ -45,7 +49,7 @@ namespace numpy { t_dtype inferred_type = inferred_types[i]; // Check whether column is a date or a datetime - if(numpy_type_as_string.find("datetime64") != std::string::npos) { + if (numpy_type_as_string.find("datetime64") != std::string::npos) { for (const std::string& unit : DATE_UNITS) { if (numpy_type_as_string.find(unit) != std::string::npos) { inferred_type = DTYPE_DATE; @@ -53,7 +57,8 @@ namespace numpy { } } - // Otherwise, numpy type takes precedence unless date/object - need specificity of inferred type + // Otherwise, numpy type takes precedence unless date/object - need + // specificity of inferred type if (inferred_type == DTYPE_DATE || numpy_type == DTYPE_OBJECT) { reconciled_types[i] = inferred_type; } else { @@ -84,7 +89,8 @@ namespace numpy { void NumpyLoader::fill_table(t_data_table& tbl, const t_schema& input_schema, - const std::string& index, std::uint32_t offset, std::uint32_t limit, bool is_update) { + const std::string& index, std::uint32_t offset, std::uint32_t limit, + bool is_update) { PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); bool implicit_index = false; std::vector col_names(input_schema.columns()); @@ -96,8 +102,10 @@ namespace numpy { if (name == "__INDEX__") { implicit_index = true; - std::shared_ptr pkey_col_sptr = tbl.add_column_sptr("psp_pkey", type, true); - fill_column(tbl, pkey_col_sptr, "__INDEX__", type, cidx, is_update); + std::shared_ptr pkey_col_sptr + = tbl.add_column_sptr("psp_pkey", type, true); + fill_column( + tbl, pkey_col_sptr, "__INDEX__", type, cidx, is_update); tbl.clone_column("psp_pkey", "psp_okey"); continue; } @@ -109,13 +117,16 @@ namespace numpy { // Fill index column - recreated every time a `t_data_table` is created. if (!implicit_index) { if (index == "") { - // Use row number as index if not explicitly provided or provided with `__INDEX__` + // Use row number as index if not explicitly provided or + // provided with `__INDEX__` auto key_col = tbl.add_column("psp_pkey", DTYPE_INT32, true); auto okey_col = tbl.add_column("psp_okey", DTYPE_INT32, true); for (std::uint32_t ridx = 0; ridx < tbl.size(); ++ridx) { - key_col->set_nth(ridx, (ridx + offset) % limit); - okey_col->set_nth(ridx, (ridx + offset) % limit); + key_col->set_nth( + ridx, (ridx + offset) % limit); + okey_col->set_nth( + ridx, (ridx + offset) % limit); } } else { tbl.clone_column(index, "psp_pkey"); @@ -124,14 +135,16 @@ namespace numpy { } } - - void - NumpyLoader::fill_column(t_data_table& tbl, std::shared_ptr col, const std::string& name, t_dtype type, std::uint32_t cidx, bool is_update) { + void + NumpyLoader::fill_column(t_data_table& tbl, std::shared_ptr col, + const std::string& name, t_dtype type, std::uint32_t cidx, + bool is_update) { PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); - // Use name index instead of column index - prevents off-by-one errors with the "index" column. - auto name_it = std::find(m_names.begin(), m_names.end(), name); - + // Use name index instead of column index - prevents off-by-one errors + // with the "index" column. + auto name_it = std::find(m_names.begin(), m_names.end(), name); + // If the column name is not in the dataset, return and move on. if (name_it == m_names.end()) { return; @@ -141,44 +154,61 @@ namespace numpy { // np_dtype is one of the integer/float/bool dtypes, or `DTYPE_OBJECT`. t_dtype np_dtype = m_types[nidx]; - + py::dict source = m_accessor.attr("_get_numpy_column")(name); py::array array = source["array"].cast(); - py::array_t mask = source["mask"].cast>(); - std::uint64_t* mask_ptr = (std::uint64_t*) mask.data(); + py::array_t mask + = source["mask"].cast>(); + std::uint64_t* mask_ptr = (std::uint64_t*)mask.data(); std::size_t mask_size = mask.size(); - // Check array dtype to make sure that `deconstruct_numpy` didn't cast it to an object. + // Check array dtype to make sure that `deconstruct_numpy` didn't cast + // it to an object. if (array.dtype().kind() == 'O') { - fill_column_iter(array, tbl, col, name, DTYPE_OBJECT, type, cidx, is_update); + fill_column_iter( + array, tbl, col, name, DTYPE_OBJECT, type, cidx, is_update); return; } - // Datetimes are not trivially copyable - they are float64 values that need to be read as int64 + // Datetimes are not trivially copyable - they are float64 values that + // need to be read as int64 if (type == DTYPE_TIME || type == DTYPE_DATE) { - fill_column_iter(array, tbl, col, name, np_dtype, type, cidx, is_update); + fill_column_iter( + array, tbl, col, name, np_dtype, type, cidx, is_update); fill_validity_map(col, mask_ptr, mask_size, is_update); return; } - - /** - * Catch common type mismatches and fill iteratively when a numpy dtype is of greater bit width than the Perspective t_dtype: - * - when `np_dtype` is int64 and `t_dtype` is `DTYPE_INT32` or `DTYPE_FLOAT64` - * - when `np_dtype` is int32 and `t_dtype` is `DTYPE_INT64` or `DTYPE_FLOAT64`, which can happen on windows where np::int_ is int32 - * - when `np_dtype` is float64 and `t_dtype` is `DTYPE_INT32` or `DTYPE_INT64` - * - when `type` is float64 and `np_dtype` is `DTYPE_FLOAT32` or `DTYPE_FLOAT64` - * - * These errors occur frqeuently when a Table is created from non-numpy data or schema, then updated with a numpy array. - * In these cases, the `t_dtype` of the Table supercedes the array dtype. + + /** + * Catch common type mismatches and fill iteratively when a numpy dtype + * is of greater bit width than the Perspective t_dtype: + * - when `np_dtype` is int64 and `t_dtype` is `DTYPE_INT32` or + * `DTYPE_FLOAT64` + * - when `np_dtype` is int32 and `t_dtype` is `DTYPE_INT64` or + * `DTYPE_FLOAT64`, which can happen on windows where np::int_ is int32 + * - when `np_dtype` is float64 and `t_dtype` is `DTYPE_INT32` or + * `DTYPE_INT64` + * - when `type` is float64 and `np_dtype` is `DTYPE_FLOAT32` or + * `DTYPE_FLOAT64` + * + * These errors occur frqeuently when a Table is created from non-numpy + * data or schema, then updated with a numpy array. In these cases, the + * `t_dtype` of the Table supercedes the array dtype. */ - bool should_iter = (np_dtype == DTYPE_INT64 && (type == DTYPE_INT32 || type == DTYPE_FLOAT64)) || \ - (np_dtype == DTYPE_INT32 && (type == DTYPE_INT64 || type == DTYPE_FLOAT64)) || \ - (np_dtype == DTYPE_FLOAT64 && (type == DTYPE_INT32 || type == DTYPE_INT64)) || \ - (type == DTYPE_INT64 && (np_dtype == DTYPE_FLOAT32 || np_dtype == DTYPE_FLOAT64)); + bool should_iter + = (np_dtype == DTYPE_INT64 + && (type == DTYPE_INT32 || type == DTYPE_FLOAT64)) + || (np_dtype == DTYPE_INT32 + && (type == DTYPE_INT64 || type == DTYPE_FLOAT64)) + || (np_dtype == DTYPE_FLOAT64 + && (type == DTYPE_INT32 || type == DTYPE_INT64)) + || (type == DTYPE_INT64 + && (np_dtype == DTYPE_FLOAT32 || np_dtype == DTYPE_FLOAT64)); if (should_iter) { // Skip straight to numeric fill - fill_numeric_iter(array, tbl, col, name, np_dtype, type, cidx, is_update); + fill_numeric_iter( + array, tbl, col, name, np_dtype, type, cidx, is_update); return; } @@ -186,16 +216,19 @@ namespace numpy { // Iterate if copy is not supported for the numpy array if (copy_status == t_fill_status::FILL_FAIL) { - fill_column_iter(array, tbl, col, name, np_dtype, type, cidx, is_update); + fill_column_iter( + array, tbl, col, name, np_dtype, type, cidx, is_update); } - + // Fill validity map using null mask fill_validity_map(col, mask_ptr, mask_size, is_update); } template void - NumpyLoader::fill_object_iter(t_data_table& tbl, std::shared_ptr col, const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { + NumpyLoader::fill_object_iter(t_data_table& tbl, + std::shared_ptr col, const std::string& name, + t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { t_uindex nrows = col->size(); for (auto i = 0; i < nrows; ++i) { @@ -214,10 +247,13 @@ namespace numpy { } } - // Add explicit instantiations for int32, int64, and float64 as they have promotion logic + // Add explicit instantiations for int32, int64, and float64 as they have + // promotion logic template <> void - NumpyLoader::fill_object_iter(t_data_table& tbl, std::shared_ptr col, const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { + NumpyLoader::fill_object_iter(t_data_table& tbl, + std::shared_ptr col, const std::string& name, + t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { t_uindex nrows = col->size(); for (auto i = 0; i < nrows; ++i) { @@ -234,13 +270,15 @@ namespace numpy { double fval = item.cast(); if (!is_update && (fval > 2147483647 || fval < -2147483648)) { - binding::WARN("Promoting column `%s` to float from int32", name); + binding::WARN( + "Promoting column `%s` to float from int32", name); tbl.promote_column(name, DTYPE_FLOAT64, i, true); col = tbl.get_column(name); type = DTYPE_FLOAT64; col->set_nth(i, fval); } else if (!is_update && isnan(fval)) { - binding::WARN("Promoting column `%s` to string from int32", name); + binding::WARN( + "Promoting column `%s` to string from int32", name); tbl.promote_column(name, DTYPE_STR, i, false); col = tbl.get_column(name); fill_object_iter( @@ -254,7 +292,9 @@ namespace numpy { template <> void - NumpyLoader::fill_object_iter(t_data_table& tbl, std::shared_ptr col, const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { + NumpyLoader::fill_object_iter(t_data_table& tbl, + std::shared_ptr col, const std::string& name, + t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { t_uindex nrows = col->size(); for (auto i = 0; i < nrows; ++i) { @@ -271,7 +311,8 @@ namespace numpy { double fval = item.cast(); if (isnan(fval)) { - binding::WARN("Promoting column `%s` to string from int64", name); + binding::WARN( + "Promoting column `%s` to string from int64", name); tbl.promote_column(name, DTYPE_STR, i, false); col = tbl.get_column(name); fill_object_iter( @@ -285,7 +326,9 @@ namespace numpy { template <> void - NumpyLoader::fill_object_iter(t_data_table& tbl, std::shared_ptr col, const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { + NumpyLoader::fill_object_iter(t_data_table& tbl, + std::shared_ptr col, const std::string& name, + t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { t_uindex nrows = col->size(); for (auto i = 0; i < nrows; ++i) { @@ -303,7 +346,8 @@ namespace numpy { bool is_float = py::isinstance(item); bool is_numpy_nan = is_float && npy_isnan(item.cast()); if (!is_float || is_numpy_nan) { - binding::WARN("Promoting column `%s` to string from float64", name); + binding::WARN( + "Promoting column `%s` to string from float64", name); tbl.promote_column(name, DTYPE_STR, i, false); col = tbl.get_column(name); fill_object_iter( @@ -316,37 +360,49 @@ namespace numpy { // Must be below `fill_object_iter` explicit instantiations. void - NumpyLoader::fill_column_iter(const py::array& array, t_data_table& tbl, std::shared_ptr col, const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { + NumpyLoader::fill_column_iter(const py::array& array, t_data_table& tbl, + std::shared_ptr col, const std::string& name, + t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); - // Numpy arrays are guaranteed to be continous and valid by the time they enter this block, - // but if they're of dtype object, then we need to pass it through `m_accessor.marshal`. + // Numpy arrays are guaranteed to be continous and valid by the time + // they enter this block, but if they're of dtype object, then we need + // to pass it through `m_accessor.marshal`. switch (type) { case DTYPE_TIME: { - // covers dtype `datetime64[us/ns/ms/s]`, date strings, and integer timestamps in ms or s since epoch + // covers dtype `datetime64[us/ns/ms/s]`, date strings, and + // integer timestamps in ms or s since epoch if (np_dtype != DTYPE_TIME) { - fill_object_iter(tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); } else { - fill_datetime_iter(array, tbl, col, name, np_dtype, type, cidx, is_update); + fill_datetime_iter( + array, tbl, col, name, np_dtype, type, cidx, is_update); } } break; case DTYPE_DATE: { - // `datetime.date` objects or `datetime64[D/W/M/Y]`, always fill by using `marshal`. + // `datetime.date` objects or `datetime64[D/W/M/Y]`, always fill + // by using `marshal`. fill_date_iter(col, name, np_dtype, type, cidx, is_update); } break; case DTYPE_BOOL: { if (np_dtype == DTYPE_OBJECT) { - fill_object_iter(tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); } else { - fill_bool_iter(array, tbl, col, name, np_dtype, type, cidx, is_update); + fill_bool_iter( + array, tbl, col, name, np_dtype, type, cidx, is_update); } } break; case DTYPE_STR: { // dtype `U` - fill_object_iter(tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); } break; default: { - // dtype `i/u/f` - fill_numeric_iter checks again for `dtype=object` - fill_numeric_iter(array, tbl, col, name, np_dtype, type, cidx, is_update); + // dtype `i/u/f` - fill_numeric_iter checks again for + // `dtype=object` + fill_numeric_iter( + array, tbl, col, name, np_dtype, type, cidx, is_update); break; } } @@ -354,22 +410,26 @@ namespace numpy { // `array.dtype=datetime64[ns/us/ms/s]` void - NumpyLoader::fill_datetime_iter(const py::array& array, t_data_table& tbl, std::shared_ptr col, - const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { + NumpyLoader::fill_datetime_iter(const py::array& array, t_data_table& tbl, + std::shared_ptr col, const std::string& name, + t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); t_uindex nrows = col->size(); // read the array as a double array because of `numpy.nat` - double* ptr = (double*) array.data(); + double* ptr = (double*)array.data(); for (auto i = 0; i < nrows; ++i) { - std::int64_t item = ptr[i]; // Perspective stores datetimes using int64 + std::int64_t item + = ptr[i]; // Perspective stores datetimes using int64 col->set_nth(i, item); } } void - NumpyLoader::fill_date_iter(std::shared_ptr col, const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { + NumpyLoader::fill_date_iter(std::shared_ptr col, + const std::string& name, t_dtype np_dtype, t_dtype type, + std::uint32_t cidx, bool is_update) { PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); t_uindex nrows = col->size(); @@ -385,24 +445,29 @@ namespace numpy { continue; } - auto date_components = item.cast>(); + auto date_components + = item.cast>(); // date_components["month"] should be [0-11] - t_date dt = t_date(date_components["year"], date_components["month"], date_components["day"]); + t_date dt = t_date(date_components["year"], + date_components["month"], date_components["day"]); col->set_nth(i, dt); } } void - NumpyLoader::fill_bool_iter(const py::array& array, t_data_table& tbl, std::shared_ptr col, const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { + NumpyLoader::fill_bool_iter(const py::array& array, t_data_table& tbl, + std::shared_ptr col, const std::string& name, + t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); t_uindex nrows = col->size(); // handle Nan/None in boolean array with dtype=object if (np_dtype == DTYPE_OBJECT) { // handle object arrays - fill_object_iter(tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); } else { - bool* ptr = (bool*) array.data(); + bool* ptr = (bool*)array.data(); for (auto i = 0; i < nrows; ++i) { bool item = ptr[i]; @@ -411,68 +476,88 @@ namespace numpy { } } - void - NumpyLoader::fill_numeric_iter(const py::array& array, t_data_table& tbl, std::shared_ptr col, const std::string& name, t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { + void + NumpyLoader::fill_numeric_iter(const py::array& array, t_data_table& tbl, + std::shared_ptr col, const std::string& name, + t_dtype np_dtype, t_dtype type, std::uint32_t cidx, bool is_update) { PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); t_uindex nrows = col->size(); const void* ptr = array.data(); - // We fill by object when `np_dtype`=object, or if there are type mismatches between `np_dtype` and `type`. - bool types_mismatched = (np_dtype == DTYPE_INT64 && (type == DTYPE_INT32 || type == DTYPE_FLOAT64)) || \ - (np_dtype == DTYPE_INT32 && (type == DTYPE_INT64 || type == DTYPE_FLOAT64)) || \ - (np_dtype == DTYPE_FLOAT64 && (type == DTYPE_INT32 || type == DTYPE_INT64)) || \ - (type == DTYPE_INT64 && (np_dtype == DTYPE_FLOAT32 || np_dtype == DTYPE_FLOAT64)); + // We fill by object when `np_dtype`=object, or if there are type + // mismatches between `np_dtype` and `type`. + bool types_mismatched + = (np_dtype == DTYPE_INT64 + && (type == DTYPE_INT32 || type == DTYPE_FLOAT64)) + || (np_dtype == DTYPE_INT32 + && (type == DTYPE_INT64 || type == DTYPE_FLOAT64)) + || (np_dtype == DTYPE_FLOAT64 + && (type == DTYPE_INT32 || type == DTYPE_INT64)) + || (type == DTYPE_INT64 + && (np_dtype == DTYPE_FLOAT32 || np_dtype == DTYPE_FLOAT64)); if (types_mismatched || np_dtype == DTYPE_OBJECT) { switch (type) { case DTYPE_UINT8: { - fill_object_iter(tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); return; } break; case DTYPE_UINT16: { - fill_object_iter (tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); return; } break; case DTYPE_UINT32: { - fill_object_iter(tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); return; } break; case DTYPE_UINT64: { - fill_object_iter(tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); return; } break; case DTYPE_INT8: { - fill_object_iter(tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); return; } break; case DTYPE_INT16: { - fill_object_iter(tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); return; } break; case DTYPE_INT32: { - fill_object_iter(tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); return; } break; case DTYPE_INT64: { - fill_object_iter(tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); return; } break; case DTYPE_FLOAT32: { - fill_object_iter(tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); return; } break; case DTYPE_FLOAT64: { - fill_object_iter(tbl, col, name, np_dtype, type, cidx, is_update); + fill_object_iter( + tbl, col, name, np_dtype, type, cidx, is_update); return; } break; default: - PSP_COMPLAIN_AND_ABORT("Unable to fill non-numeric column `" + name + "` in `fill_numeric_iter`.") + PSP_COMPLAIN_AND_ABORT("Unable to fill non-numeric column `" + + name + "` in `fill_numeric_iter`.") } } - // Iterate through the C++ array and try to cast. Array is guaranteed to be of the correct dtype and consistent in its values. + // Iterate through the C++ array and try to cast. Array is guaranteed to + // be of the correct dtype and consistent in its values. for (auto i = 0; i < nrows; ++i) { - if (isnan(((double*) ptr)[i]) || npy_isnan(((double*) ptr)[i])) { + if (isnan(((double*)ptr)[i]) || npy_isnan(((double*)ptr)[i])) { if (is_update) { col->unset(i); } else { @@ -502,13 +587,13 @@ namespace numpy { } break; case DTYPE_INT32: { // No need for promotion logic if array is consistent - col->set_nth(i, ((std::int32_t*) ptr)[i]); + col->set_nth(i, ((std::int32_t*)ptr)[i]); } break; case DTYPE_INT64: { - col->set_nth(i, ((std::int64_t*) ptr)[i]); + col->set_nth(i, ((std::int64_t*)ptr)[i]); } break; case DTYPE_FLOAT32: { - col->set_nth(i, ((float*) ptr)[i]); + col->set_nth(i, ((float*)ptr)[i]); } break; case DTYPE_FLOAT64: { col->set_nth(i, ((double*)ptr)[i]); @@ -523,8 +608,10 @@ namespace numpy { * * Copy numpy arrays into columns */ - t_fill_status - NumpyLoader::try_copy_array(const py::array& src, std::shared_ptr dest, t_dtype np_dtype, t_dtype type, const std::uint64_t offset) { + t_fill_status + NumpyLoader::try_copy_array(const py::array& src, + std::shared_ptr dest, t_dtype np_dtype, t_dtype type, + const std::uint64_t offset) { PSP_VERBOSE_ASSERT(m_init, "touching uninited object"); std::int64_t length = src.size(); @@ -569,9 +656,10 @@ namespace numpy { } void - NumpyLoader::fill_validity_map( - std::shared_ptr col, std::uint64_t* mask_ptr, std::size_t mask_size, bool is_update) { - // Validity map needs to be filled each time - None/np.nan/float('nan') might not have been parsed correctly + NumpyLoader::fill_validity_map(std::shared_ptr col, + std::uint64_t* mask_ptr, std::size_t mask_size, bool is_update) { + // Validity map needs to be filled each time - None/np.nan/float('nan') + // might not have been parsed correctly col->valid_raw_fill(); if (mask_size > 0) { @@ -587,7 +675,9 @@ namespace numpy { } template - void copy_array_helper(const void* src, std::shared_ptr dest, const std::uint64_t offset) { + void + copy_array_helper(const void* src, std::shared_ptr dest, + const std::uint64_t offset) { std::memcpy(dest->get_nth(offset), src, dest->size() * sizeof(T)); } @@ -598,7 +688,8 @@ namespace numpy { std::vector NumpyLoader::make_names() { auto data = m_accessor.attr("data")(); - auto py_names = m_accessor.attr("names")().cast>(); + auto py_names + = m_accessor.attr("names")().cast>(); // Match names to dataset - only keep names that are present in dataset. // The `m_names` variable is used internally to access the numpy arrays @@ -618,7 +709,7 @@ namespace numpy { std::vector NumpyLoader::make_types() { std::vector rval(m_names.size()); - + auto data = m_accessor.attr("data")(); for (auto i = 0; i < m_names.size(); ++i) { @@ -628,10 +719,12 @@ namespace numpy { py::array array = py::array::ensure(data[py::str(name)]); if (!array) { - PSP_COMPLAIN_AND_ABORT("Perspective does not support the mixing of ndarrays and lists."); + PSP_COMPLAIN_AND_ABORT("Perspective does not support the " + "mixing of ndarrays and lists."); } - // can't use isinstance on datetime/timedelta array, so check the dtype + // can't use isinstance on datetime/timedelta array, so check the + // dtype char dtype_code = array.dtype().kind(); if (dtype_code == 'M') { @@ -644,7 +737,8 @@ namespace numpy { continue; } - // isinstance checks equality of underlying dtype, not just pointer equality + // isinstance checks equality of underlying dtype, not just pointer + // equality if (py::isinstance>(array)) { rval[i] = DTYPE_UINT8; } else if (py::isinstance>(array)) { @@ -668,14 +762,15 @@ namespace numpy { } else if (py::isinstance>(array)) { rval[i] = DTYPE_BOOL; } else { - // DTYPE_OBJECT defers to the inferred type: this allows parsing of datetime strings, boolean strings, etc. + // DTYPE_OBJECT defers to the inferred type: this allows parsing + // of datetime strings, boolean strings, etc. rval[i] = DTYPE_OBJECT; } } return rval; } - + } // namespace numpy } // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/src/python.cpp b/python/perspective/perspective/src/python.cpp index 8e875a0cc3..944e080932 100644 --- a/python/perspective/perspective/src/python.cpp +++ b/python/perspective/perspective/src/python.cpp @@ -13,9 +13,7 @@ #include namespace perspective { -namespace binding { - -} //namespace binding -} //namespace perspective +namespace binding {} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/src/serialization.cpp b/python/perspective/perspective/src/serialization.cpp index 92fb36481b..286a33bad9 100644 --- a/python/perspective/perspective/src/serialization.cpp +++ b/python/perspective/perspective/src/serialization.cpp @@ -17,107 +17,127 @@ namespace perspective { namespace binding { -/****************************************************************************** - * - * Data serialization - */ -template -std::shared_ptr> -get_data_slice(std::shared_ptr> view, std::uint32_t start_row, - std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col) { - PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); - auto data_slice = view->get_data(start_row, end_row, start_col, end_col); - return data_slice; -} - -std::shared_ptr> -get_data_slice_unit(std::shared_ptr> view, std::uint32_t start_row, - std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col) { - return get_data_slice(view, start_row, end_row, start_col, end_col); -} - -std::shared_ptr> -get_data_slice_ctx0(std::shared_ptr> view, std::uint32_t start_row, - std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col) { - return get_data_slice(view, start_row, end_row, start_col, end_col); -} - -std::shared_ptr> -get_data_slice_ctx1(std::shared_ptr> view, std::uint32_t start_row, - std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col) { - return get_data_slice(view, start_row, end_row, start_col, end_col); -} - -std::shared_ptr> -get_data_slice_ctx2(std::shared_ptr> view, std::uint32_t start_row, - std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col) { - return get_data_slice(view, start_row, end_row, start_col, end_col); -} - -template -t_val -get_from_data_slice( - std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx) { - auto d = data_slice->get(ridx, cidx); - return scalar_to_py(d); -} - -t_val -get_from_data_slice_unit( - std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx) { - return get_from_data_slice(data_slice, ridx, cidx); -} - -t_val -get_from_data_slice_ctx0( - std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx) { - return get_from_data_slice(data_slice, ridx, cidx); -} - -t_val -get_from_data_slice_ctx1( - std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx) { - return get_from_data_slice(data_slice, ridx, cidx); -} - -t_val -get_from_data_slice_ctx2( - std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx) { - return get_from_data_slice(data_slice, ridx, cidx); -} - -template -std::vector -get_pkeys_from_data_slice(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx) { - std::vector pkeys = data_slice->get_pkeys(ridx, cidx); - std::vector rval(pkeys.size()); - - for (auto i = 0; i < pkeys.size(); ++i) { - rval[i] = scalar_to_py(pkeys[i]); + /****************************************************************************** + * + * Data serialization + */ + template + std::shared_ptr> + get_data_slice(std::shared_ptr> view, std::uint32_t start_row, + std::uint32_t end_row, std::uint32_t start_col, std::uint32_t end_col) { + PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); + auto data_slice + = view->get_data(start_row, end_row, start_col, end_col); + return data_slice; + } + + std::shared_ptr> + get_data_slice_unit(std::shared_ptr> view, + std::uint32_t start_row, std::uint32_t end_row, std::uint32_t start_col, + std::uint32_t end_col) { + return get_data_slice( + view, start_row, end_row, start_col, end_col); + } + + std::shared_ptr> + get_data_slice_ctx0(std::shared_ptr> view, + std::uint32_t start_row, std::uint32_t end_row, std::uint32_t start_col, + std::uint32_t end_col) { + return get_data_slice( + view, start_row, end_row, start_col, end_col); + } + + std::shared_ptr> + get_data_slice_ctx1(std::shared_ptr> view, + std::uint32_t start_row, std::uint32_t end_row, std::uint32_t start_col, + std::uint32_t end_col) { + return get_data_slice( + view, start_row, end_row, start_col, end_col); + } + + std::shared_ptr> + get_data_slice_ctx2(std::shared_ptr> view, + std::uint32_t start_row, std::uint32_t end_row, std::uint32_t start_col, + std::uint32_t end_col) { + return get_data_slice( + view, start_row, end_row, start_col, end_col); + } + + template + t_val + get_from_data_slice(std::shared_ptr> data_slice, + t_uindex ridx, t_uindex cidx) { + auto d = data_slice->get(ridx, cidx); + return scalar_to_py(d); + } + + t_val + get_from_data_slice_unit( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx) { + return get_from_data_slice(data_slice, ridx, cidx); + } + + t_val + get_from_data_slice_ctx0(std::shared_ptr> data_slice, + t_uindex ridx, t_uindex cidx) { + return get_from_data_slice(data_slice, ridx, cidx); + } + + t_val + get_from_data_slice_ctx1(std::shared_ptr> data_slice, + t_uindex ridx, t_uindex cidx) { + return get_from_data_slice(data_slice, ridx, cidx); + } + + t_val + get_from_data_slice_ctx2(std::shared_ptr> data_slice, + t_uindex ridx, t_uindex cidx) { + return get_from_data_slice(data_slice, ridx, cidx); } - return rval; -} + template + std::vector + get_pkeys_from_data_slice(std::shared_ptr> data_slice, + t_uindex ridx, t_uindex cidx) { + std::vector pkeys = data_slice->get_pkeys(ridx, cidx); + std::vector rval(pkeys.size()); -std::vector -get_pkeys_from_data_slice_unit(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx) { - return get_pkeys_from_data_slice(data_slice, ridx, cidx); -} + for (auto i = 0; i < pkeys.size(); ++i) { + rval[i] = scalar_to_py(pkeys[i]); + } -std::vector -get_pkeys_from_data_slice_ctx0(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx) { - return get_pkeys_from_data_slice(data_slice, ridx, cidx); -} + return rval; + } + + std::vector + get_pkeys_from_data_slice_unit( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx) { + return get_pkeys_from_data_slice(data_slice, ridx, cidx); + } -std::vector -get_pkeys_from_data_slice_ctx1(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx) { - return get_pkeys_from_data_slice(data_slice, ridx, cidx);; -} + std::vector + get_pkeys_from_data_slice_ctx0( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx) { + return get_pkeys_from_data_slice(data_slice, ridx, cidx); + } -std::vector -get_pkeys_from_data_slice_ctx2(std::shared_ptr> data_slice, t_uindex ridx, t_uindex cidx) { - return get_pkeys_from_data_slice(data_slice, ridx, cidx); -} + std::vector + get_pkeys_from_data_slice_ctx1( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx) { + return get_pkeys_from_data_slice(data_slice, ridx, cidx); + ; + } + + std::vector + get_pkeys_from_data_slice_ctx2( + std::shared_ptr> data_slice, t_uindex ridx, + t_uindex cidx) { + return get_pkeys_from_data_slice(data_slice, ridx, cidx); + } } // end namespace binding } // end namespace perspective diff --git a/python/perspective/perspective/src/table.cpp b/python/perspective/perspective/src/table.cpp index 2e569036fa..35ac8db4a0 100644 --- a/python/perspective/perspective/src/table.cpp +++ b/python/perspective/perspective/src/table.cpp @@ -22,266 +22,302 @@ namespace perspective { namespace binding { using namespace perspective::apachearrow; -/****************************************************************************** - * - * Table API - */ + /****************************************************************************** + * + * Table API + */ -std::shared_ptr
make_table_py( - t_val table, - t_data_accessor accessor, - std::uint32_t limit, - py::str index, - t_op op, - bool is_update, - bool is_arrow, - bool is_csv, - t_uindex port_id -) { - bool table_initialized = !table.is_none(); - std::shared_ptr pool; - std::shared_ptr
tbl; - std::shared_ptr gnode; - std::uint32_t offset; - void* ptr = nullptr; + std::shared_ptr
+ make_table_py(t_val table, t_data_accessor accessor, std::uint32_t limit, + py::str index, t_op op, bool is_update, bool is_arrow, bool is_csv, + t_uindex port_id) { + bool table_initialized = !table.is_none(); + std::shared_ptr pool; + std::shared_ptr
tbl; + std::shared_ptr gnode; + std::uint32_t offset; + void* ptr = nullptr; - // If the Table has already been created, use it - if (table_initialized) { - tbl = table.cast>(); - pool = tbl->get_pool(); - gnode = tbl->get_gnode(); - offset = tbl->get_offset(); - is_update = (is_update || gnode->mapping_size() > 0); - } else { - pool = std::make_shared(); - } + // If the Table has already been created, use it + if (table_initialized) { + tbl = table.cast>(); + pool = tbl->get_pool(); + gnode = tbl->get_gnode(); + offset = tbl->get_offset(); + is_update = (is_update || gnode->mapping_size() > 0); + } else { + pool = std::make_shared(); + } - std::vector column_names; - std::vector data_types; - ArrowLoader arrow_loader; - numpy::NumpyLoader numpy_loader(accessor); + std::vector column_names; + std::vector data_types; + ArrowLoader arrow_loader; + numpy::NumpyLoader numpy_loader(accessor); - // don't call `is_numpy` on an arrow binary - bool is_numpy = !is_arrow && !is_csv && accessor.attr("_is_numpy").cast(); + // don't call `is_numpy` on an arrow binary + bool is_numpy + = !is_arrow && !is_csv && accessor.attr("_is_numpy").cast(); - // Determine metadata - bool is_delete = op == OP_DELETE; - if (is_arrow && !is_delete) { - std::string csv_string; - std::int32_t binary_size; + // Determine metadata + bool is_delete = op == OP_DELETE; + if (is_arrow && !is_delete) { + std::string csv_string; + std::int32_t binary_size; - if (is_csv) { - // Load a string in CSV format - csv_string = accessor.cast(); - } else { - // Load an arrow binary - py::bytes bytes = accessor.cast(); - binary_size = bytes.attr("__len__")().cast(); - ptr = malloc(binary_size); - std::memcpy(ptr, bytes.cast().c_str(), binary_size); - } + if (is_csv) { + // Load a string in CSV format + csv_string = accessor.cast(); + } else { + // Load an arrow binary + py::bytes bytes = accessor.cast(); + binary_size = bytes.attr("__len__")().cast(); + ptr = malloc(binary_size); + std::memcpy( + ptr, bytes.cast().c_str(), binary_size); + } - { - PerspectiveScopedGILRelease acquire(pool->get_event_loop_thread_id()); + { + PerspectiveScopedGILRelease acquire( + pool->get_event_loop_thread_id()); - // With the GIL released, load the arrow - if (is_csv) { - auto map = std::unordered_map>(); + // With the GIL released, load the arrow + if (is_csv) { + auto map = std::unordered_map>(); - if (is_update) { - auto gnode_output_schema = gnode->get_output_schema(); - auto schema = gnode_output_schema.drop({"psp_okey"}); - auto column_names = schema.columns(); - auto data_types = schema.types(); - - for (auto idx = 0; idx < column_names.size(); ++idx) { - const std::string& name = column_names[idx]; - const t_dtype& type = data_types[idx]; - switch (type) { - case DTYPE_FLOAT32: - map[name] = std::make_shared(); - break; - case DTYPE_FLOAT64: - map[name] = std::make_shared(); - break; - case DTYPE_STR: - map[name] = std::make_shared(); - break; - case DTYPE_BOOL: - map[name] = std::make_shared(); - break; - case DTYPE_UINT32: - map[name] = std::make_shared(); - break; - case DTYPE_UINT64: - map[name] = std::make_shared(); - break; - case DTYPE_INT32: - map[name] = std::make_shared(); - break; - case DTYPE_INT64: - map[name] = std::make_shared(); - break; - case DTYPE_TIME: - map[name] = std::make_shared(); - break; - case DTYPE_DATE: - map[name] = std::make_shared(); - break; - default: - std::stringstream ss; - ss << "Error loading arrow type " << dtype_to_str(type) << " for column " << name << std::endl; - PSP_COMPLAIN_AND_ABORT(ss.str()) - break; + if (is_update) { + auto gnode_output_schema = gnode->get_output_schema(); + auto schema = gnode_output_schema.drop({"psp_okey"}); + auto column_names = schema.columns(); + auto data_types = schema.types(); + + for (auto idx = 0; idx < column_names.size(); ++idx) { + const std::string& name = column_names[idx]; + const t_dtype& type = data_types[idx]; + switch (type) { + case DTYPE_FLOAT32: + map[name] + = std::make_shared(); + break; + case DTYPE_FLOAT64: + map[name] + = std::make_shared(); + break; + case DTYPE_STR: + map[name] + = std::make_shared(); + break; + case DTYPE_BOOL: + map[name] = std::make_shared< + arrow::BooleanType>(); + break; + case DTYPE_UINT32: + map[name] + = std::make_shared(); + break; + case DTYPE_UINT64: + map[name] + = std::make_shared(); + break; + case DTYPE_INT32: + map[name] + = std::make_shared(); + break; + case DTYPE_INT64: + map[name] + = std::make_shared(); + break; + case DTYPE_TIME: + map[name] = std::make_shared< + arrow::TimestampType>(); + break; + case DTYPE_DATE: + map[name] + = std::make_shared(); + break; + default: + std::stringstream ss; + ss << "Error loading arrow type " + << dtype_to_str(type) << " for column " + << name << std::endl; + PSP_COMPLAIN_AND_ABORT(ss.str()) + break; + } } } - } - arrow_loader.init_csv(csv_string, is_update, map); - } else { - arrow_loader.initialize((uintptr_t)ptr, binary_size); - } + arrow_loader.init_csv(csv_string, is_update, map); + } else { + arrow_loader.initialize((uintptr_t)ptr, binary_size); + } - // Always use the `Table` column names and data types on update. - if (table_initialized && is_update) { - auto gnode_output_schema = gnode->get_output_schema(); - auto schema = gnode_output_schema.drop({"psp_okey"}); - column_names = schema.columns(); - data_types = schema.types(); + // Always use the `Table` column names and data types on update. + if (table_initialized && is_update) { + auto gnode_output_schema = gnode->get_output_schema(); + auto schema = gnode_output_schema.drop({"psp_okey"}); + column_names = schema.columns(); + data_types = schema.types(); - auto data_table = gnode->get_table(); - if (data_table->size() == 0) { - /** - * If updating a table created from schema, a 32-bit int/float - * needs to be promoted to a 64-bit int/float if specified in - * the Arrow schema. - */ - std::vector arrow_dtypes = arrow_loader.types(); - for (auto idx = 0; idx < column_names.size(); ++idx) { - const std::string& name = column_names[idx]; - bool can_retype = name != "psp_okey" && name != "psp_pkey" && name != "psp_op"; - bool is_32_bit = data_types[idx] == DTYPE_INT32 || data_types[idx] == DTYPE_FLOAT32; - if (can_retype && is_32_bit) { - t_dtype arrow_dtype = arrow_dtypes[idx]; - switch (arrow_dtype) { - case DTYPE_INT64: - case DTYPE_FLOAT64: { - std::cout << "Promoting column `" - << column_names[idx] - << "` to maintain consistency with Arrow type." - << std::endl; - gnode->promote_column(name, arrow_dtype); - } break; - default: { - continue; + auto data_table = gnode->get_table(); + if (data_table->size() == 0) { + /** + * If updating a table created from schema, a 32-bit + * int/float needs to be promoted to a 64-bit int/float + * if specified in the Arrow schema. + */ + std::vector arrow_dtypes + = arrow_loader.types(); + for (auto idx = 0; idx < column_names.size(); ++idx) { + const std::string& name = column_names[idx]; + bool can_retype = name != "psp_okey" + && name != "psp_pkey" && name != "psp_op"; + bool is_32_bit = data_types[idx] == DTYPE_INT32 + || data_types[idx] == DTYPE_FLOAT32; + if (can_retype && is_32_bit) { + t_dtype arrow_dtype = arrow_dtypes[idx]; + switch (arrow_dtype) { + case DTYPE_INT64: + case DTYPE_FLOAT64: { + std::cout + << "Promoting column `" + << column_names[idx] + << "` to maintain consistency with " + "Arrow type." + << std::endl; + gnode->promote_column( + name, arrow_dtype); + } break; + default: { + continue; + } } } } } + // Make sure promoted types are used to construct data table + auto new_schema + = gnode->get_output_schema().drop({"psp_okey"}); + data_types = new_schema.types(); + } else { + column_names = arrow_loader.names(); + data_types = arrow_loader.types(); } - // Make sure promoted types are used to construct data table - auto new_schema = gnode->get_output_schema().drop({"psp_okey"}); - data_types = new_schema.types(); - } else { - column_names = arrow_loader.names(); - data_types = arrow_loader.types(); } - } - } else if (is_update || is_delete) { - /** - * Use the names and types of the python accessor when updating/deleting. - * - * This prevents the Table from looking up new columns present in an update. - * - * Example: updating a Table with a DataFrame attempts to write the "index" column, but if the table was - * not created from a DataFrame, the "index" column would not exist. - */ - if (is_numpy) { - // `numpy_loader`s `m_names` and `m_types` variable contains only - // the column names and data types present in the update dataset, - // not the names/types of the entire `Table`. + } else if (is_update || is_delete) { + /** + * Use the names and types of the python accessor when + * updating/deleting. + * + * This prevents the Table from looking up new columns present in an + * update. + * + * Example: updating a Table with a DataFrame attempts to write the + * "index" column, but if the table was not created from a + * DataFrame, the "index" column would not exist. + */ + if (is_numpy) { + // `numpy_loader`s `m_names` and `m_types` variable contains + // only the column names and data types present in the update + // dataset, not the names/types of the entire `Table`. + numpy_loader.init(); + } + + // `column_names` and `data_types` contain every single column in + // the dataset, as well as `__INDEX__` if it exists. + column_names + = accessor.attr("names")().cast>(); + data_types = accessor.attr("types")().cast>(); + } else if (is_numpy) { + /** + * Numpy loading depends on both the `dtype` of the individual + * arrays as well as the inferred type from Perspective. Using + * `get_data_types` allows us to know the type of an array with + * `dtype=object`. + */ numpy_loader.init(); - } - // `column_names` and `data_types` contain every single column in the - // dataset, as well as `__INDEX__` if it exists. - column_names = accessor.attr("names")().cast>(); - data_types = accessor.attr("types")().cast>(); - } else if (is_numpy) { - /** - * Numpy loading depends on both the `dtype` of the individual arrays as well as the inferred type from - * Perspective. Using `get_data_types` allows us to know the type of an array with `dtype=object`. - */ - numpy_loader.init(); + // This will contain every single column in the dataset, as the + // first-time data load path does not mutate the `names` property of + // `accessor`. + column_names = numpy_loader.names(); - // This will contain every single column in the dataset, as the - // first-time data load path does not mutate the `names` property of - // `accessor`. - column_names = numpy_loader.names(); + // Infer data type for each column, and then use a composite of + // numpy dtype, inferred `t_dtype`, and stringified numpy dtype to + // get the final, canonical data type mapping. + std::vector inferred_types + = get_data_types(accessor.attr("data")(), 1, column_names, + accessor.attr("date_validator")().cast()); + data_types = numpy_loader.reconcile_dtypes(inferred_types); + } else { + // Infer names and types + t_val data = accessor.attr("data")(); + std::int32_t format + = accessor.attr("format")().cast(); + column_names = get_column_names(data, format); + data_types = get_data_types(data, format, column_names, + accessor.attr("date_validator")().cast()); + } - // Infer data type for each column, and then use a composite of numpy - // dtype, inferred `t_dtype`, and stringified numpy dtype to get the - // final, canonical data type mapping. - std::vector inferred_types = get_data_types(accessor.attr("data")(), 1, column_names, accessor.attr("date_validator")().cast()); - data_types = numpy_loader.reconcile_dtypes(inferred_types); - } else { - // Infer names and types - t_val data = accessor.attr("data")(); - std::int32_t format = accessor.attr("format")().cast(); - column_names = get_column_names(data, format); - data_types = get_data_types(data, format, column_names, accessor.attr("date_validator")().cast()); - } - - if (!table_initialized) { - tbl = std::make_shared
(pool, column_names, data_types, limit, index); - offset = 0; - } + if (!table_initialized) { + tbl = std::make_shared
( + pool, column_names, data_types, limit, index); + offset = 0; + } - // Create input schema - an input schema contains all columns to be displayed AND index + operation columns - t_schema input_schema(column_names, data_types); + // Create input schema - an input schema contains all columns to be + // displayed AND index + operation columns + t_schema input_schema(column_names, data_types); - // strip implicit index, if present - auto implicit_index_it = std::find(column_names.begin(), column_names.end(), "__INDEX__"); - if (implicit_index_it != column_names.end()) { - auto idx = std::distance(column_names.begin(), implicit_index_it); - // position of the column is at the same index in both vectors - column_names.erase(column_names.begin() + idx); - data_types.erase(data_types.begin() + idx); - } + // strip implicit index, if present + auto implicit_index_it + = std::find(column_names.begin(), column_names.end(), "__INDEX__"); + if (implicit_index_it != column_names.end()) { + auto idx = std::distance(column_names.begin(), implicit_index_it); + // position of the column is at the same index in both vectors + column_names.erase(column_names.begin() + idx); + data_types.erase(data_types.begin() + idx); + } - // Create output schema - contains only columns to be displayed to the user - t_schema output_schema(column_names, data_types); // names + types might have been mutated at this point after implicit index removal - t_data_table data_table(output_schema); - data_table.init(); - std::uint32_t row_count; + // Create output schema - contains only columns to be displayed to the + // user + t_schema output_schema(column_names, + data_types); // names + types might have been mutated at this point + // after implicit index removal + t_data_table data_table(output_schema); + data_table.init(); + std::uint32_t row_count; - if (is_arrow) { - PerspectiveScopedGILRelease acquire(pool->get_event_loop_thread_id()); - row_count = arrow_loader.row_count(); - data_table.extend(arrow_loader.row_count()); - arrow_loader.fill_table(data_table, input_schema, index, offset, limit, is_update); - } else if (is_numpy) { - row_count = numpy_loader.row_count(); - data_table.extend(row_count); - numpy_loader.fill_table(data_table, input_schema, index, offset, limit, is_update); - } else { - row_count = accessor.attr("row_count")().cast(); - data_table.extend(row_count); - _fill_data(data_table, accessor, input_schema, index, offset, limit, is_update); - } + if (is_arrow) { + PerspectiveScopedGILRelease acquire( + pool->get_event_loop_thread_id()); + row_count = arrow_loader.row_count(); + data_table.extend(arrow_loader.row_count()); + arrow_loader.fill_table( + data_table, input_schema, index, offset, limit, is_update); + } else if (is_numpy) { + row_count = numpy_loader.row_count(); + data_table.extend(row_count); + numpy_loader.fill_table( + data_table, input_schema, index, offset, limit, is_update); + } else { + row_count = accessor.attr("row_count")().cast(); + data_table.extend(row_count); + _fill_data(data_table, accessor, input_schema, index, offset, limit, + is_update); + } - if (is_arrow && !is_csv) { - free(ptr); - } + if (is_arrow && !is_csv) { + free(ptr); + } - // calculate offset, limit, and set the gnode - tbl->init(data_table, row_count, op, port_id); + // calculate offset, limit, and set the gnode + tbl->init(data_table, row_count, op, port_id); - //pool->_process(); - return tbl; -} + // pool->_process(); + return tbl; + } -} //namespace binding -} //namespace perspective +} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/src/utils.cpp b/python/perspective/perspective/src/utils.cpp index 9caaab51d4..b184a78cb8 100644 --- a/python/perspective/perspective/src/utils.cpp +++ b/python/perspective/perspective/src/utils.cpp @@ -17,192 +17,202 @@ namespace perspective { namespace binding { -t_dtype type_string_to_t_dtype(std::string value, std::string name) { - auto type = t_dtype::DTYPE_STR; + t_dtype + type_string_to_t_dtype(std::string value, std::string name) { + auto type = t_dtype::DTYPE_STR; - // TODO consider refactor - if (value == "int" || value == "integer" || value == "int64" || value == "long") { - // Python int, long, and Numpy int64 - type = t_dtype::DTYPE_INT64; - } else if (value == "int8") { - // Numpy int8 - type = t_dtype::DTYPE_INT8; - } else if (value == "int16") { - // Numpy int16 - type = t_dtype::DTYPE_INT16; - } else if (value == "int32") { - // Numpy int32 - type = t_dtype::DTYPE_INT32; - } else if (value == "float") { - // Python float - type = t_dtype::DTYPE_FLOAT64; - } else if (value == "float16") { - // TODO - // Numpy float16 - // type = t_dtype::DTYPE_FLOAT16; - type = t_dtype::DTYPE_FLOAT32; - } else if (value == "float32" || value == "float") { - // Numpy float32 - type = t_dtype::DTYPE_FLOAT32; - } else if (value == "float64") { - // Numpy float64 - type = t_dtype::DTYPE_FLOAT64; - } else if (value == "float128") { - // TODO - // Numpy float128 - type = t_dtype::DTYPE_FLOAT64; - } else if (value == "str" || value == "string" || value == "unicode") { - // Python unicode str - type = t_dtype::DTYPE_STR; - } else if (value == "bool" || value == "boolean") { - // Python bool - type = t_dtype::DTYPE_BOOL; - } else if (value == "bool_") { - // Numpy bool - type = t_dtype::DTYPE_BOOL; - } else if (value == "bool8") { - // Numpy bool8 - type = t_dtype::DTYPE_BOOL; - } else if (value == "datetime") { - // Python datetime - // TODO inheritance - type = t_dtype::DTYPE_TIME; - } else if (value == "datetime64") { - // Numpy datetime64 - type = t_dtype::DTYPE_TIME; - } else if (value == "Timestamp") { - // Pandas timestamp - type = t_dtype::DTYPE_TIME; - } else if (value == "Period") { - // Pandas period - type = t_dtype::DTYPE_TIME; - } else if (value == "date") { - // Python date - // TODO inheritance - type = t_dtype::DTYPE_DATE; - } else if (value == "timedelta64" || value == "time") { - // cast time/timedelta to string to preserve units - type = t_dtype::DTYPE_STR; - } else if (value == "object") { - // Python object, stored as PyObject * pointer - type = t_dtype::DTYPE_OBJECT; - } else { - CRITICAL("Unknown type '%s' for key '%s'", value, name); + // TODO consider refactor + if (value == "int" || value == "integer" || value == "int64" + || value == "long") { + // Python int, long, and Numpy int64 + type = t_dtype::DTYPE_INT64; + } else if (value == "int8") { + // Numpy int8 + type = t_dtype::DTYPE_INT8; + } else if (value == "int16") { + // Numpy int16 + type = t_dtype::DTYPE_INT16; + } else if (value == "int32") { + // Numpy int32 + type = t_dtype::DTYPE_INT32; + } else if (value == "float") { + // Python float + type = t_dtype::DTYPE_FLOAT64; + } else if (value == "float16") { + // TODO + // Numpy float16 + // type = t_dtype::DTYPE_FLOAT16; + type = t_dtype::DTYPE_FLOAT32; + } else if (value == "float32" || value == "float") { + // Numpy float32 + type = t_dtype::DTYPE_FLOAT32; + } else if (value == "float64") { + // Numpy float64 + type = t_dtype::DTYPE_FLOAT64; + } else if (value == "float128") { + // TODO + // Numpy float128 + type = t_dtype::DTYPE_FLOAT64; + } else if (value == "str" || value == "string" || value == "unicode") { + // Python unicode str + type = t_dtype::DTYPE_STR; + } else if (value == "bool" || value == "boolean") { + // Python bool + type = t_dtype::DTYPE_BOOL; + } else if (value == "bool_") { + // Numpy bool + type = t_dtype::DTYPE_BOOL; + } else if (value == "bool8") { + // Numpy bool8 + type = t_dtype::DTYPE_BOOL; + } else if (value == "datetime") { + // Python datetime + // TODO inheritance + type = t_dtype::DTYPE_TIME; + } else if (value == "datetime64") { + // Numpy datetime64 + type = t_dtype::DTYPE_TIME; + } else if (value == "Timestamp") { + // Pandas timestamp + type = t_dtype::DTYPE_TIME; + } else if (value == "Period") { + // Pandas period + type = t_dtype::DTYPE_TIME; + } else if (value == "date") { + // Python date + // TODO inheritance + type = t_dtype::DTYPE_DATE; + } else if (value == "timedelta64" || value == "time") { + // cast time/timedelta to string to preserve units + type = t_dtype::DTYPE_STR; + } else if (value == "object") { + // Python object, stored as PyObject * pointer + type = t_dtype::DTYPE_OBJECT; + } else { + CRITICAL("Unknown type '%s' for key '%s'", value, name); + } + return type; } - return type; -} - -t_dtype type_string_to_t_dtype(py::str type, py::str name){ - return type_string_to_t_dtype(type.cast(), name.cast()); -} -t_val -scalar_to_py(const t_tscalar& scalar, bool cast_double, bool cast_string) { - if (!scalar.is_valid()) { - return py::none(); + t_dtype + type_string_to_t_dtype(py::str type, py::str name) { + return type_string_to_t_dtype( + type.cast(), name.cast()); } - - switch (scalar.get_dtype()) { - case DTYPE_BOOL: { - if (scalar.as_bool()) { - return py::cast(true); - } else { - return py::cast(false); - } + + t_val + scalar_to_py(const t_tscalar& scalar, bool cast_double, bool cast_string) { + if (!scalar.is_valid()) { + return py::none(); } - case DTYPE_TIME: { - if (cast_double) { - auto x = scalar.to_uint64(); - double y = *reinterpret_cast(&x); - return py::cast(y); - } else if (cast_string) { - return py::cast(scalar.to_string(false)); // should reimplement - } else { - /** - * datetimes are stored as milliseconds since epoch. - * Before datetimes are loaded into Perspective, if they are - * time zone aware, they must be converted into UTC. - */ - auto i64 = scalar.to_int64(); - // check for datetime >= 10000-01-01 00:00:00 - if (i64 >= 253402318800000) { - // Python has a max year of 9999 - Perspective is able to - // store POSIX timestamps above Python's `datetime.max`, - // but it cannot be converted back out to Python so it is - // functionally useless. Instead, truncate the offending - // date and return `datetime.max`. - std::stringstream ss; - ss << "Python cannot display dates above `datetime.max` - timestamp `"; - ss << i64; - ss << "` will be truncated to `datetime.max`."; - ss << std::endl; - std::cerr << ss.str(); - i64 = 253402300799000; + switch (scalar.get_dtype()) { + case DTYPE_BOOL: { + if (scalar.as_bool()) { + return py::cast(true); + } else { + return py::cast(false); } + } + case DTYPE_TIME: { + if (cast_double) { + auto x = scalar.to_uint64(); + double y = *reinterpret_cast(&x); + return py::cast(y); + } else if (cast_string) { + return py::cast( + scalar.to_string(false)); // should reimplement + } else { + /** + * datetimes are stored as milliseconds since epoch. + * Before datetimes are loaded into Perspective, if they are + * time zone aware, they must be converted into UTC. + */ + auto i64 = scalar.to_int64(); - auto ms = std::chrono::milliseconds(i64); - auto time_point = std::chrono::time_point(ms); - /** - * Pybind converts std::time_point to local time, and the - * `datetime.datetime` object created by `py::cast` has NO - * `timezone` property. It is created using `std::localtime`, - * and cannot be made timezone-aware. - */ + // check for datetime >= 10000-01-01 00:00:00 + if (i64 >= 253402318800000) { + // Python has a max year of 9999 - Perspective is able + // to store POSIX timestamps above Python's + // `datetime.max`, but it cannot be converted back out + // to Python so it is functionally useless. Instead, + // truncate the offending date and return + // `datetime.max`. + std::stringstream ss; + ss << "Python cannot display dates above " + "`datetime.max` - timestamp `"; + ss << i64; + ss << "` will be truncated to `datetime.max`."; + ss << std::endl; + std::cerr << ss.str(); + i64 = 253402300799000; + } + + auto ms = std::chrono::milliseconds(i64); + auto time_point + = std::chrono::time_point( + ms); + /** + * Pybind converts std::time_point to local time, and the + * `datetime.datetime` object created by `py::cast` has NO + * `timezone` property. It is created using + * `std::localtime`, and cannot be made timezone-aware. + */ + return py::cast(time_point); + } + } + case DTYPE_FLOAT32: { + return py::cast(scalar.get()); + } + case DTYPE_FLOAT64: { + if (cast_double) { + auto x = scalar.to_uint64(); + double y = *reinterpret_cast(&x); + return py::cast(y); + } else { + return py::cast(scalar.to_double()); + } + } + case DTYPE_DATE: { + t_date date = scalar.get(); + std::tm tm = date.get_tm(); + auto mkt = std::mktime(&tm); + auto time_point = std::chrono::system_clock::from_time_t(mkt); return py::cast(time_point); } - } - case DTYPE_FLOAT32: { - return py::cast(scalar.get()); - } - case DTYPE_FLOAT64: { - if (cast_double) { - auto x = scalar.to_uint64(); - double y = *reinterpret_cast(&x); - return py::cast(y); - } else { - return py::cast(scalar.to_double()); + case DTYPE_UINT8: + case DTYPE_UINT16: + case DTYPE_UINT32: + case DTYPE_INT8: + case DTYPE_INT16: + case DTYPE_INT32: + case DTYPE_UINT64: + case DTYPE_INT64: { + return py::cast(scalar.to_int64()); } - } - case DTYPE_DATE: { - t_date date = scalar.get(); - std::tm tm = date.get_tm(); - auto mkt = std::mktime(&tm); - auto time_point = std::chrono::system_clock::from_time_t(mkt); - return py::cast(time_point); - } - case DTYPE_UINT8: - case DTYPE_UINT16: - case DTYPE_UINT32: - case DTYPE_INT8: - case DTYPE_INT16: - case DTYPE_INT32: - case DTYPE_UINT64: - case DTYPE_INT64: { - return py::cast(scalar.to_int64()); - } - case DTYPE_OBJECT: { - // Extract pointer - PyObject *ptr = static_cast((void *)scalar.to_uint64()); + case DTYPE_OBJECT: { + // Extract pointer + PyObject* ptr + = static_cast((void*)scalar.to_uint64()); - // nullptr - if(!scalar.to_uint64()){ + // nullptr + if (!scalar.to_uint64()) { + return py::none(); + } + // Reconstruct python object + return py::cast(ptr); + } + case DTYPE_NONE: { return py::none(); } - // Reconstruct python object - return py::cast(ptr); - } - case DTYPE_NONE: { - return py::none(); - } - case DTYPE_STR: - default: { - return py::cast(scalar.to_string()); + case DTYPE_STR: + default: { + return py::cast(scalar.to_string()); + } } } -} -} //namespace binding -} //namespace perspective +} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/src/view.cpp b/python/perspective/perspective/src/view.cpp index ede9927d8c..a189ea3921 100644 --- a/python/perspective/perspective/src/view.cpp +++ b/python/perspective/perspective/src/view.cpp @@ -9,385 +9,429 @@ #ifdef PSP_ENABLE_PYTHON #include - namespace perspective { namespace binding { -/****************************************************************************** - * - * View API - */ - -template <> -bool -is_valid_filter(t_dtype type, t_val date_parser, t_filter_op comp, t_val filter_term) { - if (comp == t_filter_op::FILTER_OP_IS_NULL - || comp == t_filter_op::FILTER_OP_IS_NOT_NULL) { - return true; - } else if (type == DTYPE_DATE || type == DTYPE_TIME) { - if (py::isinstance(filter_term)) { - t_val parsed_date = date_parser.attr("parse")(filter_term); - return !parsed_date.is_none(); + /****************************************************************************** + * + * View API + */ + + template <> + bool + is_valid_filter( + t_dtype type, t_val date_parser, t_filter_op comp, t_val filter_term) { + if (comp == t_filter_op::FILTER_OP_IS_NULL + || comp == t_filter_op::FILTER_OP_IS_NOT_NULL) { + return true; + } else if (type == DTYPE_DATE || type == DTYPE_TIME) { + if (py::isinstance(filter_term)) { + t_val parsed_date = date_parser.attr("parse")(filter_term); + return !parsed_date.is_none(); + } else { + return !filter_term.is_none(); + } } else { return !filter_term.is_none(); } - } else { - return !filter_term.is_none(); - } -}; - -template <> -std::tuple> -make_filter_term(t_dtype column_type, t_val date_parser, const std::string& column_name, const std::string& filter_op_str, t_val filter_term) { - t_filter_op filter_op = str_to_filter_op(filter_op_str); - std::vector terms; - - switch (filter_op) { - case FILTER_OP_NOT_IN: - case FILTER_OP_IN: { - std::vector filter_terms - = filter_term.cast>(); - for (auto term : filter_terms) { - terms.push_back(mktscalar(get_interned_cstr(term.c_str()))); - } - } break; - case FILTER_OP_IS_NULL: - case FILTER_OP_IS_NOT_NULL: { - terms.push_back(mktscalar(0)); - } break; - default: { - switch (column_type) { - case DTYPE_INT32: { - terms.push_back(mktscalar(filter_term.cast())); - } break; - case DTYPE_INT64: - case DTYPE_FLOAT64: { - terms.push_back(mktscalar(filter_term.cast())); - } break; - case DTYPE_BOOL: { - terms.push_back(mktscalar(filter_term.cast())); - } break; - case DTYPE_DATE: { - if (py::isinstance(filter_term)) { - t_val parsed_date = date_parser.attr("parse")(filter_term); - auto date_components = - date_parser.attr("to_date_components")(parsed_date).cast>(); - t_date dt = t_date(date_components["year"], date_components["month"], date_components["day"]); - terms.push_back(mktscalar(dt)); - } else { - auto date_components = - date_parser.attr("to_date_components")(filter_term).cast>(); - t_date dt = t_date(date_components["year"], date_components["month"], date_components["day"]); - terms.push_back(mktscalar(dt)); - } - } break; - case DTYPE_TIME: { - if (py::isinstance(filter_term)) { - t_val parsed_date = date_parser.attr("parse")(filter_term); - std::int64_t ts = date_parser.attr("to_timestamp")(parsed_date).cast(); - t_tscalar timestamp = mktscalar(t_time(ts)); - terms.push_back(timestamp); - } else { - t_tscalar timestamp = mktscalar( - t_time(date_parser.attr("to_timestamp")(filter_term).cast())); - terms.push_back(timestamp); + }; + + template <> + std::tuple> + make_filter_term(t_dtype column_type, t_val date_parser, + const std::string& column_name, const std::string& filter_op_str, + t_val filter_term) { + t_filter_op filter_op = str_to_filter_op(filter_op_str); + std::vector terms; + + switch (filter_op) { + case FILTER_OP_NOT_IN: + case FILTER_OP_IN: { + std::vector filter_terms + = filter_term.cast>(); + for (auto term : filter_terms) { + terms.push_back(mktscalar(get_interned_cstr(term.c_str()))); + } + } break; + case FILTER_OP_IS_NULL: + case FILTER_OP_IS_NOT_NULL: { + terms.push_back(mktscalar(0)); + } break; + default: { + switch (column_type) { + case DTYPE_INT32: { + terms.push_back( + mktscalar(filter_term.cast())); + } break; + case DTYPE_INT64: + case DTYPE_FLOAT64: { + terms.push_back(mktscalar(filter_term.cast())); + } break; + case DTYPE_BOOL: { + terms.push_back(mktscalar(filter_term.cast())); + } break; + case DTYPE_DATE: { + if (py::isinstance(filter_term)) { + t_val parsed_date + = date_parser.attr("parse")(filter_term); + auto date_components + = date_parser + .attr("to_date_components")(parsed_date) + .cast>(); + t_date dt = t_date(date_components["year"], + date_components["month"], + date_components["day"]); + terms.push_back(mktscalar(dt)); + } else { + auto date_components + = date_parser + .attr("to_date_components")(filter_term) + .cast>(); + t_date dt = t_date(date_components["year"], + date_components["month"], + date_components["day"]); + terms.push_back(mktscalar(dt)); + } + } break; + case DTYPE_TIME: { + if (py::isinstance(filter_term)) { + t_val parsed_date + = date_parser.attr("parse")(filter_term); + std::int64_t ts + = date_parser.attr("to_timestamp")(parsed_date) + .cast(); + t_tscalar timestamp = mktscalar(t_time(ts)); + terms.push_back(timestamp); + } else { + t_tscalar timestamp = mktscalar(t_time( + date_parser.attr("to_timestamp")(filter_term) + .cast())); + terms.push_back(timestamp); + } + } break; + default: { + terms.push_back(mktscalar(get_interned_cstr( + filter_term.cast().c_str()))); } - } break; - default: { - terms.push_back( - mktscalar(get_interned_cstr(filter_term.cast().c_str()))); } } } + return std::make_tuple(column_name, filter_op_str, terms); } - return std::make_tuple(column_name, filter_op_str, terms); -} - -template <> -std::shared_ptr -make_view_config( - const t_gnode& gnode, - std::shared_ptr schema, t_val date_parser, t_val config) { - auto row_pivots = config.attr("get_row_pivots")().cast>(); - auto column_pivots = config.attr("get_column_pivots")().cast>(); - auto columns = config.attr("get_columns")().cast>(); - auto sort = config.attr("get_sort")().cast>>(); - auto filter_op = config.attr("get_filter_op")().cast(); - - // to preserve order, do not cast to std::map - use keys and python 3.7's guarantee that dicts respect insertion order - auto p_aggregates = py::dict(config.attr("get_aggregates")()); - tsl::ordered_map> aggregates; - - for (auto& column : columns) { - py::str py_column_name = py::str(column); - if (p_aggregates.contains(py_column_name)) { - if (py::isinstance(p_aggregates[py_column_name])) { - std::vector agg{ - p_aggregates[py_column_name].cast()}; - aggregates[column] = agg; - } else { - aggregates[column] = p_aggregates[py_column_name].cast>(); + + template <> + std::shared_ptr + make_view_config(const t_gnode& gnode, std::shared_ptr schema, + t_val date_parser, t_val config) { + auto row_pivots + = config.attr("get_row_pivots")().cast>(); + auto column_pivots = config.attr("get_column_pivots")() + .cast>(); + auto columns + = config.attr("get_columns")().cast>(); + auto sort = config.attr("get_sort")() + .cast>>(); + auto filter_op = config.attr("get_filter_op")().cast(); + + // to preserve order, do not cast to std::map - use keys and + // python 3.7's guarantee that dicts respect insertion order + auto p_aggregates = py::dict(config.attr("get_aggregates")()); + tsl::ordered_map> aggregates; + + for (auto& column : columns) { + py::str py_column_name = py::str(column); + if (p_aggregates.contains(py_column_name)) { + if (py::isinstance(p_aggregates[py_column_name])) { + std::vector agg{ + p_aggregates[py_column_name].cast()}; + aggregates[column] = agg; + } else { + aggregates[column] = p_aggregates[py_column_name] + .cast>(); + } } + }; + + bool column_only = false; + + // make sure that primary keys are created for column-only views + if (row_pivots.size() == 0 && column_pivots.size() > 0) { + row_pivots.push_back("psp_okey"); + column_only = true; } - }; - bool column_only = false; + auto p_expressions = config.attr("get_expressions")() + .cast>>(); + std::vector> expressions; + expressions.reserve(p_expressions.size()); + + // Validate expressions using the vocab + t_expression_vocab& expression_vocab = *(gnode.get_expression_vocab()); + t_regex_mapping& regex_mapping + = *(gnode.get_expression_regex_mapping()); + + // Will either abort() or succeed completely, and this isn't a public + // API so we can directly index for speed. + for (t_uindex idx = 0; idx < p_expressions.size(); ++idx) { + const auto& expr = p_expressions[idx]; + std::string expression_alias = expr[0].cast(); + std::string expression_string = expr[1].cast(); + std::string parsed_expression_string = expr[2].cast(); + + // Don't allow overwriting of "real" table columns or multiple + // columns with the same alias. + if (schema->has_column(expression_alias)) { + std::stringstream ss; + ss << "View creation failed: cannot create expression column '" + << expression_alias + << "' that overwrites a column that already exists." + << std::endl; + PSP_COMPLAIN_AND_ABORT(ss.str()); + } + + auto p_column_ids = py::dict(expr[3]); + std::vector> column_ids; + column_ids.resize(p_column_ids.size()); + t_uindex cidx = 0; - // make sure that primary keys are created for column-only views - if (row_pivots.size() == 0 && column_pivots.size() > 0) { - row_pivots.push_back("psp_okey"); - column_only = true; - } + for (const auto& item : p_column_ids) { + column_ids[cidx] = std::pair( + item.first.cast(), + item.second.cast()); + ++cidx; + } + + // If the expression cannot be parsed, it will abort() here. + std::shared_ptr expression + = t_computed_expression_parser::precompute(expression_alias, + expression_string, parsed_expression_string, column_ids, + schema, expression_vocab, regex_mapping); - auto p_expressions = config.attr("get_expressions")().cast>>(); - std::vector> expressions; - expressions.reserve(p_expressions.size()); - - // Validate expressions using the vocab - t_expression_vocab& expression_vocab = *(gnode.get_expression_vocab()); - t_regex_mapping& regex_mapping = *(gnode.get_expression_regex_mapping()); - - // Will either abort() or succeed completely, and this isn't a public - // API so we can directly index for speed. - for (t_uindex idx = 0; idx < p_expressions.size(); ++idx) { - const auto& expr = p_expressions[idx]; - std::string expression_alias = expr[0].cast(); - std::string expression_string = expr[1].cast(); - std::string parsed_expression_string = expr[2].cast(); - - // Don't allow overwriting of "real" table columns or multiple - // columns with the same alias. - if (schema->has_column(expression_alias)) { - std::stringstream ss; - ss << "View creation failed: cannot create expression column '" - << expression_alias - << "' that overwrites a column that already exists." - << std::endl; - PSP_COMPLAIN_AND_ABORT(ss.str()); + expressions.push_back(expression); + schema->add_column(expression_alias, expression->get_dtype()); } - auto p_column_ids = py::dict(expr[3]); - std::vector> column_ids; - column_ids.resize(p_column_ids.size()); - t_uindex cidx = 0; - - for (const auto& item : p_column_ids) { - column_ids[cidx] = std::pair( - item.first.cast(), - item.second.cast()); - ++cidx; + // construct filters with filter terms, and fill the vector of tuples + auto p_filter = config.attr("get_filter")() + .cast>>(); + std::vector< + std::tuple>> + filter; + + for (auto f : p_filter) { + // parse filter details + std::string column_name = f[0].cast(); + std::string filter_op_str = f[1].cast(); + t_dtype column_type = schema->get_dtype(column_name); + t_filter_op filter_operator = str_to_filter_op(filter_op_str); + + // validate the filter before it goes into the core engine + t_val filter_term = py::none(); + if (f.size() > 2) { + // null/not null filters do not have a filter term + filter_term = f[2]; + } + + if (is_valid_filter( + column_type, date_parser, filter_operator, filter_term)) { + filter.push_back(make_filter_term(column_type, date_parser, + column_name, filter_op_str, filter_term)); + } } - // If the expression cannot be parsed, it will abort() here. - std::shared_ptr expression = - t_computed_expression_parser::precompute( - expression_alias, expression_string, parsed_expression_string, - column_ids, schema, expression_vocab, regex_mapping); + // create the `t_view_config` + auto view_config = std::make_shared(row_pivots, + column_pivots, aggregates, columns, filter, sort, expressions, + filter_op, column_only); - expressions.push_back(expression); - schema->add_column(expression_alias, expression->get_dtype()); - } + // transform primitive values into abstractions that the engine can use + view_config->init(schema); - // construct filters with filter terms, and fill the vector of tuples - auto p_filter = config.attr("get_filter")().cast>>(); - std::vector>> filter; - - for (auto f : p_filter) { - // parse filter details - std::string column_name = f[0].cast(); - std::string filter_op_str = f[1].cast(); - t_dtype column_type = schema->get_dtype(column_name); - t_filter_op filter_operator = str_to_filter_op(filter_op_str); - - // validate the filter before it goes into the core engine - t_val filter_term = py::none(); - if (f.size() > 2) { - // null/not null filters do not have a filter term - filter_term = f[2]; + // set pivot depths if provided + if (!config.attr("row_pivot_depth").is_none()) { + view_config->set_row_pivot_depth( + config.attr("row_pivot_depth").cast()); } - if (is_valid_filter(column_type, date_parser, filter_operator, filter_term)) { - filter.push_back(make_filter_term(column_type, date_parser, column_name, filter_op_str, filter_term)); + if (!config.attr("column_pivot_depth").is_none()) { + view_config->set_column_pivot_depth( + config.attr("column_pivot_depth").cast()); + } + + return view_config; + } + + /****************************************************************************** + * + * make_view + */ + + template + std::shared_ptr> + make_view(std::shared_ptr
table, const std::string& name, + const std::string& separator, t_val view_config, t_val date_parser) { + // Use a copy of the table schema that we can freely mutate during + // `make_view_config` and pass into the context constructors. + std::shared_ptr schema + = std::make_shared(table->get_schema()); + + // Pass the gnode into `make_view_config` so we can use its vocab to + // validate expressions. + const t_gnode& gnode = *(table->get_gnode()); + + std::shared_ptr config + = make_view_config(gnode, schema, date_parser, view_config); + { + PerspectiveScopedGILRelease acquire( + table->get_pool()->get_event_loop_thread_id()); + auto ctx = make_context(table, schema, config, name); + auto view_ptr = std::make_shared>( + table, ctx, name, separator, config); + return view_ptr; } } - // create the `t_view_config` - auto view_config = std::make_shared( - row_pivots, - column_pivots, - aggregates, - columns, - filter, - sort, - expressions, - filter_op, - column_only); - - // transform primitive values into abstractions that the engine can use - view_config->init(schema); - - // set pivot depths if provided - if (! config.attr("row_pivot_depth").is_none()) { - view_config->set_row_pivot_depth(config.attr("row_pivot_depth").cast()); + std::shared_ptr> + make_view_unit(std::shared_ptr
table, std::string name, + std::string separator, t_val view_config, t_val date_parser) { + return make_view( + table, name, separator, view_config, date_parser); } - if (! config.attr("column_pivot_depth").is_none()) { - view_config->set_column_pivot_depth(config.attr("column_pivot_depth").cast()); + std::shared_ptr> + make_view_ctx0(std::shared_ptr
table, std::string name, + std::string separator, t_val view_config, t_val date_parser) { + return make_view( + table, name, separator, view_config, date_parser); } - return view_config; -} + std::shared_ptr> + make_view_ctx1(std::shared_ptr
table, std::string name, + std::string separator, t_val view_config, t_val date_parser) { + return make_view( + table, name, separator, view_config, date_parser); + } -/****************************************************************************** - * - * make_view - */ + std::shared_ptr> + make_view_ctx2(std::shared_ptr
table, std::string name, + std::string separator, t_val view_config, t_val date_parser) { + return make_view( + table, name, separator, view_config, date_parser); + } -template -std::shared_ptr> -make_view(std::shared_ptr
table, const std::string& name, const std::string& separator, - t_val view_config, t_val date_parser) { - // Use a copy of the table schema that we can freely mutate during - // `make_view_config` and pass into the context constructors. - std::shared_ptr schema = std::make_shared(table->get_schema()); - - // Pass the gnode into `make_view_config` so we can use its vocab to - // validate expressions. - const t_gnode& gnode = *(table->get_gnode()); - - std::shared_ptr config = make_view_config(gnode, schema, date_parser, view_config); - { - PerspectiveScopedGILRelease acquire(table->get_pool()->get_event_loop_thread_id()); - auto ctx = make_context(table, schema, config, name); - auto view_ptr = std::make_shared>(table, ctx, name, separator, config); - return view_ptr; + /****************************************************************************** + * + * to_arrow + */ + + py::bytes + to_arrow_unit(std::shared_ptr> view, std::int32_t start_row, + std::int32_t end_row, std::int32_t start_col, std::int32_t end_col) { + PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); + std::shared_ptr str + = view->to_arrow(start_row, end_row, start_col, end_col, true); + return py::bytes(*str); } -} - -std::shared_ptr> -make_view_unit(std::shared_ptr
table, std::string name, std::string separator, - t_val view_config, t_val date_parser) { - return make_view(table, name, separator, view_config, date_parser); -} - -std::shared_ptr> -make_view_ctx0(std::shared_ptr
table, std::string name, std::string separator, - t_val view_config, t_val date_parser) { - return make_view(table, name, separator, view_config, date_parser); -} - -std::shared_ptr> -make_view_ctx1(std::shared_ptr
table, std::string name, std::string separator, - t_val view_config, t_val date_parser) { - return make_view(table, name, separator, view_config, date_parser); -} - -std::shared_ptr> -make_view_ctx2(std::shared_ptr
table, std::string name, std::string separator, - t_val view_config, t_val date_parser) { - return make_view(table, name, separator, view_config, date_parser); -} -/****************************************************************************** - * - * to_arrow - */ + py::bytes + to_arrow_zero(std::shared_ptr> view, std::int32_t start_row, + std::int32_t end_row, std::int32_t start_col, std::int32_t end_col) { + PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); + std::shared_ptr str + = view->to_arrow(start_row, end_row, start_col, end_col, true); + return py::bytes(*str); + } -py::bytes -to_arrow_unit( - std::shared_ptr> view, - std::int32_t start_row, - std::int32_t end_row, - std::int32_t start_col, - std::int32_t end_col -) { - PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); - std::shared_ptr str = - view->to_arrow(start_row, end_row, start_col, end_col); - return py::bytes(*str); -} - -py::bytes -to_arrow_zero( - std::shared_ptr> view, - std::int32_t start_row, - std::int32_t end_row, - std::int32_t start_col, - std::int32_t end_col -) { - PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); - std::shared_ptr str = - view->to_arrow(start_row, end_row, start_col, end_col); - return py::bytes(*str); -} - -py::bytes -to_arrow_one( - std::shared_ptr> view, - std::int32_t start_row, - std::int32_t end_row, - std::int32_t start_col, - std::int32_t end_col -) { - PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); - std::shared_ptr str = - view->to_arrow(start_row, end_row, start_col, end_col); - return py::bytes(*str); -} - -py::bytes -to_arrow_two( - std::shared_ptr> view, - std::int32_t start_row, - std::int32_t end_row, - std::int32_t start_col, - std::int32_t end_col -) { - PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); - std::shared_ptr str = - view->to_arrow(start_row, end_row, start_col, end_col); - return py::bytes(*str); -} + py::bytes + to_arrow_one(std::shared_ptr> view, std::int32_t start_row, + std::int32_t end_row, std::int32_t start_col, std::int32_t end_col) { + PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); + std::shared_ptr str + = view->to_arrow(start_row, end_row, start_col, end_col, true); + return py::bytes(*str); + } -/****************************************************************************** - * - * get_row_delta - */ + py::bytes + to_arrow_two(std::shared_ptr> view, std::int32_t start_row, + std::int32_t end_row, std::int32_t start_col, std::int32_t end_col) { + PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); + std::shared_ptr str + = view->to_arrow(start_row, end_row, start_col, end_col, true); + return py::bytes(*str); + } + + std::string + to_csv_unit(std::shared_ptr> view, std::int32_t start_row, + std::int32_t end_row, std::int32_t start_col, std::int32_t end_col) { + PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); + return *view->to_csv(start_row, end_row, start_col, end_col); + } + + std::string + to_csv_zero(std::shared_ptr> view, std::int32_t start_row, + std::int32_t end_row, std::int32_t start_col, std::int32_t end_col) { + PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); + return *view->to_csv(start_row, end_row, start_col, end_col); + } + + std::string + to_csv_one(std::shared_ptr> view, std::int32_t start_row, + std::int32_t end_row, std::int32_t start_col, std::int32_t end_col) { + PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); + return *view->to_csv(start_row, end_row, start_col, end_col); + } + + std::string + to_csv_two(std::shared_ptr> view, std::int32_t start_row, + std::int32_t end_row, std::int32_t start_col, std::int32_t end_col) { + PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); + return *view->to_csv(start_row, end_row, start_col, end_col); + } + + /****************************************************************************** + * + * get_row_delta + */ + + py::bytes + get_row_delta_unit(std::shared_ptr> view) { + PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); + std::shared_ptr> slice = view->get_row_delta(); + std::shared_ptr arrow + = view->data_slice_to_arrow(slice, false); + return py::bytes(*arrow); + } + + py::bytes + get_row_delta_zero(std::shared_ptr> view) { + PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); + std::shared_ptr> slice = view->get_row_delta(); + std::shared_ptr arrow + = view->data_slice_to_arrow(slice, false); + return py::bytes(*arrow); + } + + py::bytes + get_row_delta_one(std::shared_ptr> view) { + PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); + std::shared_ptr> slice = view->get_row_delta(); + std::shared_ptr arrow + = view->data_slice_to_arrow(slice, false); + return py::bytes(*arrow); + } + + py::bytes + get_row_delta_two(std::shared_ptr> view) { + PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); + std::shared_ptr> slice = view->get_row_delta(); + std::shared_ptr arrow + = view->data_slice_to_arrow(slice, false); + return py::bytes(*arrow); + } -py::bytes -get_row_delta_unit(std::shared_ptr> view) { - PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); - std::shared_ptr> slice = view->get_row_delta(); - std::shared_ptr arrow = view->data_slice_to_arrow(slice); - return py::bytes(*arrow); -} - -py::bytes -get_row_delta_zero(std::shared_ptr> view) { - PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); - std::shared_ptr> slice = view->get_row_delta(); - std::shared_ptr arrow = view->data_slice_to_arrow(slice); - return py::bytes(*arrow); -} - -py::bytes -get_row_delta_one(std::shared_ptr> view) { - PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); - std::shared_ptr> slice = view->get_row_delta(); - std::shared_ptr arrow = view->data_slice_to_arrow(slice); - return py::bytes(*arrow); -} - -py::bytes -get_row_delta_two( - std::shared_ptr> view) { - PerspectiveScopedGILRelease acquire(view->get_event_loop_thread_id()); - std::shared_ptr> slice = view->get_row_delta(); - std::shared_ptr arrow = view->data_slice_to_arrow(slice); - return py::bytes(*arrow); -} - -} //namespace binding -} //namespace perspective +} // namespace binding +} // namespace perspective #endif \ No newline at end of file diff --git a/python/perspective/perspective/table/_accessor.py b/python/perspective/perspective/table/_accessor.py index dd526a36f0..f9026ae768 100644 --- a/python/perspective/perspective/table/_accessor.py +++ b/python/perspective/perspective/table/_accessor.py @@ -6,16 +6,16 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six -import pandas -import numpy from distutils.util import strtobool from math import isnan -from ._date_validator import _PerspectiveDateValidator -from ..core.data import deconstruct_numpy, make_null_mask, deconstruct_pandas +import numpy +import pandas + +from ..core.data import deconstruct_numpy, deconstruct_pandas, make_null_mask from ..core.data.pd import _parse_datetime_index from ..core.exception import PerspectiveError +from ._date_validator import _PerspectiveDateValidator from .libbinding import t_dtype @@ -53,7 +53,7 @@ def _type_to_format(data_or_schema): elif isinstance(data_or_schema, dict): # schema or columns for v in data_or_schema.values(): - if isinstance(v, type) or isinstance(v, six.string_types): + if isinstance(v, type) or isinstance(v, str): # schema maps name -> type return False, 2, list(data_or_schema.keys()), data_or_schema elif isinstance(v, list): @@ -104,7 +104,10 @@ class _PerspectiveAccessor(object): common :func:`marshal` function. """ - INTEGER_TYPES = six.integer_types + (numpy.integer,) + INTEGER_TYPES = ( + int, + numpy.integer, + ) def __init__(self, data_or_schema): ( @@ -127,7 +130,7 @@ def __init__(self, data_or_schema): # Verify that column names are strings, and that numpy arrays are of # type `ndarray` for name in self._names: - if not isinstance(name, six.string_types): + if not isinstance(name, str): raise PerspectiveError( "Column names should be strings, not type `{0}`".format( type(name).__name__ @@ -241,22 +244,17 @@ def marshal(self, cidx, ridx, dtype): if isinstance(val, (bytes, bytearray)): return val.decode("utf-8") else: - if six.PY2: - # six.u mangles quotes with escape sequences - use native - # unicode() - return unicode(val) # noqa: F821 - else: - return str(val) + return str(val) elif dtype == t_dtype.DTYPE_DATE: # return datetime.date - if isinstance(val, six.string_types): + if isinstance(val, str): parsed = self._date_validator.parse(val) return self._date_validator.to_date_components(parsed) else: return self._date_validator.to_date_components(val) elif dtype == t_dtype.DTYPE_TIME: # return unix timestamps for time - if isinstance(val, six.string_types): + if isinstance(val, str): parsed = self._date_validator.parse(val) return self._date_validator.to_timestamp(parsed) else: diff --git a/python/perspective/perspective/table/_date_validator.py b/python/perspective/perspective/table/_date_validator.py index 7a0d2f0ab3..70dd729b48 100644 --- a/python/perspective/perspective/table/_date_validator.py +++ b/python/perspective/perspective/table/_date_validator.py @@ -6,7 +6,6 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six import numpy from calendar import timegm from datetime import date, datetime @@ -17,9 +16,6 @@ from time import mktime from .libbinding import t_dtype -if six.PY2: - from past.builtins import long - def _normalize_timestamp(obj): """Convert a timestamp in seconds to milliseconds. @@ -73,16 +69,12 @@ def to_date_components(self, obj): if isinstance(obj, (int, float)): obj = datetime.fromtimestamp(_normalize_timestamp(obj) / 1000) - if six.PY2: - if isinstance(obj, (long)): - obj = datetime.fromtimestamp(long(obj)) - if isinstance(obj, numpy.datetime64): if str(obj) == "NaT": return None obj = obj.astype(datetime) - if (six.PY2 and isinstance(obj, long)) or isinstance(obj, int): + if isinstance(obj, int): obj = datetime.fromtimestamp(obj / 1000000000) # Perspective stores month in `t_date` as an integer [0-11], @@ -124,12 +116,6 @@ def to_timestamp(self, obj): converter = timegm to_timetuple = "utctimetuple" - if six.PY2: - if isinstance(obj, long): - # compat with python2 long from datetime.datetime - obj = obj / 1000000000 - return long(obj) - if isinstance(obj, numpy.datetime64): if str(obj) == "NaT": return None @@ -142,10 +128,6 @@ def to_timestamp(self, obj): # into `datetime.date`. return int((converter(getattr(obj, to_timetuple)())) * 1000) - if six.PY2: - if isinstance(obj, long): - return long(round(obj / 1000000)) - if isinstance(obj, int): return round(obj / 1000000) diff --git a/python/perspective/perspective/table/_state.py b/python/perspective/perspective/table/_state.py index 069ac2b911..3c245f4fff 100644 --- a/python/perspective/perspective/table/_state.py +++ b/python/perspective/perspective/table/_state.py @@ -1,4 +1,4 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # diff --git a/python/perspective/perspective/table/table.py b/python/perspective/perspective/table/table.py index bcb971be66..736afd3296 100644 --- a/python/perspective/perspective/table/table.py +++ b/python/perspective/perspective/table/table.py @@ -6,28 +6,28 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -from six import string_types, iteritems from datetime import date, datetime -from .view import View + +from ..core.exception import PerspectiveError from ._accessor import _PerspectiveAccessor from ._callback_cache import _PerspectiveCallBackCache -from ..core.exception import PerspectiveError from ._date_validator import _PerspectiveDateValidator from ._state import _PerspectiveStateManager from ._utils import ( _dtype_to_pythontype, _dtype_to_str, - _str_to_pythontype, _parse_expression_strings, + _str_to_pythontype, ) from .libbinding import ( make_table, - validate_expressions, str_to_filter_op, + t_dtype, t_filter_op, t_op, - t_dtype, + validate_expressions, ) +from .view import View class Table(object): @@ -200,13 +200,13 @@ def validate_expressions(self, expressions, as_string=False): # full expression string in the UI. validated["expression_alias"][expression[0]] = expression[1] - for (alias, dtype) in iteritems(expression_schema): + for (alias, dtype) in expression_schema.items(): if not as_string: dtype = _str_to_pythontype(dtype) validated["expression_schema"][alias] = expression_schema[alias] - for (alias, error) in iteritems(expression_errors): + for (alias, error) in expression_errors.items(): error_dict = {} error_dict["error_message"] = error.error_message error_dict["line"] = error.line @@ -237,7 +237,7 @@ def is_valid_filter(self, filter): Returns: :obj:`bool`: Whether this filter is valid. """ - if isinstance(filter[1], string_types): + if isinstance(filter[1], str): filter_op = str_to_filter_op(filter[1]) else: filter_op = filter[1] @@ -257,7 +257,7 @@ def is_valid_filter(self, filter): schema = self.schema() in_schema = schema.get(filter[0], None) if in_schema and (schema[filter[0]] == date or schema[filter[0]] == datetime): - if isinstance(value, string_types): + if isinstance(value, str): value = self._date_validator.parse(value) return value is not None diff --git a/python/perspective/perspective/table/view.py b/python/perspective/perspective/table/view.py index 4c1f9465e3..eaf751d05a 100644 --- a/python/perspective/perspective/table/view.py +++ b/python/perspective/perspective/table/view.py @@ -6,7 +6,6 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -import os import pandas from functools import partial, wraps from random import random @@ -26,6 +25,10 @@ to_arrow_zero, to_arrow_one, to_arrow_two, + to_csv_unit, + to_csv_zero, + to_csv_one, + to_csv_two, get_row_delta_unit, get_row_delta_zero, get_row_delta_one, @@ -567,7 +570,7 @@ def to_df(self, **options): cols = self.to_numpy(**options) return pandas.DataFrame(cols) - def to_csv(self, **options): + def to_csv(self, **kwargs): """Serialize the :class:`~perspective.View`'s dataset into a CSV string. Keyword Args: @@ -589,19 +592,40 @@ def to_csv(self, **options): Returns: :obj:`str`: A CSV-formatted string containing the serialized data. """ - date_format = None - - # Handle to_csv calls from ``, which uses the - # JavaScript Intl.DateTimeFormat API that takes a locale instead of a - # string format. - # TODO This should move to portable code. - if options.pop("formatted", False): - date_format = "%Y/%m/%d %H:%M:%S" - - return self.to_df(**options).to_csv( - date_format=date_format, - line_terminator="\r\n" if os.name == "nt" else "\n", - ) + + options = _parse_format_options(self, kwargs) + if self._is_unit_context: + return to_csv_unit( + self._view, + options["start_row"], + options["end_row"], + options["start_col"], + options["end_col"], + ) + elif self._sides == 0: + return to_csv_zero( + self._view, + options["start_row"], + options["end_row"], + options["start_col"], + options["end_col"], + ) + elif self._sides == 1: + return to_csv_one( + self._view, + options["start_row"], + options["end_row"], + options["start_col"], + options["end_col"], + ) + else: + return to_csv_two( + self._view, + options["start_row"], + options["end_row"], + options["start_col"], + options["end_col"], + ) @wraps(to_records) def to_json(self, **options): diff --git a/python/perspective/perspective/tests/client_mode/test_client_mode.py b/python/perspective/perspective/tests/client_mode/test_client_mode.py index 06a8daf115..14ac76258f 100644 --- a/python/perspective/perspective/tests/client_mode/test_client_mode.py +++ b/python/perspective/perspective/tests/client_mode/test_client_mode.py @@ -6,14 +6,14 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six import os -import numpy as np -import pandas as pd from datetime import date, datetime from functools import partial from types import MethodType +import numpy as np +import pandas as pd + if os.name == 'nt': BINDING = 'libbinding.pyd' PSP = 'libpsp.dll' @@ -292,28 +292,6 @@ def test_widget_client_schema(self): "f": "string" } - def test_widget_client_schema_py2_types(self): - import perspective - assert perspective.is_libpsp() is False - if six.PY2: - widget = perspective.PerspectiveWidget({ - "a": long, # noqa: F821 - "b": float, - "c": bool, - "d": date, - "e": datetime, - "f": unicode # noqa: F821 - }) - assert hasattr(widget, "table") is False - assert widget._data == { - "a": "integer", - "b": "float", - "c": "boolean", - "d": "date", - "e": "datetime", - "f": "string" - } - def test_widget_client_update(self): import perspective assert perspective.is_libpsp() is False @@ -369,4 +347,4 @@ def test_widget_load_column_pivots_client(self): assert hasattr(widget, "table") is False assert widget.columns == ['value'] assert widget.column_pivots == ['first', 'second', 'third'] - assert widget.row_pivots == ['index'] \ No newline at end of file + assert widget.row_pivots == ['index'] diff --git a/python/perspective/perspective/tests/conftest.py b/python/perspective/perspective/tests/conftest.py index 9a42d231c0..3df808dc46 100644 --- a/python/perspective/perspective/tests/conftest.py +++ b/python/perspective/perspective/tests/conftest.py @@ -6,12 +6,12 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six import time +from datetime import datetime + import numpy as np import pandas as pd import pyarrow as pa -from datetime import datetime from pytest import fixture @@ -137,15 +137,9 @@ def to_timestamp(obj): '''Return an integer timestamp based on a date/datetime object.''' classname = obj.__class__.__name__ if classname == "date": - if six.PY2: - return int((time.mktime(obj.timetuple()) / 1000000.0)) - else: - return datetime(obj.year, obj.month, obj.day).timestamp() + return datetime(obj.year, obj.month, obj.day).timestamp() elif classname == "datetime": - if six.PY2: - return int((time.mktime(obj.timetuple()) + obj.microsecond / 1000000.0)) - else: - return obj.timestamp() + return obj.timestamp() else: return -1 diff --git a/python/perspective/perspective/tests/core/test_async.py b/python/perspective/perspective/tests/core/test_async.py index 56af7f3cc1..72475ac7eb 100644 --- a/python/perspective/perspective/tests/core/test_async.py +++ b/python/perspective/perspective/tests/core/test_async.py @@ -6,15 +6,14 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -import random -import six -import tornado.ioloop import queue +import random import threading - from functools import partial + +import tornado.ioloop +from perspective import PerspectiveError, PerspectiveManager, Table from pytest import raises -from perspective import Table, PerspectiveManager, PerspectiveError def syncify(f): @@ -56,10 +55,7 @@ def teardown_class(cls): @classmethod def loop_is_running(cls): - if six.PY2: - return cls.loop._running - else: - return cls.loop.asyncio_loop.is_running() + return cls.loop.asyncio_loop.is_running() def test_async_queue_process(self): tbl = Table({"a": int, "b": float, "c": str}) diff --git a/python/perspective/perspective/tests/table/test_remove.py b/python/perspective/perspective/tests/table/test_remove.py index 40bab3fff5..58d638955f 100644 --- a/python/perspective/perspective/tests/table/test_remove.py +++ b/python/perspective/perspective/tests/table/test_remove.py @@ -1,4 +1,4 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # diff --git a/python/perspective/perspective/tests/table/test_table.py b/python/perspective/perspective/tests/table/test_table.py index ad5da03127..5b27477565 100644 --- a/python/perspective/perspective/tests/table/test_table.py +++ b/python/perspective/perspective/tests/table/test_table.py @@ -1,4 +1,4 @@ -# ***************************************************************************** +################################################################################ # -*- coding: utf-8 -*- # # Copyright (c) 2019, the Perspective Authors. @@ -6,14 +6,14 @@ # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six import sys from datetime import date, datetime -from pytest import raises -from perspective.table import Table + from perspective.core.exception import PerspectiveError +from perspective.table import Table from perspective.table._state import _PerspectiveStateManager -from perspective.table.libbinding import t_filter_op, PerspectiveCppError +from perspective.table.libbinding import PerspectiveCppError, t_filter_op +from pytest import raises class TestTable(object): @@ -121,72 +121,6 @@ def test_table_int_column_names(self): with raises(PerspectiveError): Table(data) - def test_table_int_overflow(self): - if six.PY2: - maxint = sys.maxint + 1 - # overflows into float - data = {"a": [i for i in range(100)] + [maxint, maxint, maxint]} - tbl = Table(data) - # two promotions later - assert tbl.schema() == { - "a": float - } - - def test_table_long(self): - if six.PY2: - # don't overflow in this test - data = [long(100), long(200), long(300)] # noqa: F821 - tbl = Table({ - "a": data - }) - assert tbl.schema() == { - "a": int - } - assert tbl.view().to_dict()["a"] == [int(d) for d in data] - - def test_table_long_overflow(self): - if six.PY2: - maxint = sys.maxint - # don't overflow in this test - data = [maxint, maxint + 1, maxint + 2] - tbl = Table({ - "a": data - }) - assert tbl.schema() == { - "a": float - } - assert tbl.view().to_dict()["a"] == [float(d) for d in data] - - def test_table_int_to_long(self): - if six.PY2: - # don't overflow in this test - data = [int(100), int(200), int(300)] - tbl = Table({ - "a": long # noqa: F821 - }) - assert tbl.schema() == { - "a": int - } - tbl.update({ - "a": data - }) - assert tbl.view().to_dict()["a"] == data - - def test_table_float_to_long(self): - if six.PY2: - # don't overflow in this test - data = [1.5, 2.5, 3.5] # noqa: F821 - tbl = Table({ - "a": long # noqa: F821 - }) - assert tbl.schema() == { - "a": int - } - tbl.update({ - "a": data - }) - assert tbl.view().to_dict()["a"] == [1, 2, 3] - def test_table_nones(self): none_data = [{"a": 1, "b": None}, {"a": None, "b": 2}] tbl = Table(none_data) @@ -247,10 +181,7 @@ def test_table_str_with_escape(self): assert tbl.view().to_records() == str_data def test_table_str_unicode(self): - if six.PY2: - str_data = [{"a": u"ȀȁȀȃȀȁȀȃȀȁȀȃȀȁȀȃ", "b": u"ЖДфйЖДфйЖДфйЖДфй"}] - else: - str_data = [{"a": "ȀȁȀȃȀȁȀȃȀȁȀȃȀȁȀȃ", "b": "ЖДфйЖДфйЖДфйЖДфй"}] + str_data = [{"a": "ȀȁȀȃȀȁȀȃȀȁȀȃȀȁȀȃ", "b": "ЖДфйЖДфйЖДфйЖДфй"}] tbl = Table(str_data) assert tbl.size() == 1 assert tbl.schema() == { @@ -446,30 +377,6 @@ def test_table_symmetric_string_schema(self): assert tbl2.schema(as_string=True) == schema - def test_table_long_schema(self): - if six.PY2: - schema = { - "a": long, # noqa: F821 - "b": int - } - tbl = Table(schema) - assert tbl.schema() == { - "a": int, - "b": int - } - - def test_table_unicode_schema(self): - if six.PY2: - schema = { - "a": unicode, # noqa: F821 - "b": int - } - tbl = Table(schema) - assert tbl.schema() == { - "a": str, - "b": int - } - # is_valid_filter def test_table_is_valid_filter_str(self): diff --git a/python/perspective/perspective/tests/table/test_table_arrow.py b/python/perspective/perspective/tests/table/test_table_arrow.py index 7f34dd8d9a..632546258d 100644 --- a/python/perspective/perspective/tests/table/test_table_arrow.py +++ b/python/perspective/perspective/tests/table/test_table_arrow.py @@ -1,4 +1,4 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # diff --git a/python/perspective/perspective/tests/table/test_table_datetime.py b/python/perspective/perspective/tests/table/test_table_datetime.py index 9a924ef7a1..099d5e30d1 100644 --- a/python/perspective/perspective/tests/table/test_table_datetime.py +++ b/python/perspective/perspective/tests/table/test_table_datetime.py @@ -1,4 +1,4 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # diff --git a/python/perspective/perspective/tests/table/test_table_infer.py b/python/perspective/perspective/tests/table/test_table_infer.py index 18b2969cec..3377558e8e 100644 --- a/python/perspective/perspective/tests/table/test_table_infer.py +++ b/python/perspective/perspective/tests/table/test_table_infer.py @@ -1,15 +1,15 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six -from pytest import mark -from perspective.table import Table from datetime import date, datetime +from perspective.table import Table +from pytest import mark + class TestTableInfer(object): @@ -60,20 +60,6 @@ def test_table_bool_infer_str_all_formats_from_schema(self): "b": [False, False, False, False, False] } - def test_table_promote_float(self): - if six.PY2: - data = {"a": [1.5, 2.5, 3.5, 4.5, "abc"]} - tbl = Table(data) - assert tbl.schema() == {"a": str} - assert tbl.view().to_dict() == {"a": ["1.5", "2.5", "3.5", "4.5", "abc"]} - - def test_table_promote_float_py2(self): - if six.PY2: - data = {"a": [1, 2, 3, 4, 2147483648]} - tbl = Table(data) - assert tbl.schema() == {"a": float} - assert tbl.view().to_dict() == {"a": [1.0, 2.0, 3.0, 4.0, 2147483648.0]} - def test_table_infer_bool(self): data = {"a": [None, None, None, None, True, True, True]} tbl = Table(data) diff --git a/python/perspective/perspective/tests/table/test_table_limit.py b/python/perspective/perspective/tests/table/test_table_limit.py index 16a4ffa80d..4f1a8a13f0 100644 --- a/python/perspective/perspective/tests/table/test_table_limit.py +++ b/python/perspective/perspective/tests/table/test_table_limit.py @@ -1,4 +1,4 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # @@ -6,11 +6,10 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -import perspective -import six +from datetime import date, datetime +import perspective from pytest import mark -from datetime import date, datetime class TestTableInfer(object): diff --git a/python/perspective/perspective/tests/table/test_table_numpy.py b/python/perspective/perspective/tests/table/test_table_numpy.py index ebb123b80a..142ed48a3b 100644 --- a/python/perspective/perspective/tests/table/test_table_numpy.py +++ b/python/perspective/perspective/tests/table/test_table_numpy.py @@ -1,4 +1,4 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # @@ -6,13 +6,13 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six -import pandas as pd +from datetime import date, datetime + import numpy as np -from pytest import raises +import pandas as pd from perspective import PerspectiveError from perspective.table import Table -from datetime import date, datetime +from pytest import raises class TestTableNumpy(object): @@ -94,18 +94,6 @@ def test_table_int64(self): "b": [4, 5, 6] } - def test_table_long_numpy(self): - if six.PY2: - data = {"a": np.array([1, 2, 3], dtype=long)} # noqa: F821 - tbl = Table(data) - assert tbl.schema() == { - "a": int - } - assert tbl.size() == 3 - assert tbl.view().to_dict() == { - "a": [1, 2, 3] - } - def test_table_float(self): data = {"a": np.array([1.1, 2.2]), "b": np.array([3.3, 4.4])} tbl = Table(data) @@ -198,8 +186,6 @@ def test_table_str_object(self): def test_table_str_dtype(self): dtype = "U3" - if six.PY2: - dtype = "|S3" data = {"a": np.array(["abc", "def"], dtype=dtype), "b": np.array(["hij", "klm"], dtype=dtype)} tbl = Table(data) assert tbl.size() == 2 @@ -208,19 +194,6 @@ def test_table_str_dtype(self): "b": ["hij", "klm"] } - def test_table_unicode_py2(self): - if six.PY2: - data = { - "a": np.array([unicode("abc"), unicode("def")]), # noqa: F821 - "b": np.array([unicode("hij"), unicode("klm")]) # noqa: F821 - } - tbl = Table(data) - assert tbl.size() == 2 - assert tbl.view().to_dict() == { - "a": ["abc", "def"], - "b": ["hij", "klm"] - } - # date and datetime def test_table_date(self): @@ -623,39 +596,6 @@ def test_table_numpy_from_schema_int(self): table.update(df) assert table.view().to_dict()["a"] == [1, None, 2, None, 3, 4] - def test_table_numpy_from_schema_long(self): - if six.PY2: - df = { - "a": np.array([1, None, 2, None, 3, 4]) - } - table = Table({ - "a": long # noqa: F821 - }) - table.update(df) - assert table.view().to_dict()["a"] == [1, None, 2, None, 3, 4] - - def test_table_numpy_from_schema_int_to_long(self): - if six.PY2: - df = { - "a": np.array([1, 2, 3, 4], dtype="int64") - } - table = Table({ - "a": long # noqa: F821 - }) - table.update(df) - assert table.view().to_dict()["a"] == [1, 2, 3, 4] - - def test_table_numpy_from_schema_float_to_long(self): - if six.PY2: - df = { - "a": np.array([1, None, 2, None, 3, 4], dtype="float64") - } - table = Table({ - "a": long # noqa: F821 - }) - table.update(df) - assert table.view().to_dict()["a"] == [1, None, 2, None, 3, 4] - def test_table_numpy_from_schema_bool(self): data = [True, False, True, False] df = { @@ -1033,8 +973,6 @@ def test_table_recarray_str(self): def test_table_recarray_str_dtype(self): dtype = "U7" - if six.PY2: - dtype = "|S7" table = Table(np.array([("string1", "string2"), ("string3", "string4")], dtype=[('x', dtype), ('y', dtype)]).view(np.recarray)) assert table.schema() == { "x": str, diff --git a/python/perspective/perspective/tests/table/test_table_object.py b/python/perspective/perspective/tests/table/test_table_object.py index fdaf714a56..de1bd59123 100644 --- a/python/perspective/perspective/tests/table/test_table_object.py +++ b/python/perspective/perspective/tests/table/test_table_object.py @@ -1,15 +1,16 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six import sys +from datetime import date, datetime from random import randint + from perspective.table import Table -from datetime import date, datetime + class CustomObjectBlank(object): pass diff --git a/python/perspective/perspective/tests/table/test_table_pandas.py b/python/perspective/perspective/tests/table/test_table_pandas.py index e707776fbf..0ed6237a9f 100644 --- a/python/perspective/perspective/tests/table/test_table_pandas.py +++ b/python/perspective/perspective/tests/table/test_table_pandas.py @@ -1,16 +1,17 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six -from io import StringIO from datetime import date, datetime +from io import StringIO + import numpy as np import pandas as pd from perspective.table import Table + from ..common import superstore @@ -277,18 +278,6 @@ def test_table_pandas_from_schema_int(self): table.update(df) assert table.view().to_dict()["a"] == data - def test_table_pandas_from_schema_long(self): - if six.PY2: - data = [None, 1, None, 2, None, 3, 4] - df = pd.DataFrame({ - "a": data - }) - table = Table({ - "a": long # noqa: F821 - }) - table.update(df) - assert table.view().to_dict()["a"] == data - def test_table_pandas_from_schema_bool(self): data = [True, False, True, False] df = pd.DataFrame({ @@ -849,8 +838,6 @@ def test_table_read_nan_datetime_milliseconds(self): def test_table_pandas_correct_csv_nan_end(self): s = "str,int\n,1\n,2\nabc,3" - if six.PY2: - s = unicode(s) csv = StringIO(s) data = pd.read_csv(csv) tbl = Table(data) @@ -868,8 +855,6 @@ def test_table_pandas_correct_csv_nan_end(self): def test_table_pandas_correct_csv_nan_intermittent(self): s = "str,float\nabc,\n,2\nghi," - if six.PY2: - s = unicode(s) csv = StringIO(s) data = pd.read_csv(csv) tbl = Table(data) diff --git a/python/perspective/perspective/tests/table/test_to_arrow.py b/python/perspective/perspective/tests/table/test_to_arrow.py index 5aa7cf5524..7b0371c3f9 100644 --- a/python/perspective/perspective/tests/table/test_to_arrow.py +++ b/python/perspective/perspective/tests/table/test_to_arrow.py @@ -152,12 +152,13 @@ def test_to_arrow_one_symmetric(self): arrow = view.to_arrow() tbl2 = Table(arrow) assert tbl2.schema() == { + 'a (Group by 1)': int, "a": int, "b": int, "c": int } d = view.to_dict() - d.pop("__ROW_PATH__") + d['a (Group by 1)'] = [x[0] if len(x) > 0 else None for x in d.pop("__ROW_PATH__")] assert tbl2.view().to_dict() == d def test_to_arrow_two_symmetric(self): @@ -171,6 +172,7 @@ def test_to_arrow_two_symmetric(self): arrow = view.to_arrow() tbl2 = Table(arrow) assert tbl2.schema() == { + 'a (Group by 1)': int, "hello|a": int, "hello|b": int, "hello|c": int, @@ -185,7 +187,7 @@ def test_to_arrow_two_symmetric(self): "world2|c": int, } d = view.to_dict() - d.pop("__ROW_PATH__") + d['a (Group by 1)'] = [x[0] if len(x) > 0 else None for x in d.pop("__ROW_PATH__")] assert tbl2.view().to_dict() == d def test_to_arrow_column_only_symmetric(self): @@ -512,5 +514,6 @@ def test_to_arrow_one_mean(self): result = view2.to_columns() assert result == { + 'b (Group by 1)': [None, 'a', 'b'], "a": [2.5, 1.5, 3.5] } \ No newline at end of file diff --git a/python/perspective/perspective/tests/table/test_to_format.py b/python/perspective/perspective/tests/table/test_to_format.py index be3f869dd4..170b70ac9c 100644 --- a/python/perspective/perspective/tests/table/test_to_format.py +++ b/python/perspective/perspective/tests/table/test_to_format.py @@ -1,19 +1,20 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six import os +from datetime import date, datetime +from io import StringIO + import numpy as np import pandas as pd import pytz -from io import StringIO -from datetime import date, datetime from perspective.table import Table from pytest import mark + IS_WIN = os.name == 'nt' @@ -376,13 +377,7 @@ def test_to_pandas_df_simple(self): tbl = Table(df) view = tbl.view() df2 = view.to_df() - if six.PY2: - # order not guaranteed if columns are inferred using aggregate dict keys - for col in df2.columns: - assert col in ["index", "a", "b"] - else: - assert np.array_equal(df2.columns, pd.Index(["index", "a", "b"], dtype=object)) - + assert np.array_equal(df2.columns, pd.Index(["index", "a", "b"], dtype=object)) assert np.array_equal(df2["a"].values, df["a"].values) assert np.array_equal(df2["b"].values, df["b"].values) @@ -393,12 +388,7 @@ def test_to_pandas_df_simple_series(self): tbl = Table(inp) view = tbl.view() df2 = view.to_df() - if six.PY2: - # order not guaranteed if columns are inferred using aggregate dict keys - for col in df2.columns: - assert col in ["index", "a"] - else: - assert np.array_equal(df2.columns, pd.Index(["index", "a"], dtype=object)) + assert np.array_equal(df2.columns, pd.Index(["index", "a"], dtype=object)) assert np.array_equal(df2["a"].values, df["a"].values) # start_row/end_row @@ -782,13 +772,7 @@ def test_to_records_start_col_end_col(self): end_col=2 ) # start_col and end_col access columns at that index - dict key order not guaranteed in python2 - if six.PY2: - # in this test, column c comes before b - assert records == [{"c": 3}, {"c": 5}] - # assert that in the general dataset, just to see if it holds true - assert view.to_records() == [{"a": 1, "c": 3, "b": 2}, {"a": 3, "c": 5, "b": 4}] - else: - assert records == [{"b": 2}, {"b": 4}] + assert records == [{"b": 2}, {"b": 4}] def test_to_records_start_col_end_col_equiv(self): data = [{"a": 1, "b": 2, "c": 3}, {"a": 3, "b": 4, "c": 5}] @@ -844,13 +828,7 @@ def test_to_records_floor_start_col_ceil_end_col(self): end_col=1.5 ) # start_col and end_col access columns at that index - dict key order not guaranteed in python2 - if six.PY2: - # in this test, column c comes before b - assert records == [{"c": 3}, {"c": 5}] - # assert that in the general dataset, just to see if it holds true - assert view.to_records() == [{"a": 1, "c": 3, "b": 2}, {"a": 3, "c": 5, "b": 4}] - else: - assert records == [{"b": 2}, {"b": 4}] + assert records == [{"b": 2}, {"b": 4}] def test_to_dict_start_col_end_col(self): data = [{"a": 1, "b": 2, "c": 3, "d": 4}, {"a": 3, "b": 4, "c": 5, "d": 6}] @@ -869,39 +847,22 @@ def test_to_dict_start_col_end_col(self): def test_to_csv_symmetric(self): csv = "a,b\n1,2\n3,4" - if six.PY2: - csv = unicode(csv) df = pd.read_csv(StringIO(csv)) tbl = Table(df) view = tbl.view() - if six.PY2: - if IS_WIN: - assert view.to_csv() == ",a,b,index\r\n0,1,2,0\r\n1,3,4,1\r\n" - else: - assert view.to_csv() == ",a,b,index\n0,1,2,0\n1,3,4,1\n" - else: - if IS_WIN: - assert view.to_csv() == ",index,a,b\r\n0,0,1,2\r\n1,1,3,4\r\n" - else: - assert view.to_csv() == ",index,a,b\n0,0,1,2\n1,1,3,4\n" + assert view.to_csv() == '"index","a","b"\n0,1,2\n1,3,4\n' def test_to_csv_int(self): data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}] tbl = Table(data) view = tbl.view() - if IS_WIN: - assert view.to_csv() == ",a,b\r\n0,1,2\r\n1,3,4\r\n" - else: - assert view.to_csv() == ",a,b\n0,1,2\n1,3,4\n" + assert view.to_csv() == '"a","b"\n1,2\n3,4\n' def test_to_csv_float(self): data = [{"a": 1.5, "b": 2.5}, {"a": 3.5, "b": 4.5}] tbl = Table(data) view = tbl.view() - if IS_WIN: - assert view.to_csv() == ",a,b\r\n0,1.5,2.5\r\n1,3.5,4.5\r\n" - else: - assert view.to_csv() == ",a,b\n0,1.5,2.5\n1,3.5,4.5\n" + assert view.to_csv() == '"a","b"\n1.5,2.5\n3.5,4.5\n' def test_to_csv_date(self): today = date.today() @@ -910,23 +871,7 @@ def test_to_csv_date(self): tbl = Table(data) assert tbl.schema()["a"] == date view = tbl.view() - if IS_WIN: - assert view.to_csv() == ",a,b\r\n0,{},2\r\n1,{},4\r\n".format(dt_str, dt_str) - else: - assert view.to_csv() == ",a,b\n0,{},2\n1,{},4\n".format(dt_str, dt_str) - - @mark.skip(reason="pandas does not support date-specific formated in the date_format kwarg") - def test_to_csv_date_formatted(self): - today = date.today() - dt_str = today.strftime("%Y/%m/%d") - data = [{"a": today, "b": 2}, {"a": today, "b": 4}] - tbl = Table(data) - assert tbl.schema()["a"] == date - view = tbl.view() - if IS_WIN: - assert view.to_csv(formatted=True) == ",a,b\r\n0,{},2\r\n1,{},4\r\n".format(dt_str, dt_str) - else: - assert view.to_csv(formatted=True) == ",a,b\n0,{},2\n1,{},4\n".format(dt_str, dt_str) + assert view.to_csv() == '"a","b"\n{},2\n{},4\n'.format(dt_str, dt_str) def test_to_csv_datetime(self): dt = datetime(2019, 3, 15, 20, 30, 59, 6000) @@ -934,75 +879,43 @@ def test_to_csv_datetime(self): data = [{"a": dt, "b": 2}, {"a": dt, "b": 4}] tbl = Table(data) view = tbl.view() - if IS_WIN: - assert view.to_csv() == ",a,b\r\n0,{},2\r\n1,{},4\r\n".format(dt_str, dt_str) - else: - assert view.to_csv() == ",a,b\n0,{},2\n1,{},4\n".format(dt_str, dt_str) - - def test_to_csv_datetime_formatted(self): - dt = datetime(2019, 3, 15, 20, 30, 59, 6000) - dt_str = dt.strftime("%Y/%m/%d %H:%M:%S") - data = [{"a": dt, "b": 2}, {"a": dt, "b": 4}] - tbl = Table(data) - view = tbl.view() - if IS_WIN: - assert view.to_csv(formatted=True) == ",a,b\r\n0,{},2\r\n1,{},4\r\n".format(dt_str, dt_str) - else: - assert view.to_csv(formatted=True) == ",a,b\n0,{},2\n1,{},4\n".format(dt_str, dt_str) + assert view.to_csv() == '"a","b"\n{},2\n{},4\n'.format(dt_str, dt_str) def test_to_csv_bool(self): data = [{"a": True, "b": False}, {"a": True, "b": False}] tbl = Table(data) view = tbl.view() - if IS_WIN: - assert view.to_csv() == ",a,b\r\n0,True,False\r\n1,True,False\r\n" - else: - assert view.to_csv() == ",a,b\n0,True,False\n1,True,False\n" + assert view.to_csv() == '"a","b"\ntrue,false\ntrue,false\n' def test_to_csv_string(self): data = [{"a": "string1", "b": "string2"}, {"a": "string3", "b": "string4"}] tbl = Table(data) view = tbl.view() - if IS_WIN: - assert view.to_csv() == ",a,b\r\n0,string1,string2\r\n1,string3,string4\r\n" - else: - assert view.to_csv() == ",a,b\n0,string1,string2\n1,string3,string4\n" + assert view.to_csv() == '"a","b"\n"string1","string2"\n"string3","string4"\n' def test_to_csv_none(self): data = [{"a": None, "b": None}, {"a": None, "b": None}] tbl = Table(data) view = tbl.view() - if IS_WIN: - assert view.to_csv() == ",a,b\r\n0,,\r\n1,,\r\n" - else: - assert view.to_csv() == ",a,b\n0,,\n1,,\n" + assert view.to_csv() == '"a","b"\n,\n,\n' def test_to_csv_custom_rows(self): data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}] tbl = Table(data) view = tbl.view() - if IS_WIN: - assert view.to_csv(start_row=1) == ",a,b\r\n0,3,4\r\n" - else: - assert view.to_csv(start_row=1) == ",a,b\n0,3,4\n" + assert view.to_csv(start_row=1) == '"a","b"\n3,4\n' def test_to_csv_custom_cols(self): data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}] tbl = Table(data) view = tbl.view() - if IS_WIN: - assert view.to_csv(start_col=1) == ",b\r\n0,2\r\n1,4\r\n" - else: - assert view.to_csv(start_col=1) == ",b\n0,2\n1,4\n" + assert view.to_csv(start_col=1) == '"b"\n2\n4\n' def test_to_csv_custom_rows_cols(self): data = [{"a": 1, "b": 2}, {"a": 3, "b": 4}] tbl = Table(data) view = tbl.view() - if IS_WIN: - assert view.to_csv(start_row=1, start_col=1) == ",b\r\n0,4\r\n" - else: - assert view.to_csv(start_row=1, start_col=1) == ",b\n0,4\n" + assert view.to_csv(start_row=1, start_col=1) == '"b"\n4\n' def test_to_csv_one(self): data = [{"a": 1, "b": 2}, {"a": 1, "b": 2}] @@ -1010,16 +923,7 @@ def test_to_csv_one(self): view = tbl.view( row_pivots=["a"] ) - if six.PY2: - if IS_WIN: - assert view.to_csv() == ",__ROW_PATH__,a,b\r\n0,[],2,4\r\n1,[1],2,4\r\n" - else: - assert view.to_csv() == ",__ROW_PATH__,a,b\n0,[],2,4\n1,[1],2,4\n" - else: - if IS_WIN: - assert view.to_csv() == ",__ROW_PATH__,a,b\r\n0,[],2,4\r\n1,[1],2,4\r\n" - else: - assert view.to_csv() == ",__ROW_PATH__,a,b\n0,[],2,4\n1,[1],2,4\n" + assert view.to_csv() == '"a (Group by 1)","a","b"\n,2,4\n1,2,4\n' def test_to_csv_two(self): data = [{"a": 1, "b": 2}, {"a": 1, "b": 2}] @@ -1028,16 +932,7 @@ def test_to_csv_two(self): row_pivots=["a"], column_pivots=["b"] ) - if six.PY2: - if IS_WIN: - assert view.to_csv() == ",2|a,2|b,__ROW_PATH__\r\n0,2,4,[]\r\n1,2,4,[1]\r\n" - else: - assert view.to_csv() == ",2|a,2|b,__ROW_PATH__\n0,2,4,[]\n1,2,4,[1]\n" - else: - if IS_WIN: - assert view.to_csv() == ",__ROW_PATH__,2|a,2|b\r\n0,[],2,4\r\n1,[1],2,4\r\n" - else: - assert view.to_csv() == ",__ROW_PATH__,2|a,2|b\n0,[],2,4\n1,[1],2,4\n" + assert view.to_csv() == '"a (Group by 1)","2|a","2|b"\n,2,4\n1,2,4\n' def test_to_csv_column_only(self): data = [{"a": 1, "b": 2}, {"a": 1, "b": 2}] @@ -1045,10 +940,7 @@ def test_to_csv_column_only(self): view = tbl.view( column_pivots=["b"] ) - if IS_WIN: - assert view.to_csv() == ",2|a,2|b\r\n0,1,2\r\n1,1,2\r\n" - else: - assert view.to_csv() == ",2|a,2|b\n0,1,2\n1,1,2\n" + assert view.to_csv() == '"2|a","2|b"\n1,2\n1,2\n' def test_to_csv_one_no_columns(self): data = [{"a": 1, "b": 2}, {"a": 1, "b": 2}] @@ -1057,16 +949,7 @@ def test_to_csv_one_no_columns(self): row_pivots=["a"], columns=[] ) - if six.PY2: - if IS_WIN: - assert view.to_csv() == ",__ROW_PATH__\r\n0,[]\r\n1,[1]\r\n" - else: - assert view.to_csv() == ",__ROW_PATH__\n0,[]\n1,[1]\n" - else: - if IS_WIN: - assert view.to_csv() == ",__ROW_PATH__\r\n0,[]\r\n1,[1]\r\n" - else: - assert view.to_csv() == ",__ROW_PATH__\n0,[]\n1,[1]\n" + assert view.to_csv() == '"a (Group by 1)"\n\n1\n' def test_to_csv_two_no_columns(self): data = [{"a": 1, "b": 2}, {"a": 1, "b": 2}] @@ -1076,16 +959,7 @@ def test_to_csv_two_no_columns(self): column_pivots=["b"], columns=[] ) - if six.PY2: - if IS_WIN: - assert view.to_csv() == ",__ROW_PATH__\r\n0,[]\r\n1,[1]\r\n" - else: - assert view.to_csv() == ",__ROW_PATH__\n0,[]\n1,[1]\n" - else: - if IS_WIN: - assert view.to_csv() == ",__ROW_PATH__\r\n0,[]\r\n1,[1]\r\n" - else: - assert view.to_csv() == ",__ROW_PATH__\n0,[]\n1,[1]\n" + assert view.to_csv() == '"a (Group by 1)"\n\n1\n' def test_to_csv_column_only_no_columns(self): data = [{"a": 1, "b": 2}, {"a": 1, "b": 2}] @@ -1095,10 +969,7 @@ def test_to_csv_column_only_no_columns(self): columns=[] ) - if IS_WIN: - assert view.to_csv() == '""\r\n' - else: - assert view.to_csv() == '""\n' + assert view.to_csv() == '' # implicit index diff --git a/python/perspective/perspective/tests/table/test_update.py b/python/perspective/perspective/tests/table/test_update.py index f946bbd9c3..c48364c696 100644 --- a/python/perspective/perspective/tests/table/test_update.py +++ b/python/perspective/perspective/tests/table/test_update.py @@ -1,4 +1,4 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # diff --git a/python/perspective/perspective/tests/table/test_update_arrow.py b/python/perspective/perspective/tests/table/test_update_arrow.py index 0bc5562209..7742e6e928 100644 --- a/python/perspective/perspective/tests/table/test_update_arrow.py +++ b/python/perspective/perspective/tests/table/test_update_arrow.py @@ -1,4 +1,4 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # diff --git a/python/perspective/perspective/tests/table/test_update_numpy.py b/python/perspective/perspective/tests/table/test_update_numpy.py index 2cb2ee4c62..4ccc3b35e5 100644 --- a/python/perspective/perspective/tests/table/test_update_numpy.py +++ b/python/perspective/perspective/tests/table/test_update_numpy.py @@ -1,4 +1,4 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # diff --git a/python/perspective/perspective/tests/table/test_update_pandas.py b/python/perspective/perspective/tests/table/test_update_pandas.py index 2ae032f60c..81c71adc51 100644 --- a/python/perspective/perspective/tests/table/test_update_pandas.py +++ b/python/perspective/perspective/tests/table/test_update_pandas.py @@ -1,4 +1,4 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # diff --git a/python/perspective/perspective/tests/table/test_view_expression.py b/python/perspective/perspective/tests/table/test_view_expression.py index 037c9fee0e..b2c0913193 100644 --- a/python/perspective/perspective/tests/table/test_view_expression.py +++ b/python/perspective/perspective/tests/table/test_view_expression.py @@ -1,4 +1,4 @@ -# ***************************************************************************** +################################################################################ # # Copyright (c) 2019, the Perspective Authors. # diff --git a/python/perspective/perspective/tests/widget/test_widget.py b/python/perspective/perspective/tests/widget/test_widget.py index f1d20f3f74..bc00031a99 100644 --- a/python/perspective/perspective/tests/widget/test_widget.py +++ b/python/perspective/perspective/tests/widget/test_widget.py @@ -5,13 +5,13 @@ # This file is part of the Perspective library, distributed under the terms of # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six -import numpy as np from datetime import date, datetime from functools import partial from types import MethodType -from pytest import raises + +import numpy as np from perspective import PerspectiveError, PerspectiveWidget, Table +from pytest import raises def mock_post(self, msg, msg_id=None, assert_msg=None): diff --git a/python/perspective/perspective/tornado_handler/tornado_client.py b/python/perspective/perspective/tornado_handler/tornado_client.py index 09d7a3ef00..813c4fcc88 100644 --- a/python/perspective/perspective/tornado_handler/tornado_client.py +++ b/python/perspective/perspective/tornado_handler/tornado_client.py @@ -7,9 +7,10 @@ # import json -import six + from tornado import gen, ioloop from tornado.websocket import websocket_connect + from ..client import PerspectiveClient from ..manager.manager_internal import DateTimeEncoder @@ -101,7 +102,7 @@ def on_message(self, msg): self._pending_binary_length = None self._pending_port_id = None self._full_binary = b"" - elif isinstance(msg, six.string_types): + elif isinstance(msg, str): msg = json.loads(msg) if msg.get("binary_length"): diff --git a/python/perspective/perspective/viewer/validate.py b/python/perspective/perspective/viewer/validate.py index 2007eeabbc..28cc664d90 100644 --- a/python/perspective/perspective/viewer/validate.py +++ b/python/perspective/perspective/viewer/validate.py @@ -6,16 +6,16 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -from six import iteritems, string_types from datetime import datetime + +from ..core import ALL_FILTERS, Aggregate, Plugin, Sort from ..core.exception import PerspectiveError -from ..core import Aggregate, Plugin, ALL_FILTERS, Sort def validate_plugin(plugin): if isinstance(plugin, Plugin): return plugin.value - elif isinstance(plugin, string_types): + elif isinstance(plugin, str): if plugin not in Plugin.options(): raise PerspectiveError("Unrecognized `plugin`: {0}".format(plugin)) return plugin @@ -28,7 +28,7 @@ def validate_plugin(plugin): def validate_columns(columns): if columns is None: return [] - elif isinstance(columns, string_types): + elif isinstance(columns, str): columns = [columns] if isinstance(columns, list): @@ -40,7 +40,7 @@ def validate_columns(columns): def _validate_pivots(pivots): if pivots is None: return [] - elif isinstance(pivots, string_types): + elif isinstance(pivots, str): pivots = [pivots] if isinstance(pivots, list): @@ -61,10 +61,10 @@ def validate_aggregates(aggregates): if aggregates is None: return {} elif isinstance(aggregates, dict): - for k, v in iteritems(aggregates): + for k, v in aggregates.items(): if isinstance(v, Aggregate): aggregates[k] = v.value - elif isinstance(v, string_types): + elif isinstance(v, str): if v not in Aggregate.options(): raise PerspectiveError("Unrecognized aggregate: %s", v) elif isinstance(v, list): @@ -89,7 +89,7 @@ def validate_aggregates(aggregates): def validate_sort(sort): if sort is None: return [] - elif isinstance(sort, string_types): + elif isinstance(sort, str): sort = [sort] if isinstance(sort, list): @@ -99,7 +99,7 @@ def validate_sort(sort): for col, s in sort: if isinstance(s, Sort): s = s.value - elif not isinstance(s, string_types) or s not in Sort.options(): + elif not isinstance(s, str) or s not in Sort.options(): raise PerspectiveError("Unrecognized sort direction: %s", s) ret.append([col, s]) return ret @@ -153,13 +153,13 @@ def validate_expressions(expressions): if expressions is None: return [] - if isinstance(expressions, string_types): + if isinstance(expressions, str): # wrap in a list and return return [expressions] if isinstance(expressions, list): for expr in expressions: - if not isinstance(expr, string_types): + if not isinstance(expr, str): raise PerspectiveError( "Cannot parse non-string expression: {}".format(str(type(expr))) ) diff --git a/python/perspective/perspective/viewer/viewer.py b/python/perspective/perspective/viewer/viewer.py index 1ed21371c8..3808f23052 100644 --- a/python/perspective/perspective/viewer/viewer.py +++ b/python/perspective/perspective/viewer/viewer.py @@ -6,25 +6,25 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six from random import random + +from ..libpsp import is_libpsp from .validate import ( - validate_plugin, - validate_columns, - validate_row_pivots, - validate_column_pivots, validate_aggregates, - validate_sort, - validate_filter, + validate_column_pivots, + validate_columns, validate_expressions, + validate_filter, + validate_plugin, validate_plugin_config, + validate_row_pivots, + validate_sort, ) from .viewer_traitlets import PerspectiveTraitlets -from ..libpsp import is_libpsp if is_libpsp(): - from ..libpsp import Table, View, PerspectiveManager from ..core.exception import PerspectiveError + from ..libpsp import PerspectiveManager, Table, View class PerspectiveViewer(PerspectiveTraitlets, object): @@ -246,7 +246,7 @@ def restore(self, **kwargs): """Restore a given set of attributes, passed as kwargs (e.g. dictionary). Symmetric with `save` so that a given viewer's configuration can be reproduced.""" - for k, v in six.iteritems(kwargs): + for k, v in kwargs.items(): if k in PerspectiveViewer.PERSISTENT_ATTRIBUTES: setattr(self, k, v) @@ -281,7 +281,7 @@ def delete(self, delete_table=True): """ if delete_table: # Delete all created views on the widget's manager instance - for view in six.itervalues(self.manager._views): + for view in self.manager._views.values(): view.delete() # Reset view cache diff --git a/python/perspective/perspective/widget/widget.py b/python/perspective/perspective/widget/widget.py index 15a966dc30..720304fc9c 100644 --- a/python/perspective/perspective/widget/widget.py +++ b/python/perspective/perspective/widget/widget.py @@ -6,22 +6,21 @@ # the Apache License 2.0. The full license can be found in the LICENSE file. # -import six -import logging -import numpy -import pandas import json - +import logging from datetime import date, datetime from functools import partial + +import numpy +import pandas from ipywidgets import DOMWidget -from traitlets import observe, Unicode +from traitlets import Unicode, observe +from ..core._version import __version__ from ..core.data import deconstruct_pandas from ..core.exception import PerspectiveError from ..libpsp import is_libpsp from ..viewer import PerspectiveViewer -from ..core._version import __version__ def _type_to_string(t): @@ -29,7 +28,7 @@ def _type_to_string(t): type. Redefine here as we can't have any dependencies on libbinding in client mode. """ - if t in six.integer_types: + if t is int: return "integer" elif t is float: return "float" @@ -39,7 +38,7 @@ def _type_to_string(t): return "date" elif t is datetime: return "datetime" - elif t is six.binary_type or t is six.text_type: + elif t is bytes or t is str: return "string" else: raise PerspectiveError( @@ -82,7 +81,7 @@ def _serialize(data): "Received {} in list dataset, expected `dict`!".format(type(row)) ) - for k in six.iterkeys(row): + for k in row.keys(): if type(row[k]) is datetime: row[k] = row[k].strftime("%Y-%m-%d %H:%M:%S.%f") elif type(row[k]) is date: @@ -91,7 +90,7 @@ def _serialize(data): elif isinstance(data, dict): formatted = data - for v in six.itervalues(data): + for v in data.values(): if isinstance(v, type): # serialize schema values to string return { @@ -105,7 +104,7 @@ def _serialize(data): } break - for column_name in six.iterkeys(formatted): + for column_name in formatted.keys(): # Replace `datetime.datetime` and `datetime.date` with string formatted[column_name] = _serialize_datetime(formatted[column_name]) @@ -120,7 +119,7 @@ def _serialize(data): columns = [data[col].tolist() for col in data.dtype.names] formatted = dict(zip(data.dtype.names, columns)) - for column_name in six.iterkeys(formatted): + for column_name in formatted.keys(): # Replace `datetime.datetime` and `datetime.date` with string formatted[column_name] = _serialize_datetime(formatted[column_name]) diff --git a/python/perspective/setup.py b/python/perspective/setup.py index 192ad4d48b..06ad0dd4a0 100644 --- a/python/perspective/setup.py +++ b/python/perspective/setup.py @@ -39,7 +39,6 @@ "numpy>=1.13.1", "pandas>=0.22.0", "python-dateutil>=2.8.0", - "six>=1.11.0", "tornado>=4.5.3", "traitlets>=4.3.2", ] diff --git a/rust/perspective-viewer/Cargo.lock b/rust/perspective-viewer/Cargo.lock index fb4895a0d6..b1ef44536a 100644 --- a/rust/perspective-viewer/Cargo.lock +++ b/rust/perspective-viewer/Cargo.lock @@ -10,9 +10,9 @@ checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" [[package]] name = "anyhow" -version = "1.0.38" +version = "1.0.52" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afddf7f520a80dbf76e6f50a35bca42a2331ef227a28b3b6dc5c2e2338d114b1" +checksum = "84450d0b4a8bd1ba4144ce8ce718fbc5d071358b1e5384bace6536b3d1f2d5b3" [[package]] name = "anymap" @@ -109,9 +109,9 @@ dependencies = [ [[package]] name = "async-std" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9f06685bad74e0570f5213741bea82158279a4103d988e57bfada11ad230341" +checksum = "f8056f1455169ab86dd47b47391e4ab0cbd25410a70e9fe675544f49bafaf952" dependencies = [ "async-channel", "async-global-executor", @@ -146,9 +146,9 @@ version = "0.1.42" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8d3a45e77e34375a7923b1e8febb049bb011f064714a8e17a1a616fef01da13d" dependencies = [ - "proc-macro2 1.0.28", - "quote 1.0.8", - "syn 1.0.74", + "proc-macro2 1.0.36", + "quote 1.0.14", + "syn 1.0.84", ] [[package]] @@ -171,19 +171,18 @@ checksum = "904dfeac50f3cdaba28fc6f57fdcddb75f49ed61346676a78c4ffe55877802fd" [[package]] name = "bincode" -version = "1.3.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" dependencies = [ - "byteorder", "serde", ] [[package]] name = "blocking" -version = "1.0.2" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9" +checksum = "046e47d4b2d391b1f6f8b407b1deb8dee56c1852ccd868becf2710f601b5f427" dependencies = [ "async-channel", "async-task", @@ -201,33 +200,33 @@ checksum = "cfa8873f51c92e232f9bac4065cddef41b714152812bfc5f7672ba16d6ef8cd9" [[package]] name = "bumpalo" -version = "3.6.0" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "099e596ef14349721d9016f6b80dd3419ea1bf289ab9b44df8e4dfd3a005d5d9" +checksum = "8f1e260c3a9040a7c19a12468758f4c16f31a81a1fe087482be9570ec864bb6c" [[package]] name = "byteorder" -version = "1.4.2" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ae44d1a3d5a19df61dd0c8beb138458ac2a53a7ac09eba97d55592540004306b" +checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610" [[package]] name = "bytes" -version = "1.0.1" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" +checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8" [[package]] name = "cache-padded" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" +checksum = "c1db59621ec70f09c5e9b597b220c7a2b43611f4710dc03ceb8748637775692c" [[package]] name = "cc" -version = "1.0.68" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a72c244c1ff497a746a7e1fb3d14bd08420ecda70c8f25c7112f2781652d787" +checksum = "22a9137b95ea06864e018375b72adfb7db6e6f68cfc8df5a04d00288050485ee" [[package]] name = "cfg-if" @@ -265,19 +264,19 @@ dependencies = [ [[package]] name = "console_error_panic_hook" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8d976903543e0c48546a91908f21588a680a8c8f984df9a5d69feccb2b2a211" +checksum = "a06aeb73f470f66dcdbf7223caeebb85984942f22f1adb2a088cf9668146bbbc" dependencies = [ - "cfg-if 0.1.10", + "cfg-if 1.0.0", "wasm-bindgen", ] [[package]] name = "crc32fast" -version = "1.2.1" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +checksum = "738c290dfaea84fc1ca15ad9c168d083b05a714e1efddd8edaab678dc28d2836" dependencies = [ "cfg-if 1.0.0", ] @@ -294,12 +293,12 @@ dependencies = [ [[package]] name = "ctor" -version = "0.1.19" +version = "0.1.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8f45d9ad417bcef4817d614a501ab55cdd96a6fdb24f49aab89a54acfd66b19" +checksum = "ccc0a48a9b826acdf4028595adc9db92caea352f7af011a3034acd172a52a0aa" dependencies = [ - "quote 1.0.8", - "syn 1.0.74", + "quote 1.0.14", + "syn 1.0.84", ] [[package]] @@ -308,9 +307,9 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fcc3dd5e9e9c0b295d6e1e4d811fb6f157d5ffd784b8d202fc62eac8035a770b" dependencies = [ - "proc-macro2 1.0.28", - "quote 1.0.8", - "syn 1.0.74", + "proc-macro2 1.0.36", + "quote 1.0.14", + "syn 1.0.84", ] [[package]] @@ -327,18 +326,18 @@ checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" [[package]] name = "fastrand" -version = "1.4.1" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77b705829d1e87f762c2df6da140b26af5839e1033aa84aa5f56bb688e4e1bdb" +checksum = "779d043b6a0b90cc4c0ed7ee380a6504394cee7efd7db050e3774eee387324b2" dependencies = [ "instant", ] [[package]] name = "flate2" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd3aec53de10fe96d7d8c565eb17f2c687bb5518a2ec453b5b1252964526abe0" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" dependencies = [ "cfg-if 1.0.0", "crc32fast", @@ -354,9 +353,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" [[package]] name = "futures" -version = "0.3.12" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "da9052a1a50244d8d5aa9bf55cbc2fb6f357c86cc52e46c62ed390a7180cf150" +checksum = "28560757fe2bb34e79f907794bb6b22ae8b0e5c669b638a1132f2592b19035b4" dependencies = [ "futures-channel", "futures-core", @@ -369,9 +368,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.12" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2d31b7ec7efab6eefc7c57233bb10b847986139d88cc2f5a02a1ae6871a1846" +checksum = "ba3dda0b6588335f360afc675d0564c17a77a2bda81ca178a4b6081bd86c7f0b" dependencies = [ "futures-core", "futures-sink", @@ -379,15 +378,15 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.12" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79e5145dde8da7d1b3892dad07a9c98fc04bc39892b1ecc9692cf53e2b780a65" +checksum = "d0c8ff0461b82559810cdccfde3215c3f373807f5e5232b71479bff7bb2583d7" [[package]] name = "futures-executor" -version = "0.3.12" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9e59fdc009a4b3096bf94f740a0f2424c082521f20a9b08c5c07c48d90fd9b9" +checksum = "29d6d2ff5bb10fb95c85b8ce46538a2e5f5e7fdc755623a7d4529ab8a4ed9d2a" dependencies = [ "futures-core", "futures-task", @@ -396,9 +395,9 @@ dependencies = [ [[package]] name = "futures-io" -version = "0.3.12" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28be053525281ad8259d47e4de5de657b25e7bac113458555bb4b70bc6870500" +checksum = "b1f9d34af5a1aac6fb380f735fe510746c38067c5bf16c7fd250280503c971b2" [[package]] name = "futures-lite" @@ -417,36 +416,32 @@ dependencies = [ [[package]] name = "futures-macro" -version = "0.3.12" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c287d25add322d9f9abdcdc5927ca398917996600182178774032e9f8258fedd" +checksum = "6dbd947adfffb0efc70599b3ddcf7b5597bb5fa9e245eb99f62b3a5f7bb8bd3c" dependencies = [ - "proc-macro-hack", - "proc-macro2 1.0.28", - "quote 1.0.8", - "syn 1.0.74", + "proc-macro2 1.0.36", + "quote 1.0.14", + "syn 1.0.84", ] [[package]] name = "futures-sink" -version = "0.3.12" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf5c69029bda2e743fddd0582d1083951d65cc9539aebf8812f36c3491342d6" +checksum = "e3055baccb68d74ff6480350f8d6eb8fcfa3aa11bdc1a1ae3afdd0514617d508" [[package]] name = "futures-task" -version = "0.3.12" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13de07eb8ea81ae445aca7b69f5f7bf15d7bf4912d8ca37d6645c77ae8a58d86" -dependencies = [ - "once_cell", -] +checksum = "6ee7c6485c30167ce4dfb83ac568a849fe53274c831081476ee13e0dce1aad72" [[package]] name = "futures-util" -version = "0.3.12" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "632a8cd0f2a4b3fdea1657f08bde063848c3bd00f9bbf6e256b8be78802e624b" +checksum = "d9b5cf40b47a271f77a8b1bec03ca09044d99d2372c0de244e66430761127164" dependencies = [ "futures-channel", "futures-core", @@ -457,8 +452,6 @@ dependencies = [ "memchr", "pin-project-lite", "pin-utils", - "proc-macro-hack", - "proc-macro-nested", "slab", ] @@ -507,9 +500,9 @@ dependencies = [ [[package]] name = "gloo-timers" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f" +checksum = "6f16c88aa13d2656ef20d1c042086b8767bbe2bdb62526894275a1b062161b2e" dependencies = [ "futures-channel", "futures-core", @@ -520,9 +513,9 @@ dependencies = [ [[package]] name = "hashbrown" -version = "0.9.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04" +checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" [[package]] name = "hermit-abi" @@ -535,20 +528,20 @@ dependencies = [ [[package]] name = "http" -version = "0.2.3" +version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7245cd7449cc792608c3c8a9eaf69bd4eabbabf802713748fd739c98b82f0747" +checksum = "1323096b05d41827dadeaee54c9981958c0f94e670bc94ed80037d1a7b8b186b" dependencies = [ "bytes", "fnv", - "itoa", + "itoa 0.4.8", ] [[package]] name = "indexmap" -version = "1.6.1" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fb1fa934250de4de8aef298d81c729a7d33d8c239daa3a7575e6b92bfc7313b" +checksum = "bc633605454125dec4b66843673f01c7df2b89479b32e0ed634e43a91cff62a5" dependencies = [ "autocfg", "hashbrown", @@ -556,27 +549,33 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.9" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" dependencies = [ "cfg-if 1.0.0", ] [[package]] name = "itertools" -version = "0.10.1" +version = "0.10.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69ddb889f9d0d08a67338271fa9b62996bc788c7796a5c18cf057420aaed5eaf" +checksum = "a9a9d19fa1e79b6215ff29b9d6880b706147f16e9b1dbb1e4e5947b5b02bc5e3" dependencies = [ "either", ] [[package]] name = "itoa" -version = "0.4.7" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" +checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4" + +[[package]] +name = "itoa" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35" [[package]] name = "js-intern" @@ -612,9 +611,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.47" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5cfb73131c35423a367daf8cbd24100af0d077668c8c2943f0e7dd775fef0f65" +checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062" dependencies = [ "wasm-bindgen", ] @@ -636,9 +635,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.97" +version = "0.2.112" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12b8adadd720df158f4d70dfe7ccc6adb0472d7c55ca83445f6a5ab3e36f8fb6" +checksum = "1b03d17f364a3a042d5e5d46b053bbbf82c92c9430c592dd4c064dc6ee997125" [[package]] name = "log" @@ -652,9 +651,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.3.4" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525" +checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a" [[package]] name = "memory_units" @@ -685,7 +684,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bafe4179722c2894288ee77a9f044f02811c86af699344c498b0840c698a2465" dependencies = [ "arrayvec", - "itoa", + "itoa 0.4.8", ] [[package]] @@ -709,9 +708,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" dependencies = [ "hermit-abi", "libc", @@ -719,9 +718,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.5.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13bd41f508810a131401606d54ac32a467c97172d74ba7662562ebba5ad07fa0" +checksum = "da32515d9f6e6e489d7bc9d84c71b060db7247dc035bbe44eac88cf87486d8d5" [[package]] name = "parking" @@ -731,7 +730,7 @@ checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" [[package]] name = "perspective-viewer" -version = "1.0.1" +version = "1.0.8" dependencies = [ "async-std", "async-trait", @@ -757,9 +756,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.4" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439697af366c49a6d0a010c56a0d97685bc140ce0d377b13a2ea2aa42d64a827" +checksum = "8d31d11c69a6b52a174b42bdc0c30e5e11670f90788b2c471c31c1d17d449443" [[package]] name = "pin-utils" @@ -769,9 +768,9 @@ checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" [[package]] name = "polling" -version = "2.1.0" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92341d779fa34ea8437ef4d82d440d5e1ce3f3ff7f824aa64424cd481f9a1f25" +checksum = "685404d509889fade3e86fe3a5803bca2ec09b0c0778d5ada6ec8bf7a8de5259" dependencies = [ "cfg-if 1.0.0", "libc", @@ -780,18 +779,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "proc-macro-hack" -version = "0.5.19" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbf0c48bc1d91375ae5c3cd81e3722dff1abcf81a30960240640d223f59fe0e5" - -[[package]] -name = "proc-macro-nested" -version = "0.1.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc881b2c22681370c6a780e47af9840ef841837bc98118431d4e1868bd0c1086" - [[package]] name = "proc-macro2" version = "0.4.30" @@ -803,11 +790,11 @@ dependencies = [ [[package]] name = "proc-macro2" -version = "1.0.28" +version = "1.0.36" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7ed8b8c7b886ea3ed7dde405212185f423ab44682667c8c6dd14aa1d9f6612" +checksum = "c7342d5883fbccae1cc37a2353b09c87c9b0f3afd73f5fb9bba687a1f733b029" dependencies = [ - "unicode-xid 0.2.1", + "unicode-xid 0.2.2", ] [[package]] @@ -821,11 +808,11 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.8" +version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "991431c3519a3f36861882da93630ce66b52918dcf1b8e2fd66b397fc96f28df" +checksum = "47aa80447ce4daf1717500037052af176af5d38cc3e571d9ec1c7353fc10c87d" dependencies = [ - "proc-macro2 1.0.28", + "proc-macro2 1.0.36", ] [[package]] @@ -851,9 +838,9 @@ dependencies = [ [[package]] name = "ryu" -version = "1.0.5" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e" +checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" [[package]] name = "scoped-tls" @@ -869,46 +856,46 @@ checksum = "c2e9d7eaddb227e8fbaaa71136ae0e1e913ca159b86c7da82f3e8f0044ad3a63" [[package]] name = "serde" -version = "1.0.123" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "92d5161132722baa40d802cc70b15262b98258453e85e5d1d365c757c73869ae" +checksum = "8b9875c23cf305cd1fd7eb77234cbb705f21ea6a72c637a5c6db5fe4b8e7f008" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.123" +version = "1.0.132" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9391c295d64fc0abb2c556bad848f33cb8296276b1ad2677d1ae1ace4f258f31" +checksum = "ecc0db5cb2556c0e558887d9bbdcf6ac4471e83ff66cf696e5419024d1606276" dependencies = [ - "proc-macro2 1.0.28", - "quote 1.0.8", - "syn 1.0.74", + "proc-macro2 1.0.36", + "quote 1.0.14", + "syn 1.0.84", ] [[package]] name = "serde_json" -version = "1.0.61" +version = "1.0.73" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fceb2595057b6891a4ee808f70054bd2d12f0e97f1cbb78689b59f676df325a" +checksum = "bcbd0344bc6533bc7ec56df11d42fb70f1b912351c0825ccb7211b59d8af7cf5" dependencies = [ - "itoa", + "itoa 1.0.1", "ryu", "serde", ] [[package]] name = "slab" -version = "0.4.2" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" +checksum = "9def91fd1e018fe007022791f865d0ccc9b3a0d5001e01aabb8b40e46000afb5" [[package]] name = "socket2" -version = "0.4.0" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e3dfc207c526015c632472a77be09cf1b6e46866581aecae5cc38fb4235dea2" +checksum = "5dc90fe6c7be1a323296982db1836d1ea9e47b6839496dde9a541bc496df3516" dependencies = [ "libc", "winapi", @@ -927,33 +914,33 @@ dependencies = [ [[package]] name = "syn" -version = "1.0.74" +version = "1.0.84" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c" +checksum = "ecb2e6da8ee5eb9a61068762a32fa9619cc591ceb055b3687f4cd4051ec2e06b" dependencies = [ - "proc-macro2 1.0.28", - "quote 1.0.8", - "unicode-xid 0.2.1", + "proc-macro2 1.0.36", + "quote 1.0.14", + "unicode-xid 0.2.2", ] [[package]] name = "thiserror" -version = "1.0.23" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76cc616c6abf8c8928e2fdcc0dbfab37175edd8fb49a4641066ad1364fdab146" +checksum = "854babe52e4df1653706b98fcfc05843010039b406875930a70e4d9644e5c417" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "1.0.23" +version = "1.0.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9be73a2caec27583d0046ef3796c3794f868a5bc813db689eed00c7631275cd1" +checksum = "aa32fd3f627f367fe16f893e2597ae3c05020f8bba2666a4e6ea73d377e5714b" dependencies = [ - "proc-macro2 1.0.28", - "quote 1.0.8", - "syn 1.0.74", + "proc-macro2 1.0.36", + "quote 1.0.14", + "syn 1.0.84", ] [[package]] @@ -975,15 +962,15 @@ checksum = "fc72304796d0818e357ead4e000d19c9c174ab23dc11093ac919054d20a6a7fc" [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "value-bag" -version = "1.0.0-alpha.7" +version = "1.0.0-alpha.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd320e1520f94261153e96f7534476ad869c14022aee1e59af7c778075d840ae" +checksum = "79923f7731dc61ebfba3633098bf3ac533bbd35ccd8c57e7088d9a5eebe0263f" dependencies = [ "ctor", "version_check", @@ -1028,17 +1015,17 @@ dependencies = [ "bumpalo", "lazy_static", "log", - "proc-macro2 1.0.28", - "quote 1.0.8", - "syn 1.0.74", + "proc-macro2 1.0.36", + "quote 1.0.14", + "syn 1.0.84", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.20" +version = "0.4.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3de431a2910c86679c34283a33f66f4e4abd7e0aec27b6669060148872aadf94" +checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1" dependencies = [ "cfg-if 1.0.0", "js-sys", @@ -1052,7 +1039,7 @@ version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4" dependencies = [ - "quote 1.0.8", + "quote 1.0.14", "wasm-bindgen-macro-support", ] @@ -1062,9 +1049,9 @@ version = "0.2.74" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97" dependencies = [ - "proc-macro2 1.0.28", - "quote 1.0.8", - "syn 1.0.74", + "proc-macro2 1.0.36", + "quote 1.0.14", + "syn 1.0.84", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -1077,9 +1064,9 @@ checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f" [[package]] name = "wasm-bindgen-test" -version = "0.3.20" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0d4da138503a4cf86801b94d95781ee3619faa8feca830569cc6b54997b8b5c" +checksum = "8cab416a9b970464c2882ed92d55b0c33046b08e0bdc9d59b3b718acd4e1bae8" dependencies = [ "console_error_panic_hook", "js-sys", @@ -1091,19 +1078,19 @@ dependencies = [ [[package]] name = "wasm-bindgen-test-macro" -version = "0.3.20" +version = "0.3.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3199c33f06500c731d5544664c24d0c2b742b98debc6b1c6f0c6d6e8fb7c19b" +checksum = "dd4543fc6cf3541ef0d98bf720104cc6bd856d7eba449fd2aa365ef4fed0e782" dependencies = [ - "proc-macro2 1.0.28", - "quote 1.0.8", + "proc-macro2 1.0.36", + "quote 1.0.14", ] [[package]] name = "web-sys" -version = "0.3.47" +version = "0.3.51" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c40dc691fc48003eba817c38da7113c15698142da971298003cac3ef175680b3" +checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582" dependencies = [ "js-sys", "wasm-bindgen", @@ -1155,7 +1142,7 @@ checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" [[package]] name = "yew" version = "0.18.0" -source = "git+https://github.com/yewstack/yew?rev=f94487364f6eb1c7862e48f7c82ee4122ae8e845#f94487364f6eb1c7862e48f7c82ee4122ae8e845" +source = "git+https://github.com/yewstack/yew?rev=60c08736f1034f9402dddae4335584c103ffd7f0#60c08736f1034f9402dddae4335584c103ffd7f0" dependencies = [ "anyhow", "anymap", @@ -1180,11 +1167,11 @@ dependencies = [ [[package]] name = "yew-macro" version = "0.18.0" -source = "git+https://github.com/yewstack/yew?rev=f94487364f6eb1c7862e48f7c82ee4122ae8e845#f94487364f6eb1c7862e48f7c82ee4122ae8e845" +source = "git+https://github.com/yewstack/yew?rev=60c08736f1034f9402dddae4335584c103ffd7f0#60c08736f1034f9402dddae4335584c103ffd7f0" dependencies = [ "boolinator", "lazy_static", - "proc-macro2 1.0.28", - "quote 1.0.8", - "syn 1.0.74", + "proc-macro2 1.0.36", + "quote 1.0.14", + "syn 1.0.84", ] diff --git a/rust/perspective-viewer/Cargo.toml b/rust/perspective-viewer/Cargo.toml index 0ac5b4c258..9d5d838b2d 100644 --- a/rust/perspective-viewer/Cargo.toml +++ b/rust/perspective-viewer/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "perspective-viewer" -version = "1.0.1" +version = "1.0.8" authors = ["Andrew Stein "] edition = "2018" description = "A frontend for Perspective" @@ -30,13 +30,10 @@ serde = { version = "1.0", features = ["derive"] } serde_json = { version = "1.0.59", features = ["raw_value"] } wasm-bindgen = { version = "=0.2.74", features = ["serde-serialize"] } wasm-bindgen-futures = "0.4.20" -yew = { git = "https://github.com/yewstack/yew", rev = "f94487364f6eb1c7862e48f7c82ee4122ae8e845" } -# `wee_alloc` is a tiny allocator for wasm that is only ~1K in code size -# compared to the default allocator's ~10K. It is slower than the default -# allocator, however. -# -# Unfortunately, `wee_alloc` requires nightly Rust when targeting wasm for now. +# https://github.com/yewstack/yew/issues/2154 +yew = { git = "https://github.com/yewstack/yew", rev = "60c08736f1034f9402dddae4335584c103ffd7f0" } + wee_alloc = { version = "0.4.5", optional = true } [dependencies.web-sys] @@ -52,6 +49,7 @@ features = [ "Document", "DomRect", "DomStringMap", + "DomTokenList", "Element", # "DomTokenList", "Event", @@ -65,6 +63,8 @@ features = [ "HtmlCollection", "HtmlElement", "HtmlTextAreaElement", + "HtmlSelectElement", + "InputEvent", "KeyboardEvent", # "MouseEvent", "MutationObserver", diff --git a/rust/perspective-viewer/package.json b/rust/perspective-viewer/package.json index e0a8a04a80..9eb8d19aac 100644 --- a/rust/perspective-viewer/package.json +++ b/rust/perspective-viewer/package.json @@ -1,6 +1,6 @@ { "name": "@finos/perspective-viewer", - "version": "1.0.8", + "version": "1.1.0", "description": "The `` Custom Element, frontend for Perspective.js", "repository": { "type": "git", @@ -53,7 +53,7 @@ "access": "public" }, "dependencies": { - "@finos/perspective": "^1.0.8", + "@finos/perspective": "^1.1.0", "fflate": "^0.7.2", "mobile-drag-drop-shadow-dom": "3.0.0", "monaco-editor": "0.24.0" @@ -62,6 +62,6 @@ "monaco-editor-webpack-plugin": "3.1.0" }, "devDependencies": { - "@finos/perspective-test": "^1.0.8" + "@finos/perspective-test": "^1.1.0" } } diff --git a/rust/perspective-viewer/src/less/column-selector.less b/rust/perspective-viewer/src/less/column-selector.less index 31994016d2..25636ba0b3 100644 --- a/rust/perspective-viewer/src/less/column-selector.less +++ b/rust/perspective-viewer/src/less/column-selector.less @@ -66,11 +66,13 @@ .dragdrop-highlight { .column_selector_draggable.dragover, - .column-selector-column:not(:last-child) .column_selector_draggable { + .column-selector-column:not(:last-child) + .column_selector_draggable { border-bottom-color: var(--active--color) !important; } - .column-selector-column:not(:first-child):not([data-label]) .column_selector_draggable.dragover { + .column-selector-column:not(:first-child):not([data-label]) + .column_selector_draggable.dragover { border-top-color: rgb(0 0 0 / 5%) !important; } } diff --git a/rust/perspective-viewer/src/less/column-style.less b/rust/perspective-viewer/src/less/column-style.less index 2cd22109a4..1b00834f72 100644 --- a/rust/perspective-viewer/src/less/column-style.less +++ b/rust/perspective-viewer/src/less/column-style.less @@ -21,8 +21,27 @@ box-shadow: 0 2px 4px 0 rgb(0 0 0 / 10%); user-select: none; + #column-style-container { + margin-bottom: 4px; + width: 120px; + } + + label { + font-size: 8px; + width: 100%; + } + + input#color-selected, + input#format-selected { + float: left; + } + + input#gradient-param { + margin-bottom: 4px; + } + input.parameter { - max-width: 65px; + max-width: 80px; background: none; color: inherit; border: 0px solid transparent; @@ -83,17 +102,40 @@ } div.section { - margin-bottom: 8px; + margin-bottom: 4px; + flex: 1 1 100%; + } + + div.inner_section { + margin-top: 4px; } div.row { display: flex; align-items: center; - height: 24px; + flex-wrap: wrap; } input[type="color"] { - width: 24px; + width: 36px; + height: 36px; + cursor: pointer; + padding: 0; + margin-right: 4px; + + &:before { + position: absolute; + font-family: "Material Icons"; + margin-top: 11px; + margin-left: 12px; + font-size: 12px; + content: "add"; + color: white; + } + + &#neg-color-param:before { + content: "remove"; + } } .operator { diff --git a/rust/perspective-viewer/src/less/fonts.less b/rust/perspective-viewer/src/less/fonts.less index 8d6514a4fc..ef360fe591 100644 --- a/rust/perspective-viewer/src/less/fonts.less +++ b/rust/perspective-viewer/src/less/fonts.less @@ -1,2 +1,2 @@ @import (css) - url("https://fonts.googleapis.com/css?family=Roboto+Mono:200,400%7CMaterial+Icons%7COpen+Sans:300,400"); + url("https://fonts.googleapis.com/css?family=Roboto+Mono:200,400%7CMaterial+Icons%7COpen+Sans:300,400,700"); diff --git a/rust/perspective-viewer/src/less/radio-list.less b/rust/perspective-viewer/src/less/radio-list.less index 25fac7681d..bf7b6c6ed0 100644 --- a/rust/perspective-viewer/src/less/radio-list.less +++ b/rust/perspective-viewer/src/less/radio-list.less @@ -13,13 +13,11 @@ display: flex; flex-wrap: wrap; align-items: center; - padding-top: 6px; - width: 110px; // Fix radio buttons input[type="radio"] { - width: 24px; - margin: 0; + width: 16px; + margin: 0 4px 0 0; } input[type="radio"][disabled] ~ * { diff --git a/rust/perspective-viewer/src/rust/components/active_column.rs b/rust/perspective-viewer/src/rust/components/active_column.rs index d31336d777..370e161cef 100644 --- a/rust/perspective-viewer/src/rust/components/active_column.rs +++ b/rust/perspective-viewer/src/rust/components/active_column.rs @@ -6,6 +6,8 @@ // of the Apache License 2.0. The full license can be found in the LICENSE // file. +use super::aggregate_selector::*; +use super::expression_toolbar::*; use crate::config::*; use crate::dragdrop::*; use crate::js::plugin::*; @@ -13,9 +15,6 @@ use crate::renderer::*; use crate::session::*; use crate::*; -use super::aggregate_selector::*; -use super::expression_toolbar::*; - use itertools::Itertools; use web_sys::*; use yew::prelude::*; @@ -48,11 +47,8 @@ pub struct ActiveColumnProps { } impl PartialEq for ActiveColumnProps { - /// Equality for `ActiveColumnProps` determines when it should re-render, which - /// is only when it has changed. - /// TODO Aggregates & ViewConfig generally - fn eq(&self, rhs: &ActiveColumnProps) -> bool { - self.idx == rhs.idx && self.name == rhs.name && self.is_pivot == rhs.is_pivot + fn eq(&self, _rhs: &ActiveColumnProps) -> bool { + false } } @@ -138,8 +134,6 @@ pub enum ActiveColumnMsg { /// aggregate), and supports drag/drop and missing entries. /// TODO Break this into "Active", "Hover" and "Empty"? pub struct ActiveColumn { - link: ComponentLink, - props: ActiveColumnProps, add_expression_ref: NodeRef, column_type: Option, is_required: bool, @@ -149,51 +143,40 @@ impl Component for ActiveColumn { type Message = ActiveColumnMsg; type Properties = ActiveColumnProps; - fn create( - props: ::Properties, - link: ComponentLink, - ) -> Self { + fn create(ctx: &Context) -> Self { let add_expression_ref = NodeRef::default(); - let column_type = props.get_type(); - let is_required = props.get_is_required(); + let column_type = ctx.props().get_type(); + let is_required = ctx.props().get_is_required(); ActiveColumn { - link, - props, add_expression_ref, column_type, is_required, } } - fn change(&mut self, props: ::Properties) -> ShouldRender { - let is_required = props.get_is_required(); - let coltype = props.get_type(); - let should_render = self.props != props - || self.column_type != coltype - || is_required != self.is_required; - self.column_type = coltype; - self.is_required = is_required; - self.props = props; - should_render + fn changed(&mut self, ctx: &Context) -> bool { + self.column_type = ctx.props().get_type(); + self.is_required = ctx.props().get_is_required(); + true } - fn update(&mut self, msg: ::Message) -> ShouldRender { + fn update(&mut self, ctx: &Context, msg: ActiveColumnMsg) -> bool { match msg { ActiveColumnMsg::DeactivateColumn(column, shift_key) => { - self.props.deactivate_column(column, shift_key); - self.props.onselect.emit(()); + ctx.props().deactivate_column(column, shift_key); + ctx.props().onselect.emit(()); false } } } - fn view(&self) -> Html { + fn view(&self, ctx: &Context) -> Html { let mut classes = vec!["column_selector_draggable"]; - if self.props.is_pivot { + if ctx.props().is_pivot { classes.push("show-aggregate"); }; - let name = match &self.props.name { + let name = match &ctx.props().name { ActiveColumnState::DragOver(label) => { classes.push("dragover"); if label.is_some() && !self.is_required { @@ -202,7 +185,7 @@ impl Component for ActiveColumn { ( label.clone(), - Some(self.props.dragdrop.get_drag_column().unwrap()), + Some(ctx.props().dragdrop.get_drag_column().unwrap()), ) } ActiveColumnState::Column(label, name) => { @@ -220,8 +203,8 @@ impl Component for ActiveColumn {
+ data-index={ ctx.props().idx.to_string() } + ondragenter={ ctx.props().ondragenter.clone() }> @@ -236,7 +219,7 @@ impl Component for ActiveColumn { let remove_column = if self.is_required { None } else { - Some(self.link.callback({ + Some(ctx.link().callback({ let event_name = name.to_owned(); move |event: MouseEvent| { ActiveColumnMsg::DeactivateColumn( @@ -251,7 +234,7 @@ impl Component for ActiveColumn { let dragstart = Callback::from({ let event_name = name.to_owned(); let noderef = noderef.clone(); - let dragdrop = self.props.dragdrop.clone(); + let dragdrop = ctx.props().dragdrop.clone(); move |event: DragEvent| { let elem = noderef.cast::().unwrap(); event.data_transfer().unwrap().set_drag_image(&elem, 0, 0); @@ -263,7 +246,7 @@ impl Component for ActiveColumn { }); let is_expression = - self.props.session.metadata().is_column_expression(&name); + ctx.props().session.metadata().is_column_expression(&name); let class = if self.is_required { "is_column_active required" @@ -275,8 +258,8 @@ impl Component for ActiveColumn {
+ data-index={ ctx.props().idx.to_string() } + ondragenter={ ctx.props().ondragenter.clone() }> + ondragend={ ctx.props().ondragend.clone() }> @@ -314,9 +297,9 @@ impl Component for ActiveColumn { } { - if self.props.is_pivot { - let aggregate = self - .props + if ctx.props().is_pivot { + let aggregate = ctx + .props() .config .aggregates .get(&name) @@ -325,8 +308,8 @@ impl Component for ActiveColumn { + renderer={ ctx.props().renderer.clone() } + session={ ctx.props().session.clone() }> } } else { diff --git a/rust/perspective-viewer/src/rust/components/aggregate_selector.rs b/rust/perspective-viewer/src/rust/components/aggregate_selector.rs index 0838f33023..382ee05135 100644 --- a/rust/perspective-viewer/src/rust/components/aggregate_selector.rs +++ b/rust/perspective-viewer/src/rust/components/aggregate_selector.rs @@ -6,14 +6,12 @@ // of the Apache License 2.0. The full license can be found in the LICENSE // file. +use super::containers::dropdown::*; use crate::config::*; use crate::renderer::*; use crate::session::*; - use crate::*; -use super::containers::dropdown::*; - use yew::prelude::*; #[derive(Properties, Clone)] @@ -27,50 +25,8 @@ pub struct AggregateSelectorProps { derive_renderable_props!(AggregateSelectorProps); impl PartialEq for AggregateSelectorProps { - fn eq(&self, rhs: &Self) -> bool { - self.column == rhs.column && self.aggregate == rhs.aggregate - } -} - -impl AggregateSelectorProps { - pub fn set_aggregate(&mut self, aggregate: Aggregate) { - self.aggregate = Some(aggregate.clone()); - let ViewConfig { mut aggregates, .. } = self.session.get_view_config(); - aggregates.insert(self.column.clone(), aggregate); - self.update_and_render(ViewConfigUpdate { - aggregates: Some(aggregates), - ..ViewConfigUpdate::default() - }); - } - - pub fn get_dropdown_aggregates(&self) -> Vec> { - let aggregates = self - .session - .metadata() - .get_column_aggregates(&self.column) - .expect("Bad Aggs") - .collect::>(); - - let multi_aggregates = aggregates - .iter() - .filter(|x| matches!(x, Aggregate::MultiAggregate(_, _))) - .cloned() - .collect::>(); - - let multi_aggregates2 = if !multi_aggregates.is_empty() { - vec![DropDownItem::OptGroup("weighted mean", multi_aggregates)] - } else { - vec![] - }; - - let s = aggregates - .iter() - .filter(|x| matches!(x, Aggregate::SingleAggregate(_))) - .cloned() - .map(DropDownItem::Option) - .chain(multi_aggregates2); - - s.collect::>() + fn eq(&self, _rhs: &Self) -> bool { + false } } @@ -79,54 +35,49 @@ pub enum AggregateSelectorMsg { } pub struct AggregateSelector { - props: AggregateSelectorProps, - link: ComponentLink, aggregates: Vec>, + aggregate: Option, } impl Component for AggregateSelector { type Message = AggregateSelectorMsg; type Properties = AggregateSelectorProps; - fn create(props: Self::Properties, link: ComponentLink) -> Self { - let aggregates = props.get_dropdown_aggregates(); - AggregateSelector { - props, - link, - aggregates, - } + fn create(ctx: &Context) -> Self { + let mut selector = AggregateSelector { + aggregates: vec![], + aggregate: ctx.props().aggregate.clone(), + }; + + selector.aggregates = selector.get_dropdown_aggregates(ctx); + selector } - fn update(&mut self, msg: Self::Message) -> ShouldRender { + fn update(&mut self, ctx: &Context, msg: Self::Message) -> bool { match msg { AggregateSelectorMsg::SetAggregate(aggregate) => { - self.props.set_aggregate(aggregate); + self.set_aggregate(ctx, aggregate); false } } } - fn change(&mut self, props: Self::Properties) -> ShouldRender { - let should_render = self.props != props; - if should_render { - self.props = props; - self.aggregates = self.props.get_dropdown_aggregates(); - } - - should_render + fn changed(&mut self, ctx: &Context) -> bool { + self.aggregates = self.get_dropdown_aggregates(ctx); + true } - fn view(&self) -> Html { - let callback = self.link.callback(AggregateSelectorMsg::SetAggregate); - let selected_agg = self - .props + fn view(&self, ctx: &Context) -> Html { + let callback = ctx.link().callback(AggregateSelectorMsg::SetAggregate); + let selected_agg = ctx + .props() .aggregate .clone() .or_else(|| { - self.props + ctx.props() .session .metadata() - .get_column_table_type(&self.props.column) + .get_column_table_type(&ctx.props().column) .map(|x| x.default_aggregate()) }) .unwrap(); @@ -146,3 +97,49 @@ impl Component for AggregateSelector { } } } + +impl AggregateSelector { + pub fn set_aggregate(&mut self, ctx: &Context, aggregate: Aggregate) { + self.aggregate = Some(aggregate.clone()); + let ViewConfig { mut aggregates, .. } = ctx.props().session.get_view_config(); + aggregates.insert(ctx.props().column.clone(), aggregate); + ctx.props().update_and_render(ViewConfigUpdate { + aggregates: Some(aggregates), + ..ViewConfigUpdate::default() + }); + } + + pub fn get_dropdown_aggregates( + &self, + ctx: &Context, + ) -> Vec> { + let aggregates = ctx + .props() + .session + .metadata() + .get_column_aggregates(&ctx.props().column) + .expect("Bad Aggs") + .collect::>(); + + let multi_aggregates = aggregates + .iter() + .filter(|x| matches!(x, Aggregate::MultiAggregate(_, _))) + .cloned() + .collect::>(); + + let multi_aggregates2 = if !multi_aggregates.is_empty() { + vec![DropDownItem::OptGroup("weighted mean", multi_aggregates)] + } else { + vec![] + }; + + let s = aggregates + .iter() + .filter(|x| matches!(x, Aggregate::SingleAggregate(_))) + .cloned() + .map(DropDownItem::Option) + .chain(multi_aggregates2); + + s.collect::>() + } +} diff --git a/rust/perspective-viewer/src/rust/components/color_range_selector.rs b/rust/perspective-viewer/src/rust/components/color_range_selector.rs new file mode 100644 index 0000000000..ef2af87d61 --- /dev/null +++ b/rust/perspective-viewer/src/rust/components/color_range_selector.rs @@ -0,0 +1,57 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2018, the Perspective Authors. +// +// This file is part of the Perspective library, distributed under the terms +// of the Apache License 2.0. The full license can be found in the LICENSE +// file. + +use wasm_bindgen::JsCast; +use web_sys::*; +use yew::prelude::*; +use yew::*; + +#[derive(Properties, PartialEq, Clone)] +pub struct ColorRangeProps { + pub pos_color: String, + pub neg_color: String, + pub on_pos_color: Callback, + pub on_neg_color: Callback, +} + +#[function_component(ColorRangeSelector)] +pub fn color_chooser_component(props: &ColorRangeProps) -> Html { + let on_pos_color = props.on_pos_color.reform(|event: InputEvent| { + event + .target() + .unwrap() + .unchecked_into::() + .value() + }); + + let on_neg_color = props.on_neg_color.reform(|event: InputEvent| { + event + .target() + .unwrap() + .unchecked_into::() + .value() + }); + + html! { + <> + + + + + } +} diff --git a/rust/perspective-viewer/src/rust/components/color_selector.rs b/rust/perspective-viewer/src/rust/components/color_selector.rs new file mode 100644 index 0000000000..163c5a9014 --- /dev/null +++ b/rust/perspective-viewer/src/rust/components/color_selector.rs @@ -0,0 +1,40 @@ +//////////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2018, the Perspective Authors. +// +// This file is part of the Perspective library, distributed under the terms +// of the Apache License 2.0. The full license can be found in the LICENSE +// file. + +use wasm_bindgen::JsCast; +use web_sys::*; +use yew::prelude::*; +use yew::*; + +#[derive(Properties, PartialEq, Clone)] +pub struct ColorProps { + pub color: String, + pub on_color: Callback, +} + +#[function_component(ColorSelector)] +pub fn color_component(props: &ColorProps) -> Html { + let oninput = props.on_color.reform(|event: InputEvent| { + event + .target() + .unwrap() + .unchecked_into::() + .value() + }); + + html! { + <> + + + + } +} diff --git a/rust/perspective-viewer/src/rust/components/column_selector.rs b/rust/perspective-viewer/src/rust/components/column_selector.rs index 782cad6a91..63daddf825 100644 --- a/rust/perspective-viewer/src/rust/components/column_selector.rs +++ b/rust/perspective-viewer/src/rust/components/column_selector.rs @@ -6,6 +6,8 @@ // of the Apache License 2.0. The full license can be found in the LICENSE // file. +use super::active_column::*; +use super::inactive_column::*; use crate::config::*; use crate::custom_elements::expression_editor::ExpressionEditorElement; use crate::dragdrop::*; @@ -14,9 +16,6 @@ use crate::session::*; use crate::utils::*; use crate::*; -use super::active_column::*; -use super::inactive_column::*; - use itertools::Itertools; use std::cmp::Ordering; use std::iter::*; @@ -25,7 +24,7 @@ use wasm_bindgen::JsCast; use web_sys::*; use yew::prelude::*; -#[derive(Properties, Clone)] +#[derive(Properties, Clone, PartialEq)] pub struct ColumnSelectorProps { pub session: Session, pub renderer: Renderer, @@ -50,6 +49,7 @@ impl ColumnSelectorProps { } } +#[derive(Debug)] pub enum ColumnSelectorMsg { TableLoaded, ViewCreated, @@ -64,8 +64,6 @@ pub enum ColumnSelectorMsg { /// A `ColumnSelector` controls the `columns` field of the `ViewConfig`, deriving its /// options from the table columns and `ViewConfig` expressions. pub struct ColumnSelector { - props: ColumnSelectorProps, - link: ComponentLink, _subscriptions: [Subscription; 5], add_expression_ref: NodeRef, expression_editor: Option, @@ -75,42 +73,40 @@ impl Component for ColumnSelector { type Message = ColumnSelectorMsg; type Properties = ColumnSelectorProps; - fn create(props: Self::Properties, link: ComponentLink) -> Self { + fn create(ctx: &Context) -> Self { let table_sub = { - let cb = link.callback(|_| ColumnSelectorMsg::TableLoaded); - props.session.on_table_loaded.add_listener(cb) + let cb = ctx.link().callback(|_| ColumnSelectorMsg::TableLoaded); + ctx.props().session.on_table_loaded.add_listener(cb) }; let view_sub = { - let cb = link.callback(|_| ColumnSelectorMsg::ViewCreated); - props.session.on_view_created.add_listener(cb) + let cb = ctx.link().callback(|_| ColumnSelectorMsg::ViewCreated); + ctx.props().session.on_view_created.add_listener(cb) }; let drop_sub = { - let cb = link.callback(ColumnSelectorMsg::Drop); - props.dragdrop.add_on_drop_action(cb) + let cb = ctx.link().callback(ColumnSelectorMsg::Drop); + ctx.props().dragdrop.add_on_drop_action(cb) }; let drag_sub = { - let cb = link.callback(ColumnSelectorMsg::Drag); - props.dragdrop.add_on_drag_action(cb) + let cb = ctx.link().callback(ColumnSelectorMsg::Drag); + ctx.props().dragdrop.add_on_drag_action(cb) }; let dragend_sub = { - let cb = link.callback(|_| ColumnSelectorMsg::DragEnd); - props.dragdrop.add_on_dragend_action(cb) + let cb = ctx.link().callback(|_| ColumnSelectorMsg::DragEnd); + ctx.props().dragdrop.add_on_dragend_action(cb) }; ColumnSelector { - props, - link, _subscriptions: [table_sub, view_sub, drop_sub, drag_sub, dragend_sub], add_expression_ref: NodeRef::default(), expression_editor: None, } } - fn update(&mut self, msg: Self::Message) -> ShouldRender { + fn update(&mut self, ctx: &Context, msg: Self::Message) -> bool { match msg { ColumnSelectorMsg::Drag(DragEffect::Move(DropAction::Active)) => false, ColumnSelectorMsg::Drag(_) => true, @@ -118,10 +114,10 @@ impl Component for ColumnSelector { ColumnSelectorMsg::TableLoaded => true, ColumnSelectorMsg::ViewCreated => true, ColumnSelectorMsg::HoverActiveIndex(index) => { - let min_cols = self.props.renderer.metadata().min; + let min_cols = ctx.props().renderer.metadata().min; match index { Some(to_index) => { - let config = self.props.session.get_view_config(); + let config = ctx.props().session.get_view_config(); let is_to_empty = !config .columns .get(to_index) @@ -129,7 +125,7 @@ impl Component for ColumnSelector { .unwrap_or_default(); let from_index = - self.props.dragdrop.get_drag_column().and_then(|x| { + ctx.props().dragdrop.get_drag_column().and_then(|x| { config .columns .iter() @@ -141,28 +137,30 @@ impl Component for ColumnSelector { .unwrap_or_default() && is_to_empty { - self.props.dragdrop.drag_leave(DropAction::Active); + ctx.props().dragdrop.drag_leave(DropAction::Active); true } else { - self.props.dragdrop.drag_enter(DropAction::Active, to_index) + ctx.props() + .dragdrop + .drag_enter(DropAction::Active, to_index) } } _ => { - self.props.dragdrop.drag_leave(DropAction::Active); + ctx.props().dragdrop.drag_leave(DropAction::Active); true } } } ColumnSelectorMsg::Drop((column, DropAction::Active, effect, index)) => { - let update = self.props.session.create_drag_drop_update( + let update = ctx.props().session.create_drag_drop_update( column, index, DropAction::Active, effect, - &self.props.renderer.metadata(), + &ctx.props().renderer.metadata(), ); - self.props.update_and_render(update); + ctx.props().update_and_render(update); true } ColumnSelectorMsg::Drop(( @@ -173,7 +171,7 @@ impl Component for ColumnSelector { )) => true, ColumnSelectorMsg::Drop((_, _, _, _)) => true, ColumnSelectorMsg::SaveExpression(expression) => { - self.props.save_expr(&expression); + ctx.props().save_expr(&expression); self.expression_editor .take() .and_then(|elem| elem.destroy().ok()) @@ -182,9 +180,9 @@ impl Component for ColumnSelector { true } ColumnSelectorMsg::OpenExpressionEditor => { - let on_save = self.link.callback(ColumnSelectorMsg::SaveExpression); + let on_save = ctx.link().callback(ColumnSelectorMsg::SaveExpression); let mut element = ExpressionEditorElement::new( - self.props.session.clone(), + ctx.props().session.clone(), on_save, None, ); @@ -197,35 +195,31 @@ impl Component for ColumnSelector { } } - fn change(&mut self, _props: Self::Properties) -> ShouldRender { - true - } - - fn view(&self) -> Html { - if let Some(all_columns) = self.props.session.metadata().get_table_columns() { - let config = self.props.session.get_view_config(); + fn view(&self, ctx: &Context) -> Html { + if let Some(all_columns) = ctx.props().session.metadata().get_table_columns() { + let config = ctx.props().session.get_view_config(); let is_dragover_column = - self.props.dragdrop.is_dragover(DropAction::Active); + ctx.props().dragdrop.is_dragover(DropAction::Active); let is_pivot = config.is_pivot(); let expression_columns = - self.props.session.metadata().get_expression_columns(); + ctx.props().session.metadata().get_expression_columns(); let columns_iter = ColumnsIterator::new( &all_columns, &expression_columns, &config, - &self.props.session, - &self.props.renderer, + &ctx.props().session, + &ctx.props().renderer, &is_dragover_column, ); let dragleave = dragleave_helper({ - let link = self.link.clone(); + let link = ctx.link().clone(); move || link.send_message(ColumnSelectorMsg::HoverActiveIndex(None)) }); let dragover = Callback::from(|_event: DragEvent| _event.prevent_default()); - let dragenter = self.link.callback(move |event: DragEvent| { + let dragenter = ctx.link().callback(move |event: DragEvent| { // Safari does not set `relatedTarget` so this event must be allowed to // bubble so we can count entry/exit stacks to determine true // `"dragleave"`. @@ -250,22 +244,22 @@ impl Component for ColumnSelector { }); let drop = Callback::from({ - let dragdrop = self.props.dragdrop.clone(); + let dragdrop = ctx.props().dragdrop.clone(); move |_| dragdrop.notify_drop() }); let dragend = Callback::from({ - let dragdrop = self.props.dragdrop.clone(); + let dragdrop = ctx.props().dragdrop.clone(); move |_event| dragdrop.drag_end() }); - let add_expression = self - .link + let add_expression = ctx + .link() .callback(|_| ColumnSelectorMsg::OpenExpressionEditor); - let select = self.link.callback(|()| ColumnSelectorMsg::ViewCreated); + let select = ctx.link().callback(|()| ColumnSelectorMsg::ViewCreated); let mut active_classes = vec![]; - if self.props.dragdrop.get_drag_column().is_some() { + if ctx.props().dragdrop.get_drag_column().is_some() { active_classes.push("dragdrop-highlight"); }; @@ -288,9 +282,9 @@ impl Component for ColumnSelector {
{ - for columns_iter.expression().enumerate().map(|(idx, name)| { + for columns_iter.expression().enumerate().map(|(idx, (visible, name))| { html! { @@ -321,14 +316,15 @@ impl Component for ColumnSelector {
{ - for columns_iter.inactive().enumerate().map(|(idx, name)| { + for columns_iter.inactive().enumerate().map(|(idx, (visible, name))| { html! { @@ -562,45 +558,58 @@ impl<'a> ColumnsIterator<'a> { } /// Generate an iterator for inactive expressions. - pub fn expression(&'a self) -> Box + 'a> { + pub fn expression(&'a self) -> impl Iterator + 'a { let mut filtered = self .expression_columns .iter() - .filter(move |name| { - !self.config.columns.contains(&Some(name.to_string())) - && self - .is_dragover_column - .as_ref() - .map(|x| &x.1 != *name) - .unwrap_or(true) + .filter_map(move |name| { + let visible = + !self.config.columns.iter().any(|x| x.as_ref() == Some(name)); + + let is_drag = self + .is_dragover_column + .as_ref() + .map(|x| &x.1 != name) + .unwrap_or(true); + + if visible { + Some((is_drag, name)) + } else { + None + } }) .collect::>(); - filtered.sort_by(|x, y| self.sort_by_type(x, y)); - Box::new(filtered.into_iter()) + filtered.sort_by(|x, y| self.sort_by_type(&x.1, &y.1)); + filtered.into_iter() } /// Generate an iterator for inactive columns, which also shows the columns in /// sorted order by type, then name. - pub fn inactive(&'a self) -> Box + 'a> { + pub fn inactive(&'a self) -> impl Iterator + 'a { let dragover_col = self.is_dragover_column.as_ref(); let mut filtered = self .table_columns .iter() - .filter(move |name| { + .filter_map(move |name| { let cols = &self.config.columns; - let is_active = cols.iter().flatten().any(|x| x == *name); - let is_drag = dragover_col.map_or(false, |(_, x)| x == *name); + let is_active = cols.iter().flatten().any(|x| x == name); + let is_drag = dragover_col.map_or(false, |(_, x)| x == name); let is_swap = dragover_col.map_or(false, |(i, _)| { self.renderer.metadata().is_swap(*i) - && cols.get(*i).map(|z| z.as_ref()).flatten() == Some(*name) + && cols.get(*i).map(|z| z.as_ref()).flatten() == Some(name) }); - (!is_active || is_swap) && !is_drag + + if !is_active || is_swap { + Some((!is_drag, name)) + } else { + None + } }) .collect::>(); - filtered.sort_by(|x, y| self.sort_by_type(x, y)); - Box::new(filtered.into_iter()) + filtered.sort_by(|x, y| self.sort_by_type(&x.1, &y.1)); + filtered.into_iter() } /// A comparison function for column names, which takes into account column type as diff --git a/rust/perspective-viewer/src/rust/components/column_style.rs b/rust/perspective-viewer/src/rust/components/column_style.rs deleted file mode 100644 index 63089041d7..0000000000 --- a/rust/perspective-viewer/src/rust/components/column_style.rs +++ /dev/null @@ -1,562 +0,0 @@ -//////////////////////////////////////////////////////////////////////////////// -// -// Copyright (c) 2018, the Perspective Authors. -// -// This file is part of the Perspective library, distributed under the terms -// of the Apache License 2.0. The full license can be found in the LICENSE -// file. - -use crate::utils::WeakComponentLink; - -use super::containers::radio_list::RadioList; - -use serde::{Deserialize, Serialize}; -use std::fmt::Display; -use std::str::FromStr; -use wasm_bindgen::*; -use yew::prelude::*; - -#[cfg(test)] -use wasm_bindgen_test::*; - -pub static CSS: &str = include_str!("../../../build/css/column-style.css"); - -#[derive(PartialEq, Clone, Copy, Debug, Serialize, Deserialize)] -pub enum ColorMode { - #[serde(rename = "disabled")] - Disabled, - - #[serde(rename = "foreground")] - Foreground, - - #[serde(rename = "background")] - Background, - - #[serde(rename = "gradient")] - Gradient, - - #[serde(rename = "bar")] - Bar, -} - -impl Default for ColorMode { - fn default() -> Self { - ColorMode::Foreground - } -} - -impl Display for ColorMode { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let text = match self { - ColorMode::Foreground => "foreground", - ColorMode::Background => "background", - ColorMode::Gradient => "gradient", - ColorMode::Bar => "bar", - _ => panic!("Unknown color mode!"), - }; - - write!(f, "{}", text) - } -} - -impl FromStr for ColorMode { - type Err = String; - fn from_str(s: &str) -> Result { - match s { - "foreground" => Ok(ColorMode::Foreground), - "background" => Ok(ColorMode::Background), - "gradient" => Ok(ColorMode::Gradient), - "bar" => Ok(ColorMode::Bar), - x => Err(format!("Unknown ColorMode::{}", x)), - } - } -} - -impl ColorMode { - fn is_foreground(&self) -> bool { - *self == ColorMode::Foreground - } - - fn is_enabled(&self) -> bool { - *self != ColorMode::Disabled - } - - fn needs_gradient(&self) -> bool { - *self == ColorMode::Gradient || *self == ColorMode::Bar - } -} - -#[cfg_attr(test, derive(Debug))] -#[derive(Serialize, Deserialize, Clone, Default)] -pub struct ColumnStyleConfig { - #[serde(default = "ColorMode::default")] - #[serde(skip_serializing_if = "ColorMode::is_foreground")] - pub color_mode: ColorMode, - - #[serde(skip_serializing_if = "Option::is_none")] - pub fixed: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub pos_color: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub neg_color: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub gradient: Option, -} - -/// Exactly like a `ColumnStyleConfig`, except without `Option<>` fields, as -/// this struct represents the default values we should use in the GUI when they -/// are `None` in the real config. It is also used to decide when to omit a -/// field when serialized a `ColumnStyleConfig` to JSON. -#[derive(Deserialize, Clone, Default, Debug)] -pub struct ColumnStyleDefaultConfig { - pub gradient: f64, - pub fixed: u32, - pub pos_color: String, - pub neg_color: String, - - #[serde(default = "ColorMode::default")] - pub color_mode: ColorMode, -} - -pub enum ColumnStyleMsg { - Reset(ColumnStyleConfig, ColumnStyleDefaultConfig), - SetPos(i32, i32), - FixedChanged(String), - ColorEnabledChanged(bool), - PosColorChanged(String), - NegColorChanged(String), - ColorModeChanged(ColorMode), - GradientChanged(String), -} - -/// A `ColumnStyle` component is mounted to the window anchored at the screen -/// position of `elem`. It needs two input configs, the current configuration -/// object and a default version without `Option<>` -#[derive(Properties, Clone)] -pub struct ColumnStyleProps { - #[prop_or_default] - pub config: ColumnStyleConfig, - - #[prop_or_default] - pub default_config: ColumnStyleDefaultConfig, - - #[prop_or_default] - pub on_change: Callback, - - #[prop_or_default] - pub weak_link: WeakComponentLink, -} - -impl ColumnStyleProps { - /// When this config has changed, we must signal the wrapper element. - fn dispatch_config(&self) { - let config = match &self.config { - ColumnStyleConfig { - pos_color: Some(pos_color), - neg_color: Some(neg_color), - .. - } if *pos_color == self.default_config.pos_color - && *neg_color == self.default_config.neg_color => - { - ColumnStyleConfig { - pos_color: None, - neg_color: None, - ..self.config - } - } - x => x.clone(), - }; - - self.on_change.emit(config); - } - - /// Human readable precision hint, e.g. "Prec 0.001" for `{fixed: 3}`. - fn make_fixed_text(&self) -> String { - match self.config.fixed { - Some(x) if x > 0 => format!("0.{}1", "0".repeat(x as usize - 1)), - None if self.default_config.fixed > 0 => { - let n = self.default_config.fixed as usize - 1; - format!("0.{}1", "0".repeat(n)) - } - Some(_) | None => "1".to_owned(), - } - } -} - -/// The `ColumnStyle` component stores its UI state privately in its own struct, -/// rather than its props (which has two version of this data itself, the -/// JSON serializable config record and the defaults record). -pub struct ColumnStyle { - props: ColumnStyleProps, - top: i32, - left: i32, - color_mode: ColorMode, - pos_color: String, - neg_color: String, - gradient: f64, -} - -impl Component for ColumnStyle { - type Message = ColumnStyleMsg; - type Properties = ColumnStyleProps; - - fn create(props: Self::Properties, _link: ComponentLink) -> Self { - *props.weak_link.borrow_mut() = Some(_link); - ColumnStyle::reset(props) - } - - fn update(&mut self, _msg: Self::Message) -> ShouldRender { - match _msg { - ColumnStyleMsg::Reset(config, default_config) => { - let props = ColumnStyleProps { - config, - default_config, - ..self.props.clone() - }; - - std::mem::swap(self, &mut ColumnStyle::reset(props)); - true - } - ColumnStyleMsg::SetPos(top, left) => { - self.top = top; - self.left = left; - true - } - ColumnStyleMsg::FixedChanged(fixed) => { - self.props.config.fixed = match fixed.parse::() { - Ok(x) if x != self.props.default_config.fixed => Some(x), - Ok(_) => None, - Err(_) if fixed.is_empty() => Some(0), - Err(_) => None, - }; - self.props.dispatch_config(); - true - } - ColumnStyleMsg::ColorEnabledChanged(val) => { - if val { - let color_mode = match self.color_mode { - ColorMode::Disabled => ColorMode::default(), - x => x, - }; - - self.props.config.color_mode = color_mode; - self.props.config.pos_color = Some(self.pos_color.to_owned()); - self.props.config.neg_color = Some(self.neg_color.to_owned()); - if self.color_mode.needs_gradient() { - self.props.config.gradient = Some(self.gradient); - } else { - self.props.config.gradient = None; - } - } else { - self.props.config.color_mode = ColorMode::Disabled; - self.props.config.pos_color = None; - self.props.config.neg_color = None; - self.props.config.gradient = None; - } - - self.props.dispatch_config(); - true - } - ColumnStyleMsg::PosColorChanged(val) => { - self.pos_color = val; - self.props.config.pos_color = Some(self.pos_color.to_owned()); - self.props.dispatch_config(); - false - } - ColumnStyleMsg::NegColorChanged(val) => { - self.neg_color = val; - self.props.config.neg_color = Some(self.neg_color.to_owned()); - self.props.dispatch_config(); - false - } - ColumnStyleMsg::ColorModeChanged(val) => { - self.color_mode = val; - self.props.config.color_mode = val; - if self.color_mode.needs_gradient() { - self.props.config.gradient = Some(self.gradient); - } else { - self.props.config.gradient = None; - } - - self.props.dispatch_config(); - true - } - ColumnStyleMsg::GradientChanged(gradient) => { - self.props.config.gradient = match gradient.parse::() { - Ok(x) => { - self.gradient = x; - Some(x) - } - Err(_) if gradient.is_empty() => { - self.gradient = self.props.default_config.gradient; - Some(self.props.default_config.gradient) - } - Err(_) => { - self.gradient = self.props.default_config.gradient; - None - } - }; - self.props.dispatch_config(); - false - } - } - } - - fn change(&mut self, _props: Self::Properties) -> ShouldRender { - true - } - - fn view(&self) -> Html { - // Fixed precision control oninput callback - let fixed_oninput = self - .props - .weak_link - .borrow() - .as_ref() - .unwrap() - .callback(|event: InputData| ColumnStyleMsg::FixedChanged(event.value)); - - // Color enabled/disabled oninput callback - let color_enabled_oninput = - self.props.weak_link.borrow().as_ref().unwrap().callback( - move |event: InputData| { - let input = event - .event - .target() - .unwrap() - .unchecked_into::(); - ColumnStyleMsg::ColorEnabledChanged(input.checked()) - }, - ); - - let color_mode_selected = match self.color_mode { - ColorMode::Disabled => ColorMode::default(), - x => x, - }; - - // Color controls callback - let pos_color_oninput = - self.props.weak_link.borrow().as_ref().unwrap().callback( - |event: InputData| ColumnStyleMsg::PosColorChanged(event.value), - ); - - let neg_color_oninput = - self.props.weak_link.borrow().as_ref().unwrap().callback( - |event: InputData| ColumnStyleMsg::NegColorChanged(event.value), - ); - - // Color mode radio callback - let color_mode_changed = { - let link = self.props.weak_link.borrow(); - link.as_ref() - .unwrap() - .callback(ColumnStyleMsg::ColorModeChanged) - }; - - // Gradient input callback - let gradient_changed = - self.props.weak_link.borrow().as_ref().unwrap().callback( - move |event: InputData| ColumnStyleMsg::GradientChanged(event.value), - ); - - let gradient_gradient_enabled = if self.props.config.color_mode.is_enabled() - && self.color_mode == ColorMode::Gradient - { - "" - } else { - "display:none" - }; - - let bar_gradient_enabled = if self.props.config.color_mode.is_enabled() - && self.color_mode == ColorMode::Bar - { - "" - } else { - "display:none" - }; - - html! { - <> - -
-
- - { - format!("Prec {}", self.props.make_fixed_text()) - } -
-
- self.props.default_config.fixed, Some(x) => x }) } - oninput={ fixed_oninput }/> -
-
- - - { " + / - " } - -
- - - class="indent" - disabled={ !self.props.config.color_mode.is_enabled() } - values={ vec!(ColorMode::Foreground, ColorMode::Background, ColorMode::Gradient, ColorMode::Bar) } - selected={ color_mode_selected } - on_change={ color_mode_changed } > - - { "Foreground" } - { "Background" } - <> - { "Gradient" } -
- -
- - <> - { "Bar" } -
- -
- - - > - -
- - } - } -} - -impl ColumnStyle { - fn reset(mut props: ColumnStyleProps) -> ColumnStyle { - let config = &mut props.config; - let default_config = &props.default_config; - let gradient = match config.gradient { - Some(x) => x, - None => default_config.gradient, - }; - - let pos_color = config - .pos_color - .as_ref() - .unwrap_or(&default_config.pos_color) - .to_owned(); - - let neg_color = config - .neg_color - .as_ref() - .unwrap_or(&default_config.neg_color) - .to_owned(); - - let color_mode = match config.color_mode { - ColorMode::Disabled => ColorMode::default(), - x => { - config.pos_color = Some(pos_color.to_owned()); - config.neg_color = Some(neg_color.to_owned()); - x - } - }; - - ColumnStyle { - top: 0, - left: 0, - props, - color_mode, - pos_color, - neg_color, - gradient, - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - - fn make_props( - config: ColumnStyleConfig, - default_config: ColumnStyleDefaultConfig, - ) -> ColumnStyleProps { - ColumnStyleProps { - config, - default_config, - on_change: Default::default(), - weak_link: WeakComponentLink::default(), - } - } - - #[wasm_bindgen_test] - pub fn text_fixed_text_default() { - let config = ColumnStyleConfig::default(); - let default_config = ColumnStyleDefaultConfig { - fixed: 2, - ..ColumnStyleDefaultConfig::default() - }; - - let props = make_props(config, default_config); - assert!(props.make_fixed_text() == "0.01"); - } - - #[wasm_bindgen_test] - pub fn text_fixed_text_override() { - let config = ColumnStyleConfig { - fixed: Some(3), - ..ColumnStyleConfig::default() - }; - - let default_config = ColumnStyleDefaultConfig { - fixed: 3, - ..ColumnStyleDefaultConfig::default() - }; - - let props = make_props(config, default_config); - assert!(props.make_fixed_text() == "0.001"); - } - - #[wasm_bindgen_test] - pub fn text_fixed_text_zero() { - let config = ColumnStyleConfig::default(); - let default_config = ColumnStyleDefaultConfig::default(); - let props = make_props(config, default_config); - assert!(props.make_fixed_text() == "1"); - } -} diff --git a/rust/perspective-viewer/src/rust/components/config_selector.rs b/rust/perspective-viewer/src/rust/components/config_selector.rs index fcf1a1e222..738d707058 100644 --- a/rust/perspective-viewer/src/rust/components/config_selector.rs +++ b/rust/perspective-viewer/src/rust/components/config_selector.rs @@ -22,32 +22,23 @@ use super::sort_item::*; use std::rc::Rc; use yew::prelude::*; -#[derive(Properties, Clone)] +#[derive(Properties, Clone, PartialEq)] pub struct ConfigSelectorProps { pub session: Session, pub renderer: Renderer, pub dragdrop: DragDrop, - pub filter_dropdown: FilterDropDownElement, } derive_renderable_props!(ConfigSelectorProps); +#[derive(Debug)] pub enum ConfigSelectorMsg { DragStart(DragEffect), DragEnd, - DragOverRowPivots(usize), - DragOverColumnPivots(usize), - DragOverSort(usize), - DragOverFilter(usize), - DragLeaveRowPivots, - DragLeaveColumnPivots, - DragLeaveSort, - DragLeaveFilter, + DragOver(usize, DropAction), + DragLeave(DropAction), Drop(String, DropAction, DragEffect, usize), - CloseRowPivot(usize), - CloseColumnPivot(usize), - CloseSort(usize), - CloseFilter(usize), + Close(usize, DropAction), SetFilterValue(usize, String), TransposePivots, ViewCreated, @@ -55,169 +46,175 @@ pub enum ConfigSelectorMsg { #[derive(Clone)] pub struct ConfigSelector { - props: ConfigSelectorProps, - link: ComponentLink, + filter_dropdown: FilterDropDownElement, subscriptions: [Rc; 4], } -derive_dragdrop_list!( - RowPivotSelector, - RowPivotDragContext, - ConfigSelector, - PivotItem, - DragOverRowPivots, - DragLeaveRowPivots, - CloseRowPivot -); - -derive_dragdrop_list!( - ColumnPivotSelector, - ColumnPivotDragContext, - ConfigSelector, - PivotItem, - DragOverColumnPivots, - DragLeaveColumnPivots, - CloseColumnPivot -); - -derive_dragdrop_list!( - SortSelector, - SortDragContext, - ConfigSelector, - SortItem, - DragOverSort, - DragLeaveSort, - CloseSort -); - -derive_dragdrop_list!( - FilterSelector, - FilterDragContext, - ConfigSelector, - FilterItem, - DragOverFilter, - DragLeaveFilter, - CloseFilter -); +struct RowPivotContext {} +struct ColumnPivotContext {} +struct SortDragContext {} +struct FilterDragContext {} + +impl DragContext for RowPivotContext { + fn dragenter(index: usize) -> ConfigSelectorMsg { + ConfigSelectorMsg::DragOver(index, DropAction::RowPivots) + } + + fn close(index: usize) -> ConfigSelectorMsg { + ConfigSelectorMsg::Close(index, DropAction::RowPivots) + } + + fn dragleave() -> ConfigSelectorMsg { + ConfigSelectorMsg::DragLeave(DropAction::RowPivots) + } +} + +impl DragContext for ColumnPivotContext { + fn dragenter(index: usize) -> ConfigSelectorMsg { + ConfigSelectorMsg::DragOver(index, DropAction::ColumnPivots) + } + + fn close(index: usize) -> ConfigSelectorMsg { + ConfigSelectorMsg::Close(index, DropAction::ColumnPivots) + } + + fn dragleave() -> ConfigSelectorMsg { + ConfigSelectorMsg::DragLeave(DropAction::ColumnPivots) + } +} + +impl DragContext for SortDragContext { + fn dragenter(index: usize) -> ConfigSelectorMsg { + ConfigSelectorMsg::DragOver(index, DropAction::Sort) + } + + fn close(index: usize) -> ConfigSelectorMsg { + ConfigSelectorMsg::Close(index, DropAction::Sort) + } + + fn dragleave() -> ConfigSelectorMsg { + ConfigSelectorMsg::DragLeave(DropAction::Sort) + } +} + +impl DragContext for FilterDragContext { + fn dragenter(index: usize) -> ConfigSelectorMsg { + ConfigSelectorMsg::DragOver(index, DropAction::Filter) + } + + fn close(index: usize) -> ConfigSelectorMsg { + ConfigSelectorMsg::Close(index, DropAction::Filter) + } + + fn dragleave() -> ConfigSelectorMsg { + ConfigSelectorMsg::DragLeave(DropAction::Filter) + } +} + +type RowPivotSelector = DragDropList; +type ColumnPivotSelector = DragDropList; +type SortSelector = DragDropList; +type FilterSelector = DragDropList; impl Component for ConfigSelector { type Message = ConfigSelectorMsg; type Properties = ConfigSelectorProps; - fn create(props: Self::Properties, link: ComponentLink) -> Self { - let cb = link.callback(ConfigSelectorMsg::DragStart); - let drag_sub = Rc::new(props.dragdrop.add_on_drag_action(cb)); + fn create(ctx: &Context) -> Self { + let cb = ctx.link().callback(ConfigSelectorMsg::DragStart); + let drag_sub = Rc::new(ctx.props().dragdrop.add_on_drag_action(cb)); - let cb = link.callback(|_| ConfigSelectorMsg::DragEnd); - let dragend_sub = Rc::new(props.dragdrop.add_on_dragend_action(cb)); + let cb = ctx.link().callback(|_| ConfigSelectorMsg::DragEnd); + let dragend_sub = Rc::new(ctx.props().dragdrop.add_on_dragend_action(cb)); - let cb = link.callback(|x: (String, DropAction, DragEffect, usize)| { - ConfigSelectorMsg::Drop(x.0, x.1, x.2, x.3) - }); - let drop_sub = Rc::new(props.dragdrop.add_on_drop_action(cb)); + let cb = ctx + .link() + .callback(|x: (String, DropAction, DragEffect, usize)| { + ConfigSelectorMsg::Drop(x.0, x.1, x.2, x.3) + }); + let drop_sub = Rc::new(ctx.props().dragdrop.add_on_drop_action(cb)); - let cb = link.callback(|_| ConfigSelectorMsg::ViewCreated); - let view_sub = Rc::new(props.session.on_view_created.add_listener(cb)); + let cb = ctx.link().callback(|_| ConfigSelectorMsg::ViewCreated); + let view_sub = Rc::new(ctx.props().session.on_view_created.add_listener(cb)); + let filter_dropdown = FilterDropDownElement::new(ctx.props().session.clone()); let subscriptions = [drop_sub, view_sub, drag_sub, dragend_sub]; ConfigSelector { - props, - link, + filter_dropdown, subscriptions, } } - fn update(&mut self, msg: Self::Message) -> ShouldRender { + fn update(&mut self, ctx: &Context, msg: Self::Message) -> bool { match msg { ConfigSelectorMsg::DragStart(_) | ConfigSelectorMsg::ViewCreated => true, ConfigSelectorMsg::DragEnd => true, - ConfigSelectorMsg::DragOverRowPivots(index) => { - self.props.dragdrop.drag_enter(DropAction::RowPivots, index) - } - ConfigSelectorMsg::DragOverColumnPivots(index) => self - .props - .dragdrop - .drag_enter(DropAction::ColumnPivots, index), - ConfigSelectorMsg::DragOverSort(index) => { - self.props.dragdrop.drag_enter(DropAction::Sort, index) - } - ConfigSelectorMsg::DragOverFilter(index) => { - self.props.dragdrop.drag_enter(DropAction::Filter, index) - } - ConfigSelectorMsg::DragLeaveRowPivots => { - self.props.dragdrop.drag_leave(DropAction::RowPivots); - true - } - ConfigSelectorMsg::DragLeaveColumnPivots => { - self.props.dragdrop.drag_leave(DropAction::ColumnPivots); - true - } - ConfigSelectorMsg::DragLeaveSort => { - self.props.dragdrop.drag_leave(DropAction::Sort); - true + ConfigSelectorMsg::DragOver(index, action) => { + ctx.props().dragdrop.drag_enter(action, index) } - ConfigSelectorMsg::DragLeaveFilter => { - self.props.dragdrop.drag_leave(DropAction::Filter); + ConfigSelectorMsg::DragLeave(action) => { + ctx.props().dragdrop.drag_leave(action); true } - ConfigSelectorMsg::CloseSort(index) => { - let ViewConfig { mut sort, .. } = self.props.session.get_view_config(); + ConfigSelectorMsg::Close(index, DropAction::Sort) => { + let ViewConfig { mut sort, .. } = ctx.props().session.get_view_config(); sort.remove(index as usize); let sort = Some(sort); - self.props.update_and_render(ViewConfigUpdate { + ctx.props().update_and_render(ViewConfigUpdate { sort, ..ViewConfigUpdate::default() }); true } - ConfigSelectorMsg::CloseRowPivot(index) => { + ConfigSelectorMsg::Close(index, DropAction::RowPivots) => { let ViewConfig { mut row_pivots, .. } = - self.props.session.get_view_config(); + ctx.props().session.get_view_config(); row_pivots.remove(index as usize); let row_pivots = Some(row_pivots); - self.props.update_and_render(ViewConfigUpdate { + ctx.props().update_and_render(ViewConfigUpdate { row_pivots, ..ViewConfigUpdate::default() }); true } - ConfigSelectorMsg::CloseColumnPivot(index) => { + ConfigSelectorMsg::Close(index, DropAction::ColumnPivots) => { let ViewConfig { mut column_pivots, .. - } = self.props.session.get_view_config(); + } = ctx.props().session.get_view_config(); column_pivots.remove(index as usize); - self.props.update_and_render(ViewConfigUpdate { + ctx.props().update_and_render(ViewConfigUpdate { column_pivots: Some(column_pivots), ..ViewConfigUpdate::default() }); true } - ConfigSelectorMsg::CloseFilter(index) => { - self.props.filter_dropdown.hide().unwrap(); + ConfigSelectorMsg::Close(index, DropAction::Filter) => { + self.filter_dropdown.hide().unwrap(); let ViewConfig { mut filter, .. } = - self.props.session.get_view_config(); + ctx.props().session.get_view_config(); filter.remove(index as usize); - self.props.update_and_render(ViewConfigUpdate { + ctx.props().update_and_render(ViewConfigUpdate { filter: Some(filter), ..ViewConfigUpdate::default() }); true } + ConfigSelectorMsg::Close(_, _) => false, ConfigSelectorMsg::Drop(column, action, effect, index) if action != DropAction::Active => { - let update = self.props.session.create_drag_drop_update( + let update = ctx.props().session.create_drag_drop_update( column, index, action, effect, - &self.props.renderer.metadata(), + &ctx.props().renderer.metadata(), ); - self.props.update_and_render(update); + ctx.props().update_and_render(update); true } ConfigSelectorMsg::Drop(_, _, DragEffect::Move(action), _) @@ -227,7 +224,7 @@ impl Component for ConfigSelector { } ConfigSelectorMsg::Drop(_, _, _, _) => false, ConfigSelectorMsg::TransposePivots => { - let mut view_config = self.props.session.get_view_config(); + let mut view_config = ctx.props().session.get_view_config(); std::mem::swap( &mut view_config.row_pivots, &mut view_config.column_pivots, @@ -239,12 +236,12 @@ impl Component for ConfigSelector { ..ViewConfigUpdate::default() }; - self.props.update_and_render(update); + ctx.props().update_and_render(update); true } ConfigSelectorMsg::SetFilterValue(index, input) => { let ViewConfig { mut filter, .. } = - self.props.session.get_view_config(); + ctx.props().session.get_view_config(); filter[index].2 = FilterTerm::Scalar(Scalar::String(input)); @@ -254,29 +251,29 @@ impl Component for ConfigSelector { ..ViewConfigUpdate::default() }; - self.props.update_and_render(update); + ctx.props().update_and_render(update); false } } } /// Should not render on change, as this component only depends on service state. - fn change(&mut self, _props: Self::Properties) -> ShouldRender { + fn changed(&mut self, _ctx: &Context) -> bool { false } - fn view(&self) -> Html { - let config = self.props.session.get_view_config(); - let transpose = self.link.callback(|_| ConfigSelectorMsg::TransposePivots); + fn view(&self, ctx: &Context) -> Html { + let config = ctx.props().session.get_view_config(); + let transpose = ctx.link().callback(|_| ConfigSelectorMsg::TransposePivots); - let class = if self.props.dragdrop.get_drag_column().is_some() { + let class = if ctx.props().dragdrop.get_drag_column().is_some() { "dragdrop-highlight" } else { "" }; let dragend = Callback::from({ - let dragdrop = self.props.dragdrop.clone(); + let dragdrop = ctx.props().dragdrop.clone(); move |_event| dragdrop.drag_end() }); @@ -285,14 +282,14 @@ impl Component for ConfigSelector { + parent={ ctx.link().clone() } + is_dragover={ ctx.props().dragdrop.is_dragover(DropAction::RowPivots) } + dragdrop={ ctx.props().dragdrop.clone() }> { for config.row_pivots.iter().map(|row_pivot| { html_nested! { @@ -312,14 +309,14 @@ impl Component for ConfigSelector { + parent={ ctx.link().clone() } + is_dragover={ ctx.props().dragdrop.is_dragover(DropAction::ColumnPivots) } + dragdrop={ ctx.props().dragdrop.clone() }> { for config.column_pivots.iter().map(|column_pivot| { html_nested! { @@ -331,9 +328,9 @@ impl Component for ConfigSelector { { @@ -341,9 +338,9 @@ impl Component for ConfigSelector { html_nested! { } @@ -354,24 +351,23 @@ impl Component for ConfigSelector { { for config.filter.iter().enumerate().map(|(idx, filter)| { - let filter_keydown = self - .link + let filter_keydown = ctx.link() .callback(move |txt| ConfigSelectorMsg::SetFilterValue(idx, txt)); html_nested! { diff --git a/rust/perspective-viewer/src/rust/components/containers/dragdrop_list.rs b/rust/perspective-viewer/src/rust/components/containers/dragdrop_list.rs index 3186636a10..0ca4add5a2 100644 --- a/rust/perspective-viewer/src/rust/components/containers/dragdrop_list.rs +++ b/rust/perspective-viewer/src/rust/components/containers/dragdrop_list.rs @@ -9,32 +9,11 @@ use crate::dragdrop::*; use derivative::Derivative; +use std::marker::PhantomData; use web_sys::*; +use yew::html::Scope; use yew::prelude::*; -#[macro_export] -macro_rules! derive_dragdrop_list { - ($name:ident, $context:ident, $parent:ident, $item:ident, $dragenter:ident, $dragleave:ident, $close:ident) => { - struct $context {} - - impl DragContext<<$parent as Component>::Message> for $context { - fn dragenter(index: usize) -> ConfigSelectorMsg { - <$parent as Component>::Message::$dragenter(index) - } - - fn close(index: usize) -> ConfigSelectorMsg { - <$parent as Component>::Message::$close(index) - } - - fn dragleave() -> ConfigSelectorMsg { - <$parent as Component>::Message::$dragleave - } - } - - type $name = DragDropList<$parent, $item, $context>; - }; -} - /// Must be implemented by `Properties` of children of `DragDropList`, returning /// the value a DragDropItem represents. pub trait DragDropListItemProps: Properties { @@ -56,7 +35,7 @@ where U: Component, ::Properties: DragDropListItemProps, { - pub parent: ComponentLink, + pub parent: Scope, pub dragdrop: DragDrop, pub name: &'static str, pub children: ChildrenWithProps, @@ -71,6 +50,20 @@ where pub allow_duplicates: bool, } +impl PartialEq for DragDropListProps +where + T: Component, + U: Component, + ::Properties: DragDropListItemProps, +{ + fn eq(&self, other: &Self) -> bool { + self.name == other.name + && self.children == other.children + && self.allow_duplicates == other.allow_duplicates + && self.is_dragover == other.is_dragover + } +} + pub enum DragDropListMsg { Freeze(bool), } @@ -89,8 +82,9 @@ where ::Properties: DragDropListItemProps, V: DragContext + 'static, { - props: DragDropListProps, - link: ComponentLink, + parent_type: PhantomData, + item_type: PhantomData, + draggable_type: PhantomData, elem: NodeRef, frozen_size: Option, } @@ -105,16 +99,21 @@ where type Message = DragDropListMsg; type Properties = DragDropListProps; - fn create(props: Self::Properties, link: ComponentLink) -> Self { + fn create(_ctx: &Context) -> Self { DragDropList { - props, - link, + parent_type: PhantomData, + item_type: PhantomData, + draggable_type: PhantomData, elem: NodeRef::default(), frozen_size: None, } } - fn update(&mut self, msg: Self::Message) -> ShouldRender { + fn changed(&mut self, _ctx: &Context) -> bool { + true + } + + fn update(&mut self, _ctx: &Context, msg: Self::Message) -> bool { match msg { // When a dragover occurs and a new Column is inserted into the selector, // the geometry of the selector may expand and cause a parent reflow, @@ -152,21 +151,14 @@ where } } - /// Should always render on change, as this component only depends on the props from - /// its parent and has no `Msg` enums. - fn change(&mut self, props: Self::Properties) -> ShouldRender { - self.props = props; - true - } - - fn view(&self) -> Html { + fn view(&self, ctx: &Context) -> Html { let dragover = Callback::from(|_event: DragEvent| _event.prevent_default()); // On dragleave, signal the parent but no need to redraw as parent will call // `change()` when resetting props. let dragleave = dragleave_helper({ - let parent = self.props.parent.clone(); - let link = self.link.clone(); + let parent = ctx.props().parent.clone(); + let link = ctx.link().clone(); move || { link.send_message(DragDropListMsg::Freeze(false)); parent.send_message(V::dragleave()) @@ -174,8 +166,8 @@ where }); let drop = Callback::from({ - let dragdrop = self.props.dragdrop.clone(); - let link = self.link.clone(); + let dragdrop = ctx.props().dragdrop.clone(); + let link = ctx.link().clone(); move |_| { link.send_message(DragDropListMsg::Freeze(false)); dragdrop.notify_drop(); @@ -183,15 +175,15 @@ where }); let columns_html = { - let mut columns = self - .props + let mut columns = ctx + .props() .children .iter() .map(|x| (true, Some(x))) .enumerate() .collect::>))>>(); - if let Some((x, column)) = &self.props.is_dragover { + if let Some((x, column)) = &ctx.props().is_dragover { let index = *x as usize; let col_vchild = columns .iter() @@ -199,7 +191,7 @@ where .find(|x| x.props.get_item() == *column) .cloned(); - if !self.props.allow_duplicates { + if !ctx.props().allow_duplicates { columns.retain(|x| { x.1 .1.as_ref().unwrap().props.get_item() != *column }); @@ -218,9 +210,9 @@ where columns .into_iter() .map(|(idx, column)| { - let close = self.props.parent.callback(move |_| V::close(idx)); - let dragenter = self.props.parent.callback({ - let link = self.link.clone(); + let close = ctx.props().parent.callback(move |_| V::close(idx)); + let dragenter = ctx.props().parent.callback({ + let link = ctx.link().clone(); move |event: DragEvent| { event.stop_propagation(); event.prevent_default(); @@ -256,9 +248,9 @@ where .collect::() }; - let total = self.props.children.len(); - let dragenter = self.props.parent.callback({ - let link = self.link.clone(); + let total = ctx.props().children.len(); + let dragenter = ctx.props().parent.callback({ + let link = ctx.link().clone(); move |event: DragEvent| { dragenter_helper(event); link.send_message(DragDropListMsg::Freeze(true)); @@ -274,17 +266,17 @@ where html! {
-
    +
      { columns_html }
    - +
diff --git a/rust/perspective-viewer/src/rust/components/containers/dropdown.rs b/rust/perspective-viewer/src/rust/components/containers/dropdown.rs index 27d9c16612..dad9ccc56b 100644 --- a/rust/perspective-viewer/src/rust/components/containers/dropdown.rs +++ b/rust/perspective-viewer/src/rust/components/containers/dropdown.rs @@ -9,6 +9,7 @@ use std::fmt::Debug; use std::fmt::Display; use std::str::FromStr; +use wasm_bindgen::JsCast; use yew::prelude::*; #[derive(Clone, PartialEq)] @@ -55,9 +56,8 @@ where T: Clone + Display + FromStr + PartialEq + 'static, T::Err: Clone + Debug + 'static, { - props: DropDownProps, select_ref: NodeRef, - link: ComponentLink, + selected: T, } impl Component for DropDown @@ -68,64 +68,63 @@ where type Message = DropDownMsg; type Properties = DropDownProps; - fn create(props: Self::Properties, link: ComponentLink) -> Self { - DropDown { - props, - link, + fn create(_ctx: &Context) -> Self { + DropDown:: { select_ref: NodeRef::default(), + selected: _ctx.props().selected.clone(), } } - fn update(&mut self, msg: Self::Message) -> ShouldRender { + fn update(&mut self, ctx: &Context, msg: Self::Message) -> bool { let DropDownMsg::SelectedChanged(x) = msg; - self.props.selected = x; - self.props.on_select.emit(self.props.selected.clone()); + self.selected = x; + ctx.props().on_select.emit(self.selected.clone()); true } - fn change(&mut self, props: Self::Properties) -> ShouldRender { - let should_render = self.props != props; - self.props = props; - should_render - } + // fn changed(&mut self, props: Self::Properties) -> bool { + // let should_render = ctx.props() != props; + // ctx.props() = props; + // should_render + // } // Annoyingly, ` { - for self.props.values.iter().map(|value| match value { + for ctx.props().values.iter().map(|value| match value { DropDownItem::Option(value) => { - let selected = *value == self.props.selected; + let selected = *value == ctx.props().selected; html! {