add string(), parse strings in integer() and float()

string() parse strings in integer() and float()
finos · Jun 4, 2021 · f705593 · f705593
1 parent 8b974a4
commit f705593
Show file tree

Hide file tree

Showing 8 changed files with 399 additions and 45 deletions.
diff --git a/cpp/perspective/src/cpp/computed_expression.cpp b/cpp/perspective/src/cpp/computed_expression.cpp
@@ -74,6 +74,9 @@ t_computed_expression_parser::LOWER_VALIDATOR_FN = computed_function::lower(null
 computed_function::length
 t_computed_expression_parser::LENGTH_VALIDATOR_FN = computed_function::length(nullptr);
 
+computed_function::to_string
+t_computed_expression_parser::TO_STRING_VALIDATOR_FN = computed_function::to_string(nullptr);
+
 #define REGISTER_COMPUTE_FUNCTIONS(vocab)                                                        \
     computed_function::day_of_week day_of_week_fn = computed_function::day_of_week(vocab);         \
     computed_function::month_of_year month_of_year_fn = computed_function::month_of_year(vocab);   \
@@ -83,6 +86,7 @@ t_computed_expression_parser::LENGTH_VALIDATOR_FN = computed_function::length(nu
     computed_function::upper upper_fn = computed_function::upper(vocab);       \
     computed_function::lower lower_fn = computed_function::lower(vocab);       \
     computed_function::length length_fn = computed_function::length(vocab);    \
+    computed_function::to_string to_string_fn = computed_function::to_string(vocab);        \
     sym_table.add_function("today", computed_function::today);                              \
     sym_table.add_function("now", computed_function::now);                                  \
     sym_table.add_function("bucket", t_computed_expression_parser::BUCKET_FN);              \
@@ -95,6 +99,7 @@ t_computed_expression_parser::LENGTH_VALIDATOR_FN = computed_function::length(nu
     sym_table.add_function("upper", upper_fn);                                              \
     sym_table.add_function("lower", lower_fn);                                              \
     sym_table.add_function("length", length_fn);                                            \
+    sym_table.add_function("string", to_string_fn);                                         \
     sym_table.add_reserved_function("min", t_computed_expression_parser::MIN_FN);           \
     sym_table.add_reserved_function("max", t_computed_expression_parser::MAX_FN);           \
     sym_table.add_function("percent_of", t_computed_expression_parser::PERCENT_OF_FN);      \
@@ -123,6 +128,7 @@ t_computed_expression_parser::LENGTH_VALIDATOR_FN = computed_function::length(nu
     sym_table.add_function("percent_of", t_computed_expression_parser::PERCENT_OF_FN);      \
     sym_table.add_function("is_null", t_computed_expression_parser::IS_NULL_FN);            \
     sym_table.add_function("is_not_null", t_computed_expression_parser::IS_NOT_NULL_FN);    \
+    sym_table.add_function("string", t_computed_expression_parser::TO_STRING_VALIDATOR_FN); \
     sym_table.add_function("integer", t_computed_expression_parser::TO_INTEGER_FN);         \
     sym_table.add_function("float", t_computed_expression_parser::TO_FLOAT_FN);             \
     sym_table.add_function("date", t_computed_expression_parser::MAKE_DATE_FN);             \

diff --git a/cpp/perspective/src/cpp/computed_function.cpp b/cpp/perspective/src/cpp/computed_function.cpp
@@ -1189,6 +1189,50 @@ t_tscalar is_not_null::operator()(t_parameter_list parameters) {
     return rval;
 }
 
+to_string::to_string(std::shared_ptr<t_vocab> expression_vocab)
+    : exprtk::igeneric_function<t_tscalar>("T")
+    , m_expression_vocab(expression_vocab)  {
+        t_tscalar sentinel;
+        sentinel.clear();
+
+        // The sentinel is a string scalar that is returned to indicate a
+        // valid call to the function without actually computing any values.
+        sentinel.m_type = DTYPE_STR;
+        sentinel.m_data.m_charptr = nullptr;
+        m_sentinel = sentinel;
+    }
+
+to_string::~to_string() {}
+
+t_tscalar to_string::operator()(t_parameter_list parameters) {
+    t_tscalar val;
+    t_tscalar rval;
+    rval.clear();
+    rval.m_type = DTYPE_STR;
+    std::string temp_str;
+
+    t_generic_type& gt = parameters[0];
+    t_scalar_view temp(gt);
+    val.set(temp());
+
+    if (!val.is_valid()) {
+        return rval;
+    }
+
+    temp_str = val.to_string();
+
+    // don't try to intern an empty string as it will throw an error, but
+    // by this point we know the params are valid - so return the sentinel
+    // string value.
+    if (temp_str == "" || m_expression_vocab == nullptr) {
+        return m_sentinel;
+    }
+
+    t_uindex interned = m_expression_vocab->get_interned(temp_str);
+    rval.set(m_expression_vocab->unintern_c(interned));
+    return rval;
+}
+
 to_integer::to_integer()
     : exprtk::igeneric_function<t_tscalar>("T") {}
 
@@ -1210,26 +1254,31 @@ t_tscalar to_integer::operator()(t_parameter_list parameters) {
     t_scalar_view temp(gt);
     val.set(temp());
 
-    if (val.get_dtype() == DTYPE_STR) {
-        rval.m_status = STATUS_CLEAR;
-        return rval;
-    }
-
     if (!val.is_valid()) {
         return rval;
     }
 
-    double value = val.to_double();
+    double number = 0;
+
+    // Parse numbers inside strings
+    if (val.get_dtype() == DTYPE_STR) {
+        std::stringstream ss(val.to_string());
+        ss >> number;
+
+        if (ss.fail()) return rval;
+    } else {
+        number = val.to_double();
+    }
 
 #ifdef PSP_ENABLE_WASM
     // check for overflow
-    if (value > std::numeric_limits<std::int32_t>::max() || value < std::numeric_limits<std::int32_t>::min()) {
+    if (number > std::numeric_limits<std::int32_t>::max() || number < std::numeric_limits<std::int32_t>::min()) {
         return rval;
     }
 
-    rval.set(static_cast<std::int32_t>(value));
+    rval.set(static_cast<std::int32_t>(number));
 #else
-    rval.set(static_cast<std::int64_t>(value));
+    rval.set(static_cast<std::int64_t>(number));
 #endif
 
     return rval;
@@ -1250,16 +1299,28 @@ t_tscalar to_float::operator()(t_parameter_list parameters) {
     t_scalar_view temp(gt);
     val.set(temp());
 
-    if (val.get_dtype() == DTYPE_STR) {
-        rval.m_status = STATUS_CLEAR;
+    if (!val.is_valid()) {
         return rval;
     }
 
-    if (!val.is_valid()) {
+    double number = 0;
+
+    // Parse numbers inside strings
+    if (val.get_dtype() == DTYPE_STR) {
+        std::stringstream ss(val.to_string());
+        ss >> number;
+
+        if (ss.fail()) return rval;
+    } else {
+        number = val.to_double();
+    }
+
+    // Don't allow NaN to leak
+    if (std::isnan(number)) {
         return rval;
     }
 
-    rval.set(val.to_double());
+    rval.set(number);
     return rval;
 }
 

diff --git a/cpp/perspective/src/include/perspective/computed_expression.h b/cpp/perspective/src/include/perspective/computed_expression.h
@@ -136,6 +136,7 @@ class PERSPECTIVE_EXPORT t_computed_expression_parser {
     static computed_function::max_fn MAX_FN;
     static computed_function::is_null IS_NULL_FN;
     static computed_function::is_not_null IS_NOT_NULL_FN;
+    static computed_function::to_string TO_STRING_VALIDATOR_FN;
     static computed_function::to_integer TO_INTEGER_FN;
     static computed_function::to_float TO_FLOAT_FN;
     static computed_function::make_date MAKE_DATE_FN;

diff --git a/cpp/perspective/src/include/perspective/computed_function.h b/cpp/perspective/src/include/perspective/computed_function.h
@@ -198,19 +198,18 @@ FUNCTION_HEADER(is_not_null)
  * @brief Convert a column or scalar of any type to a string.
  * 
  */
-FUNCTION_HEADER(to_string)
+STRING_FUNCTION_HEADER(to_string)
 
 /**
- * @brief Convert a column or scalar of a non-string type to an integer, or null
- * if the value is not parsable as an integer. If a string is passed in,
- * return 0.
+ * @brief Convert a column or scalar to an integer, or null if the value is not
+ * parsable as an integer. In the WASM runtime, the integer is 32-bit and will
+ * return none if the result under/over flows.
  */
 FUNCTION_HEADER(to_integer)
 
 /**
- * @brief Convert a column or scalar of a non-string type to a float, or null
- * if the value is not parsable as an float. If a string is passed in,
- * return 0.
+ * @brief Convert a column or scalar to a float, or null if the value is not
+ * parsable as an float.
  */
 FUNCTION_HEADER(to_float)
 

diff --git a/examples/workspace/src/index.js b/examples/workspace/src/index.js
@@ -19,7 +19,7 @@ const datasource = async () => {
     const resp = await req;
     const buffer = await resp.arrayBuffer();
     const worker = perspective.shared_worker();
-    return await worker.table(buffer, {limit: 10});
+    return await worker.table(buffer);
 };
 
 const DEFAULT_LAYOUT = {