parse strings in integer() and float()

finos · Jun 4, 2021 · ec35801 · ec35801
1 parent bbbc2eb
commit ec35801
Show file tree

Hide file tree

Showing 4 changed files with 170 additions and 43 deletions.
diff --git a/cpp/perspective/src/cpp/computed_function.cpp b/cpp/perspective/src/cpp/computed_function.cpp
@@ -1254,26 +1254,31 @@ t_tscalar to_integer::operator()(t_parameter_list parameters) {
     t_scalar_view temp(gt);
     val.set(temp());
 
-    if (val.get_dtype() == DTYPE_STR) {
-        rval.m_status = STATUS_CLEAR;
-        return rval;
-    }
-
     if (!val.is_valid()) {
         return rval;
     }
 
-    double value = val.to_double();
+    double number = 0;
+
+    // Parse numbers inside strings
+    if (val.get_dtype() == DTYPE_STR) {
+        std::stringstream ss(val.to_string());
+        ss >> number;
+
+        if (ss.fail()) return rval;
+    } else {
+        number = val.to_double();
+    }
 
 #ifdef PSP_ENABLE_WASM
     // check for overflow
-    if (value > std::numeric_limits<std::int32_t>::max() || value < std::numeric_limits<std::int32_t>::min()) {
+    if (number > std::numeric_limits<std::int32_t>::max() || number < std::numeric_limits<std::int32_t>::min()) {
         return rval;
     }
 
-    rval.set(static_cast<std::int32_t>(value));
+    rval.set(static_cast<std::int32_t>(number));
 #else
-    rval.set(static_cast<std::int64_t>(value));
+    rval.set(static_cast<std::int64_t>(number));
 #endif
 
     return rval;
@@ -1294,16 +1299,28 @@ t_tscalar to_float::operator()(t_parameter_list parameters) {
     t_scalar_view temp(gt);
     val.set(temp());
 
-    if (val.get_dtype() == DTYPE_STR) {
-        rval.m_status = STATUS_CLEAR;
+    if (!val.is_valid()) {
         return rval;
     }
 
-    if (!val.is_valid()) {
+    double number = 0;
+
+    // Parse numbers inside strings
+    if (val.get_dtype() == DTYPE_STR) {
+        std::stringstream ss(val.to_string());
+        ss >> number;
+
+        if (ss.fail()) return rval;
+    } else {
+        number = val.to_double();
+    }
+
+    // Don't allow NaN to leak
+    if (std::isnan(number)) {
         return rval;
     }
 
-    rval.set(val.to_double());
+    rval.set(number);
     return rval;
 }
 

diff --git a/cpp/perspective/src/include/perspective/computed_function.h b/cpp/perspective/src/include/perspective/computed_function.h
@@ -201,16 +201,15 @@ FUNCTION_HEADER(is_not_null)
 STRING_FUNCTION_HEADER(to_string)
 
 /**
- * @brief Convert a column or scalar of a non-string type to an integer, or null
- * if the value is not parsable as an integer. If a string is passed in,
- * return 0.
+ * @brief Convert a column or scalar to an integer, or null if the value is not
+ * parsable as an integer. In the WASM runtime, the integer is 32-bit and will
+ * return none if the result under/over flows.
  */
 FUNCTION_HEADER(to_integer)
 
 /**
- * @brief Convert a column or scalar of a non-string type to a float, or null
- * if the value is not parsable as an float. If a string is passed in,
- * return 0.
+ * @brief Convert a column or scalar to a float, or null if the value is not
+ * parsable as an float.
  */
 FUNCTION_HEADER(to_float)
 

diff --git a/packages/perspective/test/js/expressions/conversions.js b/packages/perspective/test/js/expressions/conversions.js
@@ -112,11 +112,18 @@ module.exports = perspective => {
                     `//computed5\ninteger(0.00001)`,
                     `//computed6\ninteger(1.9999999999)`,
                     `//computed7\ninteger(2147483647)`,
-                    `//computed8\ninteger(-2147483647)`
+                    `//computed8\ninteger(-2147483647)`,
+                    `//computed9\ninteger('957187281')`,
+                    `//computed10\ninteger('1928.2817')`,
+                    `//computed11\ninteger('1234abcd')`,
+                    `//computed12\ninteger('abcdefg1234')`,
+                    `//computed13\ninteger('2147483648')`,
+                    `//computed14\ninteger('-2147483649')`
                 ]
             });
 
             const result = await view.to_columns();
+
             expect(result["computed"]).toEqual([999999]);
             expect(result["computed2"]).toEqual([null]);
             expect(result["computed3"]).toEqual([null]);
@@ -125,6 +132,15 @@ module.exports = perspective => {
             expect(result["computed6"]).toEqual([1]);
             expect(result["computed7"]).toEqual([2147483647]);
             expect(result["computed8"]).toEqual([-2147483647]);
+            expect(result["computed9"]).toEqual([957187281]);
+            expect(result["computed10"]).toEqual([1928]);
+
+            // unparsable strings
+            expect(result["computed11"]).toEqual([null]);
+            expect(result["computed12"]).toEqual([null]);
+            // check for overflow
+            expect(result["computed13"]).toEqual([null]);
+            expect(result["computed14"]).toEqual([null]);
 
             await view.delete();
             await table.delete();
@@ -218,10 +234,17 @@ module.exports = perspective => {
             await table.delete();
         });
 
-        it("Should validate types", async () => {
-            const table = await perspective.table({x: "string"});
-            const validated = await table.validate_expressions([`integer('abc')`, `integer("x")`]);
-            expect(validated.expression_schema).toEqual({});
+        it("Should create integers from string columns", async () => {
+            const table = await perspective.table({x: ["1", "2", "3", "abc", "4.5", "0.101928317581729083", "-123456"]});
+
+            const view = await table.view({
+                expressions: [`//computed\ninteger("x")`]
+            });
+
+            const result = await view.to_columns();
+            expect(result["computed"]).toEqual([1, 2, 3, null, 4, 0, -123456]);
+
+            await view.delete();
             await table.delete();
         });
     });
@@ -238,7 +261,17 @@ module.exports = perspective => {
                     `//computed5\n float(0.00001)`,
                     `//computed6\n float(1.9999999992)`,
                     `//computed7\n float(2147483647.1234567)`,
-                    `//computed8\n float(-2147483647)`
+                    `//computed8\n float(-2147483647)`,
+                    `//computed9\n float('957187281.00000001')`,
+                    `//computed10\n float('1928.2817')`,
+                    `//computed11\n float('abcdefg')`,
+                    `//computed12\n float('abcdefg1234.123125')`,
+                    `//computed13\n float('2147483648.1234566')`,
+                    `//computed14\n float('-2147483649')`,
+                    `//computed15\n float('inf')`,
+                    `//computed16\n float('-inf')`,
+                    `//computed17\n float(inf)`,
+                    `//computed18\n float(-inf)`
                 ]
             });
 
@@ -250,7 +283,17 @@ module.exports = perspective => {
                 computed5: "float",
                 computed6: "float",
                 computed7: "float",
-                computed8: "float"
+                computed8: "float",
+                computed9: "float",
+                computed10: "float",
+                computed11: "float",
+                computed12: "float",
+                computed13: "float",
+                computed14: "float",
+                computed15: "float",
+                computed16: "float",
+                computed17: "float",
+                computed18: "float"
             });
 
             const result = await view.to_columns();
@@ -262,6 +305,16 @@ module.exports = perspective => {
             expect(result["computed6"]).toEqual([1.9999999992]);
             expect(result["computed7"]).toEqual([2147483647.1234567]);
             expect(result["computed8"]).toEqual([-2147483647]);
+            expect(result["computed9"]).toEqual([957187281.00000001]);
+            expect(result["computed10"]).toEqual([1928.2817]);
+            expect(result["computed11"]).toEqual([null]);
+            expect(result["computed12"]).toEqual([null]);
+            expect(result["computed13"]).toEqual([2147483648.1234566]);
+            expect(result["computed14"]).toEqual([-2147483649]);
+            expect(result["computed15"]).toEqual([Infinity]);
+            expect(result["computed16"]).toEqual([-Infinity]);
+            expect(result["computed17"]).toEqual([Infinity]);
+            expect(result["computed18"]).toEqual([-Infinity]);
 
             await view.delete();
             await table.delete();
@@ -354,10 +407,19 @@ module.exports = perspective => {
             await table.delete();
         });
 
-        it("Should validate types", async () => {
-            const table = await perspective.table({x: "string"});
-            const validated = await table.validate_expressions([`float('abc')`, `float("x")`]);
-            expect(validated.expression_schema).toEqual({});
+        it("Should create floats from string columns", async () => {
+            const table = await perspective.table({
+                x: ["1.1238757869112321", "2.0000001", "abcdefg1234.12878591", "12354.1827389abc", "4.555555555", "0.101928317581729083", "-123456.21831729054781"]
+            });
+
+            const view = await table.view({
+                expressions: [`//computed\n float("x")`]
+            });
+
+            const result = await view.to_columns();
+            expect(result["computed"]).toEqual([1.1238757869112321, 2.0000001, null, null, 4.555555555, 0.101928317581729083, -123456.21831729054781]);
+
+            await view.delete();
             await table.delete();
         });
     });

diff --git a/rust/perspective-vieux/src/rust/exprtk.rs b/rust/perspective-vieux/src/rust/exprtk.rs
@@ -36,6 +36,13 @@ thread_local! {
 
     static COMPLETIONS: RegisterCompletionItemSuggestions = RegisterCompletionItemSuggestions {
         suggestions: vec![
+            CompletionItemSuggestion {
+                label: "var",
+                kind: 17,
+                insert_text: "var ${1:x := 1}",
+                insert_text_rules: 4,
+                documentation: "Declare a new local variable",
+            },
             CompletionItemSuggestion {
                 label: "abs",
                 kind: 1,
@@ -53,7 +60,7 @@ thread_local! {
             CompletionItemSuggestion {
                 label: "bucket",
                 kind: 1,
-                insert_text: "bucket(${1:x})",
+                insert_text: "bucket(${1:x}, ${2:y})",
                 insert_text_rules: 4,
                 documentation: "Bucket x by y",
             },
@@ -130,7 +137,7 @@ thread_local! {
             CompletionItemSuggestion {
                 label: "logn",
                 kind: 1,
-                insert_text: "logn(${1:x})",
+                insert_text: "logn(${1:x}, ${2:N})",
                 insert_text_rules: 4,
                 documentation: "Base N log of x where N >= 0",
             },
@@ -165,14 +172,14 @@ thread_local! {
             CompletionItemSuggestion {
                 label: "pow",
                 kind: 1,
-                insert_text: "pow(${1:x})",
+                insert_text: "pow(${1:x}, ${2:y})",
                 insert_text_rules: 4,
                 documentation: "x to the power of y",
             },
             CompletionItemSuggestion {
                 label: "root",
                 kind: 1,
-                insert_text: "root(${1:x})",
+                insert_text: "root(${1:x}, ${2:N})",
                 insert_text_rules: 4,
                 documentation: "N-th root of x where N >= 0",
             },
@@ -340,7 +347,7 @@ thread_local! {
             CompletionItemSuggestion {
                 label: "concat",
                 kind: 1,
-                insert_text: "concat(${1:x})",
+                insert_text: "concat(${1:x}, ${2:y})",
                 insert_text_rules: 4,
                 documentation: "Concatenate string literals and columns",
             },
@@ -382,14 +389,14 @@ thread_local! {
             CompletionItemSuggestion {
                 label: "now",
                 kind: 1,
-                insert_text: "now(${1:x})",
+                insert_text: "now()",
                 insert_text_rules: 4,
                 documentation: "The current datetime in local time",
             },
             CompletionItemSuggestion {
                 label: "today",
                 kind: 1,
-                insert_text: "today(${1:x})",
+                insert_text: "today()",
                 insert_text_rules: 4,
                 documentation: "The current date in local time",
             },
@@ -429,19 +436,61 @@ thread_local! {
                 documentation: "Boolean value false",
             },
             CompletionItemSuggestion {
-                label: "if else",
-                kind: 1,
-                insert_text: "if else(${1:x})",
+                label: "if",
+                kind: 17,
+                insert_text: "if (${1:condition}) {} else {}",
+                insert_text_rules: 4,
+                documentation: "If/else conditional",
+            },
+            CompletionItemSuggestion {
+                label: "else if",
+                kind: 17,
+                insert_text: "else if (${1:condition}) {}",
                 insert_text_rules: 4,
-                documentation: "if/else conditional",
+                documentation: "Else if conditional",
             },
             CompletionItemSuggestion {
                 label: "for",
-                kind: 1,
-                insert_text: "for(${1:x})",
+                kind: 17,
+                insert_text: "for (${1:x}) {}",
                 insert_text_rules: 4,
                 documentation: "For loop",
             },
+            CompletionItemSuggestion {
+                label: "string",
+                kind: 1,
+                insert_text: "string(${1:x})",
+                insert_text_rules: 4,
+                documentation: "Convert the argument to a string",
+            },
+            CompletionItemSuggestion {
+                label: "integer",
+                kind: 1,
+                insert_text: "integer(${1:x})",
+                insert_text_rules: 4,
+                documentation: "Convert the argument to an integer",
+            },
+            CompletionItemSuggestion {
+                label: "float",
+                kind: 1,
+                insert_text: "float(${1:x})",
+                insert_text_rules: 4,
+                documentation: "Convert the argument to a float",
+            },
+            CompletionItemSuggestion {
+                label: "date",
+                kind: 1,
+                insert_text: "date(${1:year}, ${1:month}, ${1:day})",
+                insert_text_rules: 4,
+                documentation: "Given a year, month (1-12) and day, create a new date",
+            },
+            CompletionItemSuggestion {
+                label: "datetime",
+                kind: 1,
+                insert_text: "datetime(${1:timestamp})",
+                insert_text_rules: 4,
+                documentation: "Given a POSIX timestamp of milliseconds since epoch, create a new datetime",
+            },
         ]
     };