From a92a9317969f5cc1be93ea0e99842ea4093ebef5 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Tue, 5 Jul 2022 00:08:55 -0400 Subject: [PATCH 01/16] Add missing manylinux dep --- scripts/_wheel_python.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/_wheel_python.js b/scripts/_wheel_python.js index d899c36ff2..73bca2ae3c 100644 --- a/scripts/_wheel_python.js +++ b/scripts/_wheel_python.js @@ -61,7 +61,7 @@ try { if (MANYLINUX_VERSION) { // install deps const boost = [ - `yum -y install wget`, + `yum -y install wget libffi-devel`, `wget https://boostorg.jfrog.io/artifactory/main/release/1.71.0/source/boost_1_71_0.tar.gz >/dev/null`, `tar xfz boost_1_71_0.tar.gz`, "cd boost_1_71_0", From e5ed5c8b8a6b3bdf4906dc2b887ae6490d9e385b Mon Sep 17 00:00:00 2001 From: Brian Blakely Date: Wed, 8 Jun 2022 14:59:30 -0400 Subject: [PATCH 02/16] [ExprTK Docs] Add section and WIP documentation. --- docs/md/expressions.md | 257 +++++++++++++++++++++++++++++++++++++++++ docs/sidebars.json | 3 +- 2 files changed, 259 insertions(+), 1 deletion(-) create mode 100644 docs/md/expressions.md diff --git a/docs/md/expressions.md b/docs/md/expressions.md new file mode 100644 index 0000000000..ef2314ee5d --- /dev/null +++ b/docs/md/expressions.md @@ -0,0 +1,257 @@ +--- +id: expressions +title: Column Expressions +--- + + + + +Add new columns to an existing data view by using expressions based on (but not +identical to) ExprTK ([source code](https://github.com/ArashPartow/exprtk), +[homepage](http://www.partow.net/programming/exprtk/)). This can be considered +similar to formulas in Excel, utilizing existing column values or arbitrary +scalar values defined within the expression, as well as conditional and +functional statements. + +## Basics + +### Creating a Custom Column + +Open a data view's side panel and press the "New Column" button, which will open +the expression editor. + + + +### Naming a Custom Column + +A comment on the first line of an expression will be used as its name. + +```html +// My Column Name +``` + +### Editing or Deleting a Custom Column + +Once a Custom Column is added, the expression editor can be opened by pressing +the hamburger button beside the column name. The column may be deleted by +pressing the X button, but to do so, it must first be removed from the data view +by unchecking the box beside it. + + + +### Giving a Custom Column a Value + +Column Expressions use an "implicit return," in this case meaning the last line +of the expression will be used as the value. + +```html +// My Column Name + +2 // Every cell in this column will be `2.00`. +``` + +Below you will see our first interactive example. + +
+ +
+ +## Working with Data + +### Access an Existing Column's Value + +Write an existing column's name in quotes. + +```html +// My Column Name + +"Sales" // Every cell in this column will be identical to "Sales". +``` + +
+ +
+ +### Variables + +While operations may be made directly against a column value (e.g. `"Sales" + +200`), multiple operations are better organized into variables. Variable +declarations must end with a semicolon `;`. + +```html +// My Column Name + +var incrementedBy200 := "Sales" + 200; +var half := incrementedBy200 / 2; + +half // This returns the value of `half`. +``` + +
+ +
+ +## Functions + +ExprTK and Perspective provide many functions for data manipulation. + +> **NOTE** +> +> For as full list of capabilities *with autocomplete*, press `Ctrl+Space` in +> the expression editor. + +### Functions Example + +Edit the Custom Column and paste the below Column Expression into the editor. + +```html +// My Column Name + +var upperCustomer := upper("Customer Name"); +var separator := concat(upperCustomer, ' | '); +var profitRatio := floor(percent_of("Profit", "Sales")); // Remove trailing decimal. +var combined := concat(separator, string(profitRatio)); +var percentDisplay := concat(combined, '%'); + +percentDisplay +``` + +
+ +
+ +### Strings (text) + +* `substring('string', start_idx, length)` - Perspective uses this function + instead of ExprTK's substring syntax. Get a portion of `string` starting at + the position `start_idx` (where 0 is the first character) and continuing for + `length` characters (including the starting character). The original string is + unchanged. Ex: `substring('strawberry', 4, 2) // 'wb'` +* `concat('string1', 'string2')` - Concatenate two string into one. +* `upper('string')` - Make a string all-uppercase. +* `lower('string')` - Make a string all-lowercase. +* `match('string', 'pattern')` +* `match_all('string', 'pattern')` +* `replace('string', 'pattern', 'replacer')` - Replace the first occurrence of + `pattern` in `string` with `replacer`. +* `replace_all('string', 'pattern', 'replacer')` - Replace all occurrences of + `pattern` in `string` with `replacer`. +* `indexof('string', 'pattern', output_vector)` +* `search('string', 'pattern')` + +### Numbers + +* General Math: `min, max, avg, sum, abs, ceil, floor, round, round, exp, log, + log10, log1p, log2, logn, pow, root, sqrt, iclamp, inrange` +* Trigonometry: `sin, cos, tan, acos, asin, atan, atan2, cosh, cot, csc, sec, + sinh, tanh, deg2rad, rad2deg, deg2grad, grad2deg` +* `percent_of(value, total)` - Produce the ratio between the value and total. +* `random()` - Produce a random number between 0.00 and 1.00. + +### Time + +* `date(year, month, day)` - Generate a date. +* `datetime(timestamp)` - Generate a datetime, where `timestamp` is [Unix + time](https://en.wikipedia.org/wiki/Unix_time) in milliseconds. +* `today()` - Present date. +* `now()` - Present datetime. +* `hour_of_day(x)` +* `day_of_week(x)` +* `month_of_year(x)` + +### Type Conversions + +* `integer(x)` - Convert `x` to an integer number. +* `float(x)` - Convert `x` to a decimal number. +* `boolean(x)` - Convert `x` to true/false. +* `string(x)` - Convert `x` to text. + +### Misc + +* `is_null(x)` - Is `true` if `x` has no value. +* `is_not_null(x)` - Is `true` if `x` has a value. +* `bucket(x, y)` +* `not(x)` +* `order("input column", value, ...)` +* `frac(x)` + +## Conditional Structures + +Making operations based on conditions is similar to most programming languages. + +### Conditional Example + +### `if-then-else` Block + +### Ternary Operators + +### `switch case` Block + +### Conditional Return Value + + + +## Loops + +### Loops Example + +### `for` Loop + +### `while` Loop + +### `repeat until` Loop + +### `break` + +### `continue` + +This keyword will skip the current iteration of the loop. + +## Advanced Example + +This **[Mandelbrot +demo](https://bl.ocks.org/texodus/5485f6b630b08d38218822e507f09f21)** calculates +the popular fractal in a Column Expression and displays it with a heatmap. Edit +the `color` column to view and manipulate the expression, whose contents are +below: + +```html +// color +var c := float("iterations"); +var x := floor("index" / "height"); +var y := "index" % "height"; + +var cx := "xmin" + (("xmax" - "xmin") * x) / ("width" - 1); +var cy := "ymin" + (("ymax" - "ymin") * y) / ("height" - 1); + +var vx := 0; +var vy := 0; +var vxx := 0; +var vyy := 0; +var vxy := 0; + +for (var ii := 0; ii < float("iterations"); ii += 1) { + if (vxx + vyy <= float(4)) { + vxy := vx * vy; + vxx := vx * vx; + vyy := vy * vy; + vx := vxx - vyy + cx; + vy := vxy + vxy + cy; + c -= 1; + } +}; + +c +``` diff --git a/docs/sidebars.json b/docs/sidebars.json index c5cbf734d3..38ff4fdf5d 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -6,7 +6,8 @@ "md/table", "md/view", "md/server", - "md/development" + "md/development", + "md/expressions" ], "API": [ "obj/perspective", From 27803792443718e91f044a2d0dcfa4199ff31921 Mon Sep 17 00:00:00 2001 From: Brian Blakely Date: Thu, 9 Jun 2022 22:10:32 -0400 Subject: [PATCH 03/16] [ExprTK Docs] Functions, conditions, loops. --- docs/md/expressions.md | 222 ++++++++++++++++++++++++++++++++++++----- 1 file changed, 197 insertions(+), 25 deletions(-) diff --git a/docs/md/expressions.md b/docs/md/expressions.md index ef2314ee5d..9033953610 100644 --- a/docs/md/expressions.md +++ b/docs/md/expressions.md @@ -42,7 +42,7 @@ by unchecking the box beside it. ### Giving a Custom Column a Value Column Expressions use an "implicit return," in this case meaning the last line -of the expression will be used as the value. +of the expression will be used as the value. This is called a "return value." ```html // My Column Name @@ -63,7 +63,7 @@ Below you will see our first interactive example. ### Access an Existing Column's Value -Write an existing column's name in quotes. +Write an existing column's name in double quotes. ```html // My Column Name @@ -100,6 +100,10 @@ half // This returns the value of `half`. > +### Text Values (Strings) + +A string is represented with single quotes `var keyphrase := 'Hello World'`. + ## Functions ExprTK and Perspective provide many functions for data manipulation. @@ -132,7 +136,7 @@ percentDisplay > -### Strings (text) +### Strings (Text) * `substring('string', start_idx, length)` - Perspective uses this function instead of ExprTK's substring syntax. Get a portion of `string` starting at @@ -142,23 +146,32 @@ percentDisplay * `concat('string1', 'string2')` - Concatenate two string into one. * `upper('string')` - Make a string all-uppercase. * `lower('string')` - Make a string all-lowercase. -* `match('string', 'pattern')` -* `match_all('string', 'pattern')` +* `match('string', 'pattern')` - It returns `true` if `pattern` is found within `string`, and `false` otherwise. * `replace('string', 'pattern', 'replacer')` - Replace the first occurrence of `pattern` in `string` with `replacer`. * `replace_all('string', 'pattern', 'replacer')` - Replace all occurrences of `pattern` in `string` with `replacer`. -* `indexof('string', 'pattern', output_vector)` -* `search('string', 'pattern')` + + + + + + ### Numbers * General Math: `min, max, avg, sum, abs, ceil, floor, round, round, exp, log, - log10, log1p, log2, logn, pow, root, sqrt, iclamp, inrange` + log10, log1p, log2, logn, pow, root, sqrt` * Trigonometry: `sin, cos, tan, acos, asin, atan, atan2, cosh, cot, csc, sec, sinh, tanh, deg2rad, rad2deg, deg2grad, grad2deg` +* `iclamp(min, value, max)` - If `value` is between `min` and `max`, then the + function produces the number to which `value` is closest, using `min` if the + distance is equal. If `value` is not in range, then `value` is used as-is. +* `inrange(min, value, max)` - Produces `true` or `false` depending on whether + `value` is between `min` and `max` (inclusive). * `percent_of(value, total)` - Produce the ratio between the value and total. * `random()` - Produce a random number between 0.00 and 1.00. +* `frac(x)` - Fractional portion (after the decimal) of `x` ### Time @@ -167,9 +180,9 @@ percentDisplay time](https://en.wikipedia.org/wiki/Unix_time) in milliseconds. * `today()` - Present date. * `now()` - Present datetime. -* `hour_of_day(x)` -* `day_of_week(x)` -* `month_of_year(x)` +* `hour_of_day(x)` - Return a datetime's hour of the day as a string. +* `day_of_week(x)` - Return a datetime's day of week as a string. +* `month_of_year(x)`- Return a datetime's month of the year as a string. ### Type Conversions @@ -182,42 +195,201 @@ percentDisplay * `is_null(x)` - Is `true` if `x` has no value. * `is_not_null(x)` - Is `true` if `x` has a value. -* `bucket(x, y)` -* `not(x)` -* `order("input column", value, ...)` -* `frac(x)` +* `order("input column", 'value', ...)` - Generates a sort order for a string + column based on the input order of the parameters, such as: `order("State", + 'Texas', 'New York')`. This would produce the following output: -## Conditional Structures + | State | My Column | + | ----------- | ------------ | + | Texas | 0.00 | + | New York | 1.00 | + | Connecticut | 2.00 | + | Maine | 2.00 | + | Oregon | 2.00 | + + + + +### Bucket + +`bucket(x, y)` - Bucket value `x` by `y`, where `y` is dependent on `x`'s value +type. -Making operations based on conditions is similar to most programming languages. +#### Bucketing by Number -### Conditional Example +`bucket("Sales", 100)` - Produces number bucketed by increments of 100 (always +rounded down). + +
+ +
+ +#### Bucketing by Date + +`bucket("Order Date", 'D')` - Produces date from datetime. +`bucket("Order Date", 'W')` - Produces date from the first day of that week. +`bucket("Order Date", 'M')` - Produces date from the first day of that month. +`bucket("Order Date", 'Y')` - Produces date from the first day of that year. + +## Conditional Structures + +Acting based on conditions is similar to most programming languages. ### `if-then-else` Block +An `if` block will be executed if the condition in parentheses is true. Commands +within the block must be indented. If there is more than one command in the +block, they must be wrapped in curly braces `{}`. An optional `else` block may +be used as a catch-all when the `if` condition is not met. + +```html +// My Column Name + +var priceAdjustmentDate := date(2016, 6, 18); +var finalPrice := "Sales" - "Discount"; +var additionalModifier := 0; + +if("Order Date" > priceAdjustmentDate) { + finalPrice -= 5; + additionalModifier -= 2; +} +else + finalPrice += 5; + +finalPrice + additionalModifier; +``` + +
+ +
+ +### Conditional Return Value + +One of the commands in a conditional block may be a return value. + +```html +// My Column Name + +var priceAdjustmentDate := date(2016, 6, 18); +var initialPrice := "Sales" - "Discount"; + +if("Order Date" > priceAdjustmentDate) + initialPrice - 10; // This value will be returned. +else + initialPrice + 5; // A different return value. +``` + +
+ +
+ ### Ternary Operators +This can be used to create a value based on a condition. It may be used anywhere +a value is expected, such as a variable assignment or a return value. + +The format is `CONDITION ? VALUE_WHEN_TRUE : VALUE_WHEN_FALSE;`. + +```html +// My Column Name + +var priceAdjustmentDate := date(2016, 6, 18); +var initialPrice := "Sales" - "Discount"; + +"Order Date" > priceAdjustmentDate ? initialPrice - 5 : initialPrice + 5; +``` + +
+ +
+ ### `switch case` Block -### Conditional Return Value +Run different commands based on any number of exclusive conditions. The optional +`default` condition is taken when no other conditions are met. +```html +// My Column Name + +var priceAdjustmentDate := date(2015, 10, 11); +var price := "Sales" - "Discount"; +switch { + case "Order Date" > priceAdjustmentDate: + price -= 5; + case "Order Date" < priceAdjustmentDate: + price += 5; + default: + price := max(price - 100, 0); // Special discount on day of adjustment. +} + +price; +``` + +
+ +
## Loops -### Loops Example +Repeat the same actions for aggregate effect. ### `for` Loop -### `while` Loop +The basic form of this loop is such: + +```html +for (var x := 0; x < NUMBER_OF_REPETITIONS; x += 1) { + // COMMANDS +} +``` + +#### `for` Example + +```html +// My Column Name + +var count := 0; +for (var x := 0; x < 5; x += 1) { + count += 1; +} + +count; +``` + +
+ +
-### `repeat until` Loop + + -### `break` + + -### `continue` + + -This keyword will skip the current iteration of the loop. + + ## Advanced Example From aaa7039406d91515a9afcb54249de092db4ed84d Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Mon, 11 Jul 2022 01:26:34 -0400 Subject: [PATCH 04/16] Fix review items --- docs/js/concepts.js | 5 +- docs/md/expressions.md | 474 +++++------------- docs/sidebars.json | 4 +- .../src/less/column-selector.less | 1 + 4 files changed, 132 insertions(+), 352 deletions(-) diff --git a/docs/js/concepts.js b/docs/js/concepts.js index febdd129ec..0c778aeb6f 100644 --- a/docs/js/concepts.js +++ b/docs/js/concepts.js @@ -34,7 +34,10 @@ async function main() { let ARROW_ICON = `arrow_forward`; for (const pre of document.querySelectorAll("pre")) { const code = pre.children[0]; - if (code.classList.contains("language-html")) { + if ( + !code.classList.contains("language-python") && + !code.classList.contains("language-javascript") + ) { continue; } const name = code.classList.contains("language-javascript") diff --git a/docs/md/expressions.md b/docs/md/expressions.md index 9033953610..1ddedd7006 100644 --- a/docs/md/expressions.md +++ b/docs/md/expressions.md @@ -1,253 +1,188 @@ --- id: expressions -title: Column Expressions +title: Expression Columns --- -Add new columns to an existing data view by using expressions based on (but not -identical to) ExprTK ([source code](https://github.com/ArashPartow/exprtk), -[homepage](http://www.partow.net/programming/exprtk/)). This can be considered -similar to formulas in Excel, utilizing existing column values or arbitrary -scalar values defined within the expression, as well as conditional and -functional statements. +Perspective supports _expression columns_, which are virtual columns calculated +as part of the `View`, optionally using values from its underlying `Table`'s +columns. Such expression columns are defined in Perspective's expression +language, an extended version of +[ExprTK](https://github.com/ArashPartow/exprtk), which is itself quite similar +(in design and features) to expressions in Excel. -## Basics +## UI -### Creating a Custom Column +Expression columns can be created in `` by clicking the +"New Column" button at the bottom of the column list (in +red below), or via the API by adding the +expression to the `expressions` config key when calling `restore()`. -Open a data view's side panel and press the "New Column" button, which will open -the expression editor. - - - -### Naming a Custom Column - -A comment on the first line of an expression will be used as its name. + +
+ +
+
+ +By default, such expression columns are not "used", and will appear above the +`Table`'s other deselected columns in the column list, with an additional set of +buttons for: + +- _Editing_ the column's expression. Doing so will update the definitions of + _all_ usage of the expression column. +- _Deleting_ the column. Clicking `Reset` (or calling the `reset()` method) + will not delete expressions unless the `Shift` key is held (or `true` + parameter supplied, respectively). This button only appears if the expression + column i unused. + +To use the column, just drag/select the column as you would a normal column, +e.g. as a "Filter", "Group By", etc. Expression columns cannot be edited or +updated (as they exist on the `View()` and are generated from the `Table()`'s +_real_ columns). However, they will automatically update whenever their +dependent columns update. + +## Perspective Extensions to ExprTK + +ExprTK has its own +[excellent documentation](http://www.partow.net/programming/exprtk/) which +covers the core langauge in depth, which is an excellent place to start in +learning the basics. In addition to these features, Perspective adds a few of +its own custom extensions and syntax. + +#### Static Typing + +In addition to `float` values which ExprTK supports natively, Perspective's +expression language also supports Perspective's other types `date`, `datetime`, +`integer`, `boolean`; as well as rudimentary type-checking, which will report +an error when the values/columns supplied as +arguments cannot be resolved to the expected type, e.g. `length(x)` expects an +argument `x` of type `string` and is not a valid expression for an `x` of +another type. Perspective supplies a set of _cast_ functions for converting +between types where possible e.g. `string(x)` to cast a variable `x` to a +`string`. + +#### Expression Column Name + +Expressions can be _named_ by providing a comment as the first line of the +expression. This name will be used in the `` UI when +referring to the column, but will also be used in the API when specifying e.g. +`group_by` or `order_by` fields. When creating a new column via +``'s expression editor, new columns will get a default name +(which you may delete or change): ```html -// My Column Name +// New Column 1 ``` -### Editing or Deleting a Custom Column +Without such a comment, an expression will show up in the `` +API and UI as itself (clipped to a reasonable length for the latter). -Once a Custom Column is added, the expression editor can be opened by pressing -the hamburger button beside the column name. The column may be deleted by -pressing the X button, but to do so, it must first be removed from the data view -by unchecking the box beside it. +#### Referencing `Table()` Columns - +Columns from the `Table()` can be referenced in an expression with +_double quotes_. -### Giving a Custom Column a Value +``` +// Expected Sales +("Sales" * 10) + "Profit" +``` -Column Expressions use an "implicit return," in this case meaning the last line -of the expression will be used as the value. This is called a "return value." +
+ +
-```html -// My Column Name +#### String Literals -2 // Every cell in this column will be `2.00`. -``` +In contrast to standard ExprTK, string literals are declared with +_single quotes_: -Below you will see our first interactive example. +``` +// Profitable +if ("Profit" > 0) { + 'Stonks' +} else { + 'Not Stonks' +} +```
-## Working with Data +#### Extended Library -### Access an Existing Column's Value +Perspective adds many of its own functions in addition to `ExprTK`'s standard +ones, including common functions for `datetime` and `string` types such as +`substring()`, `bucket()`, `day_of_week()`, etc. A full list of available +functions is available in the +[Expression Columns API](obj/perspective-viewer-exprtk). -Write an existing column's name in double quotes. +## Examples -```html -// My Column Name +#### Casting -"Sales" // Every cell in this column will be identical to "Sales". +Just `2`, as an `integer` (numeric literals currently default to `float` unless +cast). + +``` +integer(2) ```
+
-### Variables +#### Variables -While operations may be made directly against a column value (e.g. `"Sales" + -200`), multiple operations are better organized into variables. Variable -declarations must end with a semicolon `;`. - -```html +``` // My Column Name - var incrementedBy200 := "Sales" + 200; var half := incrementedBy200 / 2; - -half // This returns the value of `half`. +half ```
+
-### Text Values (Strings) - -A string is represented with single quotes `var keyphrase := 'Hello World'`. - -## Functions - -ExprTK and Perspective provide many functions for data manipulation. - -> **NOTE** -> -> For as full list of capabilities *with autocomplete*, press `Ctrl+Space` in -> the expression editor. - -### Functions Example - -Edit the Custom Column and paste the below Column Expression into the editor. - -```html -// My Column Name - +``` +// Complex Expression var upperCustomer := upper("Customer Name"); var separator := concat(upperCustomer, ' | '); var profitRatio := floor(percent_of("Profit", "Sales")); // Remove trailing decimal. var combined := concat(separator, string(profitRatio)); var percentDisplay := concat(combined, '%'); - percentDisplay ```
-
- -### Strings (Text) - -* `substring('string', start_idx, length)` - Perspective uses this function - instead of ExprTK's substring syntax. Get a portion of `string` starting at - the position `start_idx` (where 0 is the first character) and continuing for - `length` characters (including the starting character). The original string is - unchanged. Ex: `substring('strawberry', 4, 2) // 'wb'` -* `concat('string1', 'string2')` - Concatenate two string into one. -* `upper('string')` - Make a string all-uppercase. -* `lower('string')` - Make a string all-lowercase. -* `match('string', 'pattern')` - It returns `true` if `pattern` is found within `string`, and `false` otherwise. -* `replace('string', 'pattern', 'replacer')` - Replace the first occurrence of - `pattern` in `string` with `replacer`. -* `replace_all('string', 'pattern', 'replacer')` - Replace all occurrences of - `pattern` in `string` with `replacer`. - - - - - - - -### Numbers - -* General Math: `min, max, avg, sum, abs, ceil, floor, round, round, exp, log, - log10, log1p, log2, logn, pow, root, sqrt` -* Trigonometry: `sin, cos, tan, acos, asin, atan, atan2, cosh, cot, csc, sec, - sinh, tanh, deg2rad, rad2deg, deg2grad, grad2deg` -* `iclamp(min, value, max)` - If `value` is between `min` and `max`, then the - function produces the number to which `value` is closest, using `min` if the - distance is equal. If `value` is not in range, then `value` is used as-is. -* `inrange(min, value, max)` - Produces `true` or `false` depending on whether - `value` is between `min` and `max` (inclusive). -* `percent_of(value, total)` - Produce the ratio between the value and total. -* `random()` - Produce a random number between 0.00 and 1.00. -* `frac(x)` - Fractional portion (after the decimal) of `x` - -### Time - -* `date(year, month, day)` - Generate a date. -* `datetime(timestamp)` - Generate a datetime, where `timestamp` is [Unix - time](https://en.wikipedia.org/wiki/Unix_time) in milliseconds. -* `today()` - Present date. -* `now()` - Present datetime. -* `hour_of_day(x)` - Return a datetime's hour of the day as a string. -* `day_of_week(x)` - Return a datetime's day of week as a string. -* `month_of_year(x)`- Return a datetime's month of the year as a string. - -### Type Conversions - -* `integer(x)` - Convert `x` to an integer number. -* `float(x)` - Convert `x` to a decimal number. -* `boolean(x)` - Convert `x` to true/false. -* `string(x)` - Convert `x` to text. - -### Misc - -* `is_null(x)` - Is `true` if `x` has no value. -* `is_not_null(x)` - Is `true` if `x` has a value. -* `order("input column", 'value', ...)` - Generates a sort order for a string - column based on the input order of the parameters, such as: `order("State", - 'Texas', 'New York')`. This would produce the following output: - - | State | My Column | - | ----------- | ------------ | - | Texas | 0.00 | - | New York | 1.00 | - | Connecticut | 2.00 | - | Maine | 2.00 | - | Oregon | 2.00 | - - - - -### Bucket - -`bucket(x, y)` - Bucket value `x` by `y`, where `y` is dependent on `x`'s value -type. - -#### Bucketing by Number - -`bucket("Sales", 100)` - Produces number bucketed by increments of 100 (always -rounded down). - -
-
+
-#### Bucketing by Date - -`bucket("Order Date", 'D')` - Produces date from datetime. -`bucket("Order Date", 'W')` - Produces date from the first day of that week. -`bucket("Order Date", 'M')` - Produces date from the first day of that month. -`bucket("Order Date", 'Y')` - Produces date from the first day of that year. - -## Conditional Structures - -Acting based on conditions is similar to most programming languages. - -### `if-then-else` Block - -An `if` block will be executed if the condition in parentheses is true. Commands -within the block must be indented. If there is more than one command in the -block, they must be wrapped in curly braces `{}`. An optional `else` block may -be used as a catch-all when the `if` condition is not met. - -```html -// My Column Name +#### Conditionals +``` +// Conditional var priceAdjustmentDate := date(2016, 6, 18); var finalPrice := "Sales" - "Discount"; var additionalModifier := 0; @@ -259,171 +194,12 @@ if("Order Date" > priceAdjustmentDate) { else finalPrice += 5; -finalPrice + additionalModifier; -``` - -
- -
- -### Conditional Return Value - -One of the commands in a conditional block may be a return value. - -```html -// My Column Name - -var priceAdjustmentDate := date(2016, 6, 18); -var initialPrice := "Sales" - "Discount"; - -if("Order Date" > priceAdjustmentDate) - initialPrice - 10; // This value will be returned. -else - initialPrice + 5; // A different return value. +finalPrice + additionalModifier ```
- -### Ternary Operators - -This can be used to create a value based on a condition. It may be used anywhere -a value is expected, such as a variable assignment or a return value. - -The format is `CONDITION ? VALUE_WHEN_TRUE : VALUE_WHEN_FALSE;`. - -```html -// My Column Name - -var priceAdjustmentDate := date(2016, 6, 18); -var initialPrice := "Sales" - "Discount"; - -"Order Date" > priceAdjustmentDate ? initialPrice - 5 : initialPrice + 5; -``` - -
- -
- -### `switch case` Block - -Run different commands based on any number of exclusive conditions. The optional -`default` condition is taken when no other conditions are met. - -```html -// My Column Name - -var priceAdjustmentDate := date(2015, 10, 11); -var price := "Sales" - "Discount"; - -switch { - case "Order Date" > priceAdjustmentDate: - price -= 5; - case "Order Date" < priceAdjustmentDate: - price += 5; - default: - price := max(price - 100, 0); // Special discount on day of adjustment. -} - -price; -``` - -
- -
- -## Loops - -Repeat the same actions for aggregate effect. - -### `for` Loop - -The basic form of this loop is such: - -```html -for (var x := 0; x < NUMBER_OF_REPETITIONS; x += 1) { - // COMMANDS -} -``` - -#### `for` Example - -```html -// My Column Name - -var count := 0; -for (var x := 0; x < 5; x += 1) { - count += 1; -} - -count; -``` - -
- -
- - - - - - - - - - - - - -## Advanced Example - -This **[Mandelbrot -demo](https://bl.ocks.org/texodus/5485f6b630b08d38218822e507f09f21)** calculates -the popular fractal in a Column Expression and displays it with a heatmap. Edit -the `color` column to view and manipulate the expression, whose contents are -below: - -```html -// color -var c := float("iterations"); -var x := floor("index" / "height"); -var y := "index" % "height"; - -var cx := "xmin" + (("xmax" - "xmin") * x) / ("width" - 1); -var cy := "ymin" + (("ymax" - "ymin") * y) / ("height" - 1); - -var vx := 0; -var vy := 0; -var vxx := 0; -var vyy := 0; -var vxy := 0; - -for (var ii := 0; ii < float("iterations"); ii += 1) { - if (vxx + vyy <= float(4)) { - vxy := vx * vy; - vxx := vx * vx; - vyy := vy * vy; - vx := vxx - vyy + cx; - vy := vxy + vxy + cy; - c -= 1; - } -}; - -c -``` diff --git a/docs/sidebars.json b/docs/sidebars.json index 38ff4fdf5d..6ba6286e8b 100644 --- a/docs/sidebars.json +++ b/docs/sidebars.json @@ -5,9 +5,9 @@ "md/python", "md/table", "md/view", + "md/expressions", "md/server", - "md/development", - "md/expressions" + "md/development" ], "API": [ "obj/perspective", diff --git a/rust/perspective-viewer/src/less/column-selector.less b/rust/perspective-viewer/src/less/column-selector.less index 50b8904ac5..08c3442faf 100644 --- a/rust/perspective-viewer/src/less/column-selector.less +++ b/rust/perspective-viewer/src/less/column-selector.less @@ -57,6 +57,7 @@ white-space: nowrap; } #add-expression { + border: var(--column-add--border, none); padding: 6px 0 6px 0; margin: -6px 0 0 0; &.expr_editor_open { From 580db0c82d81a2b5640073d6fa18be1b492d5fb4 Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Mon, 11 Jul 2022 01:30:22 -0400 Subject: [PATCH 05/16] Fix webpack plugin compat with emsdk single-link build --- packages/perspective-webpack-plugin/index.js | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/packages/perspective-webpack-plugin/index.js b/packages/perspective-webpack-plugin/index.js index 0daba7dc27..8680b7d9c2 100644 --- a/packages/perspective-webpack-plugin/index.js +++ b/packages/perspective-webpack-plugin/index.js @@ -42,6 +42,17 @@ class PerspectiveWebpackPlugin { const moduleOptions = compilerOptions.module || (compilerOptions.module = {}); const rules = []; + + // Emscripten outputs require statements for these which are not called + // when loaded in browser. It's not polite to delete these but ... + const resolveOptions = + compilerOptions.resolve || (compilerOptions.resolve = {}); + const fallbackOptions = + resolveOptions.fallback || (resolveOptions.fallback = {}); + + fallbackOptions.path = false; + fallbackOptions.fs = false; + rules.push({ test: /perspective\.worker\.js$/, type: "javascript/auto", From aae3ca88f5d9cacc32b3d3f14d1e4fa20688c76b Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Mon, 11 Jul 2022 10:51:53 -0400 Subject: [PATCH 06/16] Fix sdist CI build ... --- .github/workflows/build.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ed572b079a..028bb996c4 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1466,7 +1466,7 @@ jobs: ################ # Python - name: Install python dependencies - run: python -m pip install --upgrade pip wheel setuptools "jupyterlab>=3.2" numpy "pyarrow>=5" pytest pytest-cov mock Faker psutil pytest-tornado pytz "tornado==6.1" + run: python -m pip install --upgrade pip wheel setuptools "jupyterlab>=3.2" numpy "pyarrow>=5" pytest pytest-cov mock aiohttp fastapi Faker psutil pytest-aiohttp pytest-asyncio pytest-tornado pytz "tornado==6.1" #~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~# #~~~~~~~~~ Build Pipelines ~~~~~~~~~# @@ -1482,7 +1482,7 @@ jobs: # Test sdist - name: Run tests against from-scratch sdist build - run: python -m pytest -vvv `python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))"`/perspective/tests --ignore=`python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))"`/perspective/tests/client_mode + run: python -m pytest -vvv --asyncio-mode=strict `python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))"`/perspective/tests --ignore=`python3 -c "import sysconfig; print(sysconfig.get_path('purelib'))"`/perspective/tests/client_mode ########################################################################################################################## ########################################################################################################################## From 4444a758e32761d6ce06809034132deb11f4517e Mon Sep 17 00:00:00 2001 From: Andrew Stein Date: Wed, 13 Jul 2022 00:48:09 -0400 Subject: [PATCH 07/16] Add categories to plugin selector --- .../src/js/charts/area.js | 1 + .../src/js/charts/bar.js | 1 + .../src/js/charts/candlestick.js | 1 + .../src/js/charts/column.js | 1 + .../src/js/charts/heatmap.js | 1 + .../src/js/charts/line.js | 1 + .../src/js/charts/ohlc.js | 1 + .../src/js/charts/sunburst.js | 1 + .../src/js/charts/treemap.js | 1 + .../src/js/charts/xy-line.js | 1 + .../src/js/charts/xy-scatter.js | 1 + .../src/js/charts/y-scatter.js | 1 + .../src/js/plugin/plugin.js | 4 +++ .../src/js/plugin.js | 4 +++ .../src/rust/components/aggregate_selector.rs | 6 +++- .../src/rust/components/containers/select.rs | 32 +++++++++++++---- .../src/rust/components/copy_dropdown.rs | 9 +++-- .../src/rust/components/export_dropdown.rs | 8 +++-- .../src/rust/components/plugin_selector.rs | 31 +++++++++++------ rust/perspective-viewer/src/rust/js/plugin.rs | 3 ++ rust/perspective-viewer/src/rust/renderer.rs | 3 +- .../src/rust/renderer/plugin_store.rs | 7 ++-- .../src/rust/renderer/registry.rs | 25 ++++++++++---- .../test/results/results.json | 34 +++++++++---------- 24 files changed, 126 insertions(+), 52 deletions(-) diff --git a/packages/perspective-viewer-d3fc/src/js/charts/area.js b/packages/perspective-viewer-d3fc/src/js/charts/area.js index f0a51f5b54..4bb5aa4749 100644 --- a/packages/perspective-viewer-d3fc/src/js/charts/area.js +++ b/packages/perspective-viewer-d3fc/src/js/charts/area.js @@ -98,6 +98,7 @@ function areaChart(container, settings) { } areaChart.plugin = { name: "Y Area", + category: "Y Chart", max_cells: 4000, max_columns: 50, render_warning: true, diff --git a/packages/perspective-viewer-d3fc/src/js/charts/bar.js b/packages/perspective-viewer-d3fc/src/js/charts/bar.js index 05def95846..3eb2fef497 100644 --- a/packages/perspective-viewer-d3fc/src/js/charts/bar.js +++ b/packages/perspective-viewer-d3fc/src/js/charts/bar.js @@ -66,6 +66,7 @@ function barChart(container, settings) { } barChart.plugin = { name: "X Bar", + category: "X Chart", max_cells: 1000, max_columns: 50, render_warning: true, diff --git a/packages/perspective-viewer-d3fc/src/js/charts/candlestick.js b/packages/perspective-viewer-d3fc/src/js/charts/candlestick.js index 7349356770..61459ba435 100644 --- a/packages/perspective-viewer-d3fc/src/js/charts/candlestick.js +++ b/packages/perspective-viewer-d3fc/src/js/charts/candlestick.js @@ -12,6 +12,7 @@ import ohlcCandle from "./ohlcCandle"; const candlestick = ohlcCandle(seriesCanvasCandlestick); candlestick.plugin = { name: "Candlestick", + category: "Y Chart", max_cells: 4000, max_columns: 50, render_warning: true, diff --git a/packages/perspective-viewer-d3fc/src/js/charts/column.js b/packages/perspective-viewer-d3fc/src/js/charts/column.js index 2586b58329..45180bc0c3 100644 --- a/packages/perspective-viewer-d3fc/src/js/charts/column.js +++ b/packages/perspective-viewer-d3fc/src/js/charts/column.js @@ -93,6 +93,7 @@ function columnChart(container, settings) { } columnChart.plugin = { name: "Y Bar", + category: "Y Chart", max_cells: 1000, max_columns: 50, render_warning: true, diff --git a/packages/perspective-viewer-d3fc/src/js/charts/heatmap.js b/packages/perspective-viewer-d3fc/src/js/charts/heatmap.js index 8ce006114c..2e7caf0412 100644 --- a/packages/perspective-viewer-d3fc/src/js/charts/heatmap.js +++ b/packages/perspective-viewer-d3fc/src/js/charts/heatmap.js @@ -82,6 +82,7 @@ function heatmapChart(container, settings) { } heatmapChart.plugin = { name: "Heatmap", + category: "Hierarchial Chart", max_cells: 50000, max_columns: 500, render_warning: true, diff --git a/packages/perspective-viewer-d3fc/src/js/charts/line.js b/packages/perspective-viewer-d3fc/src/js/charts/line.js index cd2ee899cf..3492193344 100644 --- a/packages/perspective-viewer-d3fc/src/js/charts/line.js +++ b/packages/perspective-viewer-d3fc/src/js/charts/line.js @@ -97,6 +97,7 @@ function lineChart(container, settings) { lineChart.plugin = { name: "Y Line", + category: "Y Chart", max_cells: 4000, max_columns: 50, render_warning: true, diff --git a/packages/perspective-viewer-d3fc/src/js/charts/ohlc.js b/packages/perspective-viewer-d3fc/src/js/charts/ohlc.js index 2b99321d02..044cc32604 100644 --- a/packages/perspective-viewer-d3fc/src/js/charts/ohlc.js +++ b/packages/perspective-viewer-d3fc/src/js/charts/ohlc.js @@ -12,6 +12,7 @@ import ohlcCandle from "./ohlcCandle"; const ohlc = ohlcCandle(seriesCanvasOhlc); ohlc.plugin = { name: "OHLC", + category: "Y Chart", max_cells: 3500, max_columns: 50, render_warning: true, diff --git a/packages/perspective-viewer-d3fc/src/js/charts/sunburst.js b/packages/perspective-viewer-d3fc/src/js/charts/sunburst.js index 0264858c95..98e7d6bf3d 100644 --- a/packages/perspective-viewer-d3fc/src/js/charts/sunburst.js +++ b/packages/perspective-viewer-d3fc/src/js/charts/sunburst.js @@ -87,6 +87,7 @@ function sunburst(container, settings) { sunburst.plugin = { name: "Sunburst", + category: "Hierarchial Chart", max_cells: 7500, max_columns: 50, render_warning: true, diff --git a/packages/perspective-viewer-d3fc/src/js/charts/treemap.js b/packages/perspective-viewer-d3fc/src/js/charts/treemap.js index 9b3caf7011..0ea1737bf5 100644 --- a/packages/perspective-viewer-d3fc/src/js/charts/treemap.js +++ b/packages/perspective-viewer-d3fc/src/js/charts/treemap.js @@ -80,6 +80,7 @@ function treemap(container, settings) { treemap.plugin = { type: "Treemap", name: "Treemap", + category: "Hierarchial Chart", max_cells: 5000, max_columns: 50, render_warning: true, diff --git a/packages/perspective-viewer-d3fc/src/js/charts/xy-line.js b/packages/perspective-viewer-d3fc/src/js/charts/xy-line.js index 709c0dda87..0165e32b9d 100644 --- a/packages/perspective-viewer-d3fc/src/js/charts/xy-line.js +++ b/packages/perspective-viewer-d3fc/src/js/charts/xy-line.js @@ -95,6 +95,7 @@ function xyLine(container, settings) { xyLine.plugin = { name: "X/Y Line", + category: "X/Y Chart", max_cells: 50000, max_columns: 50, render_warning: true, diff --git a/packages/perspective-viewer-d3fc/src/js/charts/xy-scatter.js b/packages/perspective-viewer-d3fc/src/js/charts/xy-scatter.js index 41b0273326..1d6f0cdd9f 100644 --- a/packages/perspective-viewer-d3fc/src/js/charts/xy-scatter.js +++ b/packages/perspective-viewer-d3fc/src/js/charts/xy-scatter.js @@ -132,6 +132,7 @@ function xyScatter(container, settings) { xyScatter.plugin = { name: "X/Y Scatter", + category: "X/Y Chart", max_cells: 50000, max_columns: 50, render_warning: true, diff --git a/packages/perspective-viewer-d3fc/src/js/charts/y-scatter.js b/packages/perspective-viewer-d3fc/src/js/charts/y-scatter.js index ee2ca18a10..af1e85a832 100644 --- a/packages/perspective-viewer-d3fc/src/js/charts/y-scatter.js +++ b/packages/perspective-viewer-d3fc/src/js/charts/y-scatter.js @@ -103,6 +103,7 @@ function yScatter(container, settings) { yScatter.plugin = { name: "Y Scatter", + category: "Y Chart", max_cells: 4000, max_columns: 50, render_warning: true, diff --git a/packages/perspective-viewer-d3fc/src/js/plugin/plugin.js b/packages/perspective-viewer-d3fc/src/js/plugin/plugin.js index 02680b6f70..c203a85259 100644 --- a/packages/perspective-viewer-d3fc/src/js/plugin/plugin.js +++ b/packages/perspective-viewer-d3fc/src/js/plugin/plugin.js @@ -88,6 +88,10 @@ export function register(...plugins) { return chart.plugin.name; } + get category() { + return chart.plugin.category; + } + get select_mode() { return chart.plugin.selectMode || "select"; } diff --git a/packages/perspective-viewer-datagrid/src/js/plugin.js b/packages/perspective-viewer-datagrid/src/js/plugin.js index 52433da95b..a317943f86 100644 --- a/packages/perspective-viewer-datagrid/src/js/plugin.js +++ b/packages/perspective-viewer-datagrid/src/js/plugin.js @@ -133,6 +133,10 @@ export class PerspectiveViewerDatagridPluginElement extends HTMLElement { return "Datagrid"; } + get category() { + return "Basic"; + } + get select_mode() { return "toggle"; } diff --git a/rust/perspective-viewer/src/rust/components/aggregate_selector.rs b/rust/perspective-viewer/src/rust/components/aggregate_selector.rs index 298777537d..f071013c47 100644 --- a/rust/perspective-viewer/src/rust/components/aggregate_selector.rs +++ b/rust/perspective-viewer/src/rust/components/aggregate_selector.rs @@ -90,6 +90,7 @@ impl Component for AggregateSelector { class={ "aggregate-selector" } values={ values } + label="weighted mean" selected={ selected_agg } on_select={ callback }> @@ -126,7 +127,10 @@ impl AggregateSelector { .collect::>(); let multi_aggregates2 = if !multi_aggregates.is_empty() { - vec![SelectItem::OptGroup("weighted mean", multi_aggregates)] + vec![SelectItem::OptGroup( + "weighted mean".into(), + multi_aggregates, + )] } else { vec![] }; diff --git a/rust/perspective-viewer/src/rust/components/containers/select.rs b/rust/perspective-viewer/src/rust/components/containers/select.rs index 984ab34da8..c177548097 100644 --- a/rust/perspective-viewer/src/rust/components/containers/select.rs +++ b/rust/perspective-viewer/src/rust/components/containers/select.rs @@ -6,6 +6,8 @@ // of the Apache License 2.0. The full license can be found in the LICENSE // file. +use std::borrow::Borrow; +use std::borrow::Cow; use std::fmt::Debug; use std::fmt::Display; use std::str::FromStr; @@ -15,7 +17,16 @@ use yew::prelude::*; #[derive(Clone, Eq, PartialEq)] pub enum SelectItem { Option(T), - OptGroup(&'static str, Vec), + OptGroup(Cow<'static, str>, Vec), +} + +impl SelectItem { + pub fn name<'a>(&self) -> Cow<'a, str> { + match self { + Self::Option(x) => format!("{}", x).into(), + Self::OptGroup(x, _) => x.clone(), + } + } } pub enum SelectMsg { @@ -32,6 +43,9 @@ where pub selected: T, pub on_select: Callback, + #[prop_or_default] + pub label: Option<&'static str>, + #[prop_or_default] pub id: Option<&'static str>, @@ -137,15 +151,19 @@ where } }, SelectItem::OptGroup(name, group) => html! { - + { for group.iter().map(|value| { let selected = *value == ctx.props().selected; - let label = format!("{}", value) - .strip_prefix(name) - .unwrap() + let label = format!("{}", value); + let category: &str = name.borrow(); + let label = label + .strip_prefix(category) + .unwrap_or(&label) .trim() .to_owned(); @@ -167,8 +185,8 @@ where }; html! { - if is_group_selected { - + if is_group_selected && ctx.props().label.is_some() { +