feat(examples): add gesture-analysis example

thi-ng · Sep 8, 2018 · 6051439 · 6051439
1 parent bfd3c2f
commit 6051439
Show file tree

Hide file tree

Showing 6 changed files with 332 additions and 0 deletions.
diff --git a/examples/gesture-analysis/.gitignore b/examples/gesture-analysis/.gitignore
@@ -0,0 +1,5 @@
+.cache
+out
+node_modules
+yarn.lock
+*.js
diff --git a/examples/gesture-analysis/README.md b/examples/gesture-analysis/README.md
@@ -0,0 +1,23 @@
+# gesture-analysis
+
+[Live demo](http://demo.thi.ng/umbrella/gesture-analysis/)
+
+Mouse / touch gesture processing, analysis and SVG visualization, using
+[@thi.ng/rstream](https://github.com/thi-ng/umbrella/tree/master/packages/rstream)
+constructs and
+[@thi.ng/transducers](https://github.com/thi-ng/umbrella/tree/master/packages/transducers).
+
+```bash
+git clone https://github.com/thi-ng/umbrella.git
+cd umbrella/examples/gesture-analysis
+yarn install
+yarn start
+```
+
+## Authors
+
+- Karsten Schmidt
+
+## License
+
+&copy; 2018 Karsten Schmidt // Apache Software License 2.0
diff --git a/examples/gesture-analysis/package.json b/examples/gesture-analysis/package.json
@@ -0,0 +1,30 @@
+{
+    "name": "gesture-analysis",
+    "version": "0.0.1",
+    "repository": "https://github.com/thi-ng/umbrella",
+    "author": "Karsten Schmidt <k+npm@thi.ng>",
+    "license": "Apache-2.0",
+    "scripts": {
+        "build": "parcel build index.html -d out --no-source-maps --no-cache --detailed-report",
+        "start": "parcel index.html -p 8080 --open"
+    },
+    "devDependencies": {
+        "parcel-bundler": "^1.9.7",
+        "terser": "^3.8.2",
+        "typescript": "^3.0.1"
+    },
+    "dependencies": {
+        "@thi.ng/hiccup-svg": "latest",
+        "@thi.ng/rstream": "latest",
+        "@thi.ng/rstream-gestures": "latest",
+        "@thi.ng/transducers": "latest",
+        "@thi.ng/transducers-hdom": "latest",
+        "@thi.ng/vectors": "latest"
+    },
+    "browserslist": [
+        "last 3 Chrome versions"
+    ],
+    "browser": {
+        "process": false
+    }
+}
diff --git a/examples/gesture-analysis/src/config.ts b/examples/gesture-analysis/src/config.ts
@@ -0,0 +1,70 @@
+import { transformVectors1 } from "@thi.ng/vectors/common";
+import { mul2, Vec2 } from "@thi.ng/vectors/vec2";
+
+// initial call to action gesture
+// (recorded handwriting)
+const raw = [
+    204, 37, 204, 38, 204, 39, 204, 45, 204, 58, 204, 73, 204, 94, 204, 114,
+    202, 137, 199, 162, 196, 186, 194, 208, 190, 242, 189, 257, 188, 279, 186, 296,
+    186, 310, 185, 317, 185, 324, 185, 329, 184, 334, 184, 336, 183, 339, 182, 340,
+    181, 340, 179, 340, 176, 340, 169, 336, 160, 329, 144, 315, 134, 303, 128, 296,
+    121, 289, 117, 283, 113, 277, 111, 273, 108, 267, 107, 263, 105, 257, 104, 253,
+    104, 249, 104, 246, 104, 240, 106, 235, 112, 227, 120, 218, 132, 205, 138, 199,
+    146, 192, 153, 187, 160, 184, 166, 181, 173, 178, 181, 176, 189, 174, 195, 173,
+    200, 173, 204, 173, 207, 172, 208, 172, 208, 172, 209, 171, 210, 171, 210, 171,
+    211, 171, 211, 170, 211, 170, 211, 170, 212, 170, 212, 170, 215, 169, 218, 167,
+    221, 166, 224, 164, 226, 162, 228, 160, 231, 156, 232, 154, 233, 152, 234, 150,
+    234, 147, 235, 142, 236, 138, 237, 132, 237, 129, 238, 126, 238, 124, 238, 123,
+    238, 123, 238, 123, 238, 123, 237, 123, 237, 124, 235, 126, 234, 129, 232, 132,
+    231, 135, 230, 137, 230, 139, 229, 141, 228, 145, 228, 150, 226, 155, 225, 159,
+    224, 165, 224, 167, 223, 170, 223, 172, 222, 174, 221, 179, 221, 182, 220, 187,
+    220, 192, 220, 199, 220, 205, 220, 213, 220, 223, 221, 236, 221, 249, 221, 259,
+    221, 268, 222, 275, 223, 282, 224, 286, 224, 290, 224, 295, 224, 299, 225, 303,
+    225, 306, 225, 308, 225, 310, 225, 311, 225, 312, 225, 313, 225, 313, 225, 314,
+    225, 314, 225, 312, 225, 311, 225, 306, 225, 302, 225, 299, 225, 296, 225, 292,
+    225, 288, 225, 283, 225, 277, 224, 272, 224, 266, 224, 261, 224, 256, 224, 252,
+    224, 250, 225, 248, 226, 244, 226, 241, 227, 238, 228, 232, 228, 229, 229, 226,
+    230, 222, 231, 218, 232, 215, 233, 211, 234, 208, 235, 205, 236, 202, 237, 200,
+    238, 198, 239, 196, 240, 194, 241, 192, 243, 190, 245, 186, 248, 183, 251, 179,
+    255, 174, 259, 170, 264, 164, 267, 160, 272, 155, 275, 150, 279, 146, 282, 143,
+    285, 141, 288, 139, 291, 138, 295, 138, 298, 138, 303, 138, 307, 138, 310, 138,
+    313, 139, 315, 141, 317, 144, 318, 145, 319, 146, 319, 147, 320, 147, 320, 148,
+    321, 148, 321, 149, 322, 149, 322, 149, 322, 149, 322, 149, 322, 150, 322, 150,
+    322, 150, 322, 150, 323, 151, 323, 152, 324, 153, 324, 156, 325, 160, 325, 163,
+    326, 166, 327, 170, 328, 175, 330, 181, 331, 185, 333, 193, 334, 199, 336, 206,
+    338, 213, 340, 219, 341, 225, 344, 233, 346, 240, 347, 243, 349, 250, 349, 253,
+    350, 258, 351, 262, 351, 267, 351, 271, 351, 276, 351, 280, 351, 283, 351, 286,
+    351, 288, 351, 290, 351, 292, 351, 293, 351, 293, 351, 294, 351, 295, 351, 295,
+    350, 296, 347, 297, 343, 298, 338, 299, 332, 300, 326, 300, 320, 301, 316, 302,
+    311, 302, 309, 303, 306, 303, 303, 303, 298, 303, 294, 303, 290, 302, 287, 301,
+    285, 300, 283, 298, 282, 298, 280, 297, 279, 297, 277, 296, 276, 295, 275, 294,
+    274, 293, 273, 292, 271, 291, 267, 288, 263, 285, 260, 283, 257, 280, 256, 278,
+    255, 277, 254, 275, 254, 273, 254, 270, 254, 268, 254, 266, 254, 263, 256, 261,
+    258, 258, 259, 256, 261, 254, 263, 252, 266, 249, 270, 246, 273, 244, 276, 241,
+    280, 239, 284, 236, 287, 234, 294, 230, 298, 228, 303, 225, 307, 224, 311, 222,
+    315, 221, 320, 219, 324, 218, 326, 216, 328, 215, 329, 214, 331, 212, 333, 210,
+    336, 208, 339, 206, 340, 205, 341, 205, 342, 204, 342, 204, 342, 204, 342, 204,
+    343, 204, 343, 205, 343, 205, 343, 205, 343, 205, 343, 206, 343, 206, 343, 206,
+    343, 206, 343, 207, 343, 207, 343, 207, 343, 208, 343, 208, 344, 208, 344, 208,
+    344, 208, 345, 207, 345, 207, 346, 206, 348, 204, 350, 202, 352, 199, 354, 195,
+    357, 190, 358, 186, 359, 181, 360, 174, 360, 168, 360, 164, 360, 156, 360, 151,
+    360, 147, 360, 143, 360, 138, 361, 134, 361, 130, 362, 127, 363, 125, 363, 123,
+    363, 122, 364, 121, 364, 120, 365, 119, 366, 117, 366, 116, 366, 116, 366, 117,
+    366, 120, 365, 125, 364, 132, 364, 142, 364, 152, 364, 163, 364, 174, 364, 186,
+    365, 197, 366, 209, 367, 220, 370, 231, 377, 248, 380, 255, 387, 270, 390, 276,
+    394, 283, 397, 288, 400, 292, 404, 295, 407, 297, 410, 299, 412, 300, 413, 301,
+    415, 301, 415, 301, 416, 301, 416, 301, 416, 298, 416, 294, 416, 287, 414, 275,
+    412, 266, 411, 260, 410, 254, 408, 249, 408, 245, 407, 241, 406, 238, 406, 233,
+    406, 230, 406, 227, 406, 225, 405, 223, 405, 222, 405, 221, 406, 221, 410, 226,
+    418, 233, 426, 239, 435, 247, 446, 255, 458, 263, 471, 270, 482, 277, 491, 281,
+    496, 283, 501, 285, 505, 287, 508, 288, 511, 289, 513, 290, 514, 290, 515, 290,
+    515, 284, 507, 261, 497, 238, 492, 222, 486, 203, 481, 191, 476, 182, 471, 175,
+    464, 170, 457, 165, 449, 162, 445, 160, 439, 159, 435, 158, 433, 157, 431, 157,
+    430, 156, 430, 155, 430, 154
+];
+
+// downscale & transform into memory mapped Vec2 array
+export const CTA = Vec2.mapBuffer(
+    transformVectors1(mul2, raw, [0.75, 0.75], raw.length / 2, 2),
+    raw.length / 2
+);
diff --git a/examples/gesture-analysis/src/index.ts b/examples/gesture-analysis/src/index.ts
@@ -0,0 +1,193 @@
+import { circle } from "@thi.ng/hiccup-svg/circle";
+import { group } from "@thi.ng/hiccup-svg/group";
+import { polyline } from "@thi.ng/hiccup-svg/polyline";
+import { svg } from "@thi.ng/hiccup-svg/svg";
+import { GestureEvent, gestureStream, GestureType } from "@thi.ng/rstream-gestures";
+import { fromIterable } from "@thi.ng/rstream/from/iterable";
+import { merge } from "@thi.ng/rstream/stream-merge";
+import { sync } from "@thi.ng/rstream/stream-sync";
+import { updateDOM } from "@thi.ng/transducers-hdom";
+import { comp } from "@thi.ng/transducers/func/comp";
+import { identity } from "@thi.ng/transducers/func/identity";
+import { peek } from "@thi.ng/transducers/func/peek";
+import { push } from "@thi.ng/transducers/rfn/push";
+import { transduce } from "@thi.ng/transducers/transduce";
+import { filter } from "@thi.ng/transducers/xform/filter";
+import { map } from "@thi.ng/transducers/xform/map";
+import { multiplexObj } from "@thi.ng/transducers/xform/multiplex-obj";
+import { partition } from "@thi.ng/transducers/xform/partition";
+import { Vec2 } from "@thi.ng/vectors/vec2";
+
+import { CTA } from "./config";
+
+/**
+ * Root component function, attached to rstream (see further below).
+ * Receives raw & processed gesture paths to visualize as SVG.
+ *
+ * @param raw
+ * @param processed
+ */
+const app = ({ raw, processed }) =>
+    ["div",
+        svg(
+            {
+                width: window.innerWidth,
+                height: window.innerHeight,
+                stroke: "none",
+                fill: "none",
+            },
+            path(raw || [], processed.path, processed.corners || [])
+        ),
+        ["div.fixed.top-0.left-0.ma3",
+            ["div", `raw: ${(raw && raw.length) || 0}`],
+            ["div", `resampled: ${(processed && processed.path.length) || 0}`],
+            ["div", `corners: ${(processed && processed.corners.length) || 0}`],
+        ]
+    ];
+
+/**
+ * Gesture visualization component. Creates an SVG group of shape
+ * elements & iterables.
+ *
+ * @param raw raw gesture path
+ * @param sampled resampled path
+ * @param corners array of corner points
+ */
+const path = (raw: Vec2[], sampled: Vec2[], corners: Vec2[]) =>
+    group({},
+        polyline(raw, { stroke: "#444" }),
+        map((p) => circle(p, 2, { fill: "#444" }), raw),
+        polyline(sampled, { stroke: "#fff" }),
+        map((p) => circle(p, 2, { fill: "#fff" }), sampled),
+        map((p) => circle(p, 6, { fill: "#cf0" }), corners),
+        circle(sampled[0], 6, { fill: "#f0c" }),
+        circle(peek(sampled), 6, { fill: "#0cf" }),
+    );
+
+/**
+ * Re-samples given polyline at given uniform distance. Returns array of
+ * interpolated points (does not modify original).
+ *
+ * @param step sample distance
+ * @param pts
+ */
+const sampleUniform = (step: number, pts: Vec2[]) => {
+    if (!pts.length) return [];
+    let prev = pts[0];
+    const res: Vec2[] = [prev];
+    for (let i = 1, n = pts.length; i < n; prev = peek(res), i++) {
+        const p = pts[i];
+        let d = p.dist(prev);
+        while (d >= step) {
+            res.push(prev = prev.copy().mixN(p, step / d));
+            d -= step;
+        }
+    }
+    res.push(peek(pts));
+    return res;
+};
+
+/**
+ * Applies low-pass filter to given polyline. I.e. Each point in the
+ * array (apart from the 1st) is interpolated towards the last point in
+ * the result array. Returns new array of smoothed points.
+ *
+ * @param path
+ */
+const smoothPath = (smooth: number, path: Vec2[]) => {
+    const res: Vec2[] = [path[0]];
+    for (let i = 1, n = path.length; i < n; i++) {
+        res.push(path[i].copy().mixN(res[i - 1], smooth));
+    }
+    return res;
+};
+
+/**
+ * Corner detector HOF. Returns new function which takes 3 successive
+ * path points and returns true if the middle point is a corner.
+ *
+ * @param thresh normalized angle threshold
+ */
+const isCorner = (thresh: number) => {
+    thresh = Math.PI * (1 - thresh);
+    return ([a, b, c]: Vec2[]) =>
+        b.copy().sub(a).angleBetween(b.copy().sub(c), true) < thresh;
+};
+
+/**
+ * Gesture event processor. Collects gesture event positions into an
+ * array of Vec2.
+ */
+const collectPath = () => {
+    let pts: Vec2[] = [];
+    return (g: GestureEvent) => {
+        const pos = new Vec2(g[1].pos);
+        switch (g[0]) {
+            case GestureType.START:
+                pts = [pos];
+                break;
+            case GestureType.DRAG:
+                pts.push(pos);
+                break;
+            // uncomment to destroy path on mouseup / touchend
+            // case GestureType.END:
+            // pts = [];
+        }
+        return pts;
+    }
+};
+
+// gesture input stream(s)
+const gesture = merge({
+    src: [
+        // the initial CTA (call-to-action) gesture (see config.ts)
+        // will be shown prior to first user interaction.
+        // this stream only emits this one single gesture path,
+        // then closes and will be removed from the stream merge
+        fromIterable([CTA]),
+        // mouse & touch event stream attached to document.body
+        // we're filtering out move & zoom events to avoid extraneous work
+        gestureStream(document.body)
+            .transform(
+                filter((g) => g[0] != GestureType.MOVE && g[0] != GestureType.ZOOM),
+                map(collectPath())
+            )
+    ]
+});
+
+// main gesture processor
+// uses 2 inputs, both based on above `gesture` stream
+// however one of them will be transformed via multi-stage transducer
+// to create a resampled version and apply a corner detector
+// the resulting stream will emit tuple objects of this structure:
+// `{ raw: Vec2[], processed: { path: Vec2[], corners: Vec2[] } }
+sync({
+    src: {
+        raw: gesture,
+        processed: gesture.transform(
+            comp(
+                map((pts: Vec2[]) => smoothPath(3 / 4, pts)),
+                map((pts: Vec2[]) => sampleUniform(20, pts)),
+                multiplexObj({
+                    path: map(identity),
+                    corners: map(
+                        (pts) => transduce(
+                            comp(
+                                partition(3, 1),
+                                filter(isCorner(1 / 4)),
+                                map((x) => x[1])
+                            ),
+                            push(),
+                            pts
+                        )
+                    )
+                })
+            )
+        ),
+    },
+}).transform(
+    // transform result tuples into HDOM components
+    map(app),
+    // update UI diff
+    updateDOM()
+);
diff --git a/examples/gesture-analysis/tsconfig.json b/examples/gesture-analysis/tsconfig.json
@@ -0,0 +1,11 @@
+{
+    "extends": "../../tsconfig.json",
+    "compilerOptions": {
+        "outDir": ".",
+        "target": "es6",
+        "sourceMap": true
+    },
+    "include": [
+        "./src/**/*.ts"
+    ]
+}