Merge pull request #468 from jpmorganchase/windows-build

Ported Javascript build, test and clean tasks to Windows
finos · Mar 6, 2019 · 0b6523c · 0b6523c
2 parents 5c242ff + 0034434
commit 0b6523c
Show file tree

Hide file tree

Showing 14 changed files with 1,129 additions and 73 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -37,7 +37,7 @@ env:
 install:
   - if [ "$TEST" = "CPP_OSX" ]; then brew install yarn; fi
   - yarn
-
+ 
 script:
   - if [ "$TEST" = "WASM" ]; then PSP_DOCKER=1 yarn lint; fi
   - if [ "$TEST" = "WASM" ]; then PSP_DOCKER=1 yarn build; fi

diff --git a/package.json b/package.json
@@ -6,20 +6,21 @@
     ],
     "devDependencies": {
         "@apache-arrow/es5-esm": "^0.4.0",
+        "@babel/cli": "^7.2.3",
         "@babel/core": "^7.1.0",
         "@babel/plugin-proposal-decorators": "^7.1.2",
         "@babel/plugin-transform-for-of": "^7.0.0",
         "@babel/plugin-transform-runtime": "^7.1.0",
         "@babel/polyfill": "^7.0.0",
         "@babel/preset-env": "^7.1.5",
-        "babel-plugin-transform-custom-element-classes": "^0.1.0",
-        "@babel/cli": "^7.2.3",
-        "babel-polyfill": "^6.26.0",
         "babel-eslint": "^8.2.3",
         "babel-jest": "^23.6.0",
+        "babel-plugin-transform-custom-element-classes": "^0.1.0",
+        "babel-polyfill": "^6.26.0",
         "chart.js": "^2.7.1",
         "codemirror": "^5.30.0",
         "codemirror-javascript": "^0.2.0",
+        "cpx": "^1.5.0",
         "documentation": "^5.3.3",
         "eslint": "^4.19.1",
         "eslint-config-prettier": "^3.0.1",

diff --git a/packages/perspective-viewer-d3fc/package.json b/packages/perspective-viewer-d3fc/package.json
@@ -12,16 +12,16 @@
         "bench": "npm-run-all bench:build bench:run",
         "bench:build": "echo \"No Benchmarks\"",
         "bench:run": "echo \"No Benchmarks\"",
-        "prebuild": "mkdir -p cjs",
-        "build:babel": "cp -r src/* cjs && babel src/js --source-maps --out-dir cjs/js",
+        "prebuild": "mkdirp cjs",
+        "build:babel": "cpx \"src/**/*\" cjs && babel src/js --source-maps --out-dir cjs/js",
         "build:webpack": "webpack --color --config src/config/d3fc.plugin.config.js",
         "build": "npm-run-all build:babel build:webpack",
-        "test:build": "cp test/html/* build",
-        "watch": "webpack --color --watch --config src/config/d3fc.watch.config.js",
+        "test:build": "cpx \"test/html/*\" build",
+        "watch": "webpack --color --watch --config src/config/d3fc.plugin.config.js",
         "test:run": "jest --silent --color 2>&1",
         "test": "npm-run-all test:build test:run",
-        "clean": "find build -mindepth 1 -delete && find cjs -mindepth 1 -delete",
-        "clean:screenshots": "find screenshots/ \\( -name *.diff.png -o -name *.failed.png \\) -mindepth 1 -delete"
+        "clean": "rimraf build && rimraf cjs",
+        "clean:screenshots": "rimraf \"screenshots/**/*.@(failed|diff).png\""
     },
     "jest": {
         "roots": [

diff --git a/packages/perspective-viewer-highcharts/package.json b/packages/perspective-viewer-highcharts/package.json
@@ -13,16 +13,16 @@
         "bench": "npm-run-all bench:build bench:run",
         "bench:build": "echo \"No Benchmarks\"",
         "bench:run": "echo \"No Benchmarks\"",
-        "prebuild": "mkdir -p cjs",
-        "build:babel": "cp -r src/* cjs && babel src/js --source-maps --out-dir cjs/js",
+        "prebuild": "mkdirp cjs",
+        "build:babel": "cpx \"src/**/*\" cjs && babel src/js --source-maps --out-dir cjs/js",
         "build:webpack": "webpack --color --config src/config/highcharts.plugin.config.js",
         "build": "npm-run-all build:babel build:webpack",
-        "test:build": "cp test/html/* build",
+        "test:build": "cpx \"test/html/*\" build",
         "watch": "webpack --color --watch --config src/config/highcharts.plugin.config.js",
         "test:run": "jest --silent --color 2>&1",
         "test": "npm-run-all test:build test:run",
-        "clean": "find build -mindepth 1 -delete && find cjs -mindepth 1 -delete",
-        "clean:screenshots": "find screenshots/ \\( -name *.diff.png -o -name *.failed.png \\) -mindepth 1 -delete"
+        "clean": "rimraf build && rimraf cjs",
+        "clean:screenshots": "rimraf \"screenshots/**/*.@(failed|diff).png\""
     },
     "jest": {
         "roots": [

diff --git a/packages/perspective-viewer-hypergrid/package.json b/packages/perspective-viewer-hypergrid/package.json
@@ -13,15 +13,15 @@
         "bench": "npm-run-all bench:build bench:run",
         "bench:build": "echo \"No Benchmarks\"",
         "bench:run": "echo \"No Benchmarks\"",
-        "prebuild": "mkdir -p cjs",
-        "build:babel": "cp -r src/* cjs && babel src/js --source-maps --out-dir cjs/js",
+        "prebuild": "mkdirp cjs",
+        "build:babel": "cpx \"src/**/*\" cjs && babel src/js --source-maps --out-dir cjs/js",
         "build:webpack": "webpack --color --config src/config/hypergrid.plugin.config.js",
         "build": "npm-run-all build:babel build:webpack",
-        "test:build": "cp test/html/* build",
+        "test:build": "cpx \"test/html/*\" build",
         "test:run": "jest --silent --color",
         "test": "npm-run-all test:build test:run",
-        "clean": "find build -mindepth 1 -delete && find cjs -mindepth 1 -delete",
-        "clean:screenshots": "find screenshots/ \\( -name *.diff.png -o -name *.failed.png \\) -mindepth 1 -delete"
+        "clean": "rimraf build && rimraf cjs",
+        "clean:screenshots": "rimraf \"screenshots/**/*.@(failed|diff).png\""
     },
     "jest": {
         "roots": [

diff --git a/packages/perspective-viewer/package.json b/packages/perspective-viewer/package.json
@@ -15,17 +15,17 @@
         "bench": "npm-run-all bench:build bench:run",
         "bench:build": "echo \"No Benchmarks\"",
         "bench:run": "echo \"No Benchmarks\"",
-        "prebuild": "mkdir -p cjs",
-        "build:babel": "cp -r src/* cjs && babel src/js --source-maps --out-dir cjs/js",
+        "prebuild": "mkdirp cjs",
+        "build:babel": "cpx \"src/**/*\" cjs && babel src/js --source-maps --out-dir cjs/js",
         "build:webpack:viewer": "webpack --color --config src/config/view.config.js",
-        "build:webpack:themes": "webpack --color --config src/config/themes.config.js && rm build/__themes.js",
+        "build:webpack:themes": "webpack --color --config src/config/themes.config.js && rimraf build/__themes.js",
         "build": "npm-run-all build:babel build:webpack:*",
         "watch": "webpack --color --watch --config src/config/view.config.js",
-        "test:build": "cp test/html/* build && cp test/csv/* build && cp test/css/* build",
+        "test:build": "cpx \"test/html/*\" build && cpx \"test/csv/*\" build && cpx \"test/css/*\" build",
         "test:run": "jest --silent --color",
         "test": "npm-run-all test:build test:run",
-        "clean": "find build -mindepth 1 -delete",
-        "clean:screenshots": "find screenshots/ \\( -name *.failed.png -o -name *.diff.png \\) -mindepth 1 -delete",
+        "clean": "rimraf build",
+        "clean:screenshots": "rimraf \"screenshots/**/*.@(failed|diff).png\"",
         "docs": "documentation build src/js/viewer.js -f md --shallow > README.md && cp README.md ../../docs/perspective-viewer.md"
     },
     "publishConfig": {

diff --git a/packages/perspective-webpack-plugin/src/js/psp-worker-loader.js b/packages/perspective-webpack-plugin/src/js/psp-worker-loader.js
@@ -52,7 +52,7 @@ exports.default = function loader(content) {
             inputPath = inputPath
                 .replace(path.join("perspective", "build"), "perspective")
                 .replace(/\.js/, ".worker.js")
-                .replace(/(cjs\/js)/, "build");
+                .replace(path.join("cjs", "js"), "build");
         }
         content = fs.readFileSync(inputPath).toString();
         if (!options.inline) {

diff --git a/packages/perspective-webpack-plugin/src/js/wasm-loader.js b/packages/perspective-webpack-plugin/src/js/wasm-loader.js
@@ -32,7 +32,7 @@ exports.pitch = function pitch(request) {
     validateOptions(schema, options, "Cross Origin File Loader");
 
     var context = options.context || this.rootContext || (this.options && this.options.context);
-    var content = fs.readFileSync(request.replace("cjs/js", "build").replace("wasm.js", "wasm"));
+    var content = fs.readFileSync(request.replace(path.join("cjs", "js"), "build").replace("wasm.js", "wasm"));
     var emitPath = loaderUtils.interpolateName(this, options.name, {
         context,
         content,

diff --git a/packages/perspective/package.json b/packages/perspective/package.json
@@ -17,18 +17,18 @@
     "typings": "index.d.ts",
     "scripts": {
         "bench": "node ../../scripts/bench.js",
-        "prebuild": "mkdir -p build && mkdir -p obj",
+        "prebuild": "mkdirp build && mkdirp obj",
         "cpp": "npm-run-all build:cpp test:cpp",
         "build": "npm-run-all build:babel build:webpack",
         "build:babel": "babel src/js --source-maps --out-dir cjs/js",
         "build:webpack": "npm-run-all -p build:webpack:*",
         "build:webpack:umd": "webpack --color --config src/config/perspective.config.js",
         "build:webpack:node": "webpack --color --config src/config/perspective.node.config.js",
         "docs": "documentation build src/js/perspective.js -f md --shallow > README.md && cp README.md ../../docs/perspective.md",
-        "test:build": "cp test/html/* build",
+        "test:build": "cpx \"test/html/*\" build",
         "test:run": "jest --color --ci",
         "test": "npm-run-all test:build test:run",
-        "clean": "find build -mindepth 1 -delete && find obj -mindepth 1 -delete && find cjs -mindepth 1 -delete"
+        "clean": "rimraf build && rimraf obj && rimraf cjs"
     },
     "jest": {
         "roots": [

diff --git a/scripts/build.js b/scripts/build.js
@@ -115,7 +115,7 @@ function docker(image = "emsdk") {
     if (process.env.PSP_CPU_COUNT) {
         cmd += ` --cpus="${parseInt(process.env.PSP_CPU_COUNT)}.0"`;
     }
-    cmd += ` -v $(pwd):/src -e PACKAGE=${process.env.PACKAGE} perspective/${image}`;
+    cmd += ` -v ${process.cwd()}:/src -e PACKAGE=${process.env.PACKAGE} perspective/${image}`;
     return cmd;
 }
 
@@ -127,7 +127,7 @@ function compileCPP(packageName) {
     }
     cmd += `&& emmake make -j${process.env.PSP_CPU_COUNT || os.cpus().length}`;
     if (process.env.PSP_DOCKER) {
-        cmd = `${docker()} bash -c 'cd cpp/${packageName}/obj && ${cmd}'`;
+        cmd = `${docker()} bash -c "cd cpp/${packageName}/obj && ${cmd}"`;
     } else {
         cmd = `cd ${BASE_DIRECTORY} && ${cmd}`;
     }
@@ -144,7 +144,7 @@ function lerna() {
 
 try {
     if (!process.env.PACKAGE || minimatch("perspective", process.env.PACKAGE)) {
-        execute("mkdir -p cpp/perspective/obj");
+        mkdirp("cpp/perspective/obj");
         compileCPP("perspective");
         RUNTIMES.map(compileRuntime);
     }

diff --git a/scripts/clean.js b/scripts/clean.js
@@ -30,7 +30,7 @@ function clean(dir) {
 }
 
 function clean_screenshots() {
-    execute("lerna exec -- mkdir -p screenshots");
+    execute("lerna exec -- mkdirp screenshots");
     execute(`lerna run clean:screenshots --ignore-missing ${process.env.PACKAGE ? `--scope=@jpmorganchase/${process.env.PACKAGE}` : ""}`);
 }
 

diff --git a/scripts/lint_cpp.js b/scripts/lint_cpp.js
@@ -25,9 +25,9 @@ function docker(image = "emsdk") {
 
 function lint(dir) {
     if (process.env.PSP_DOCKER) {
-        execute(docker() + ` bash -c 'diff -u <(cat ${dir}/*) <(clang-format -style=file ${dir}/*)'`);
+        execute(docker() + ` bash -c "diff -u <(cat ${dir}/*) <(clang-format -style=file ${dir}/*)"`);
     } else {
-        execute(`bash -c 'diff -u <(cat ${dir}/*) <(clang-format -style=file ${dir}/*)'`);
+        execute(`bash -c "diff -u <(cat ${dir}/*) <(clang-format -style=file ${dir}/*)"`);
     }
 }
 

diff --git a/scripts/test.js b/scripts/test.js
@@ -51,7 +51,7 @@ function slow_jest() {
 
 function docker() {
     console.log("-- Creating puppeteer docker image");
-    let cmd = "docker run -it --rm --shm-size=2g -u root -e PACKAGE=${PACKAGE} -v $(pwd):/src -w /src";
+    let cmd = `docker run -it --rm --shm-size=2g -u root ${process.env.PACKAGE ? `-e PACKAGE=${process.env.PACKAGE}` : ""} -v ${process.cwd()}:/src -w /src`;
     if (IS_WRITE) {
         console.log("-- Running the test suite in Write mode");
         cmd += " -e WRITE_TESTS=1";
@@ -74,7 +74,7 @@ function emsdk() {
     if (process.env.PSP_CPU_COUNT) {
         cmd += ` --cpus="${parseInt(process.env.PSP_CPU_COUNT)}.0"`;
     }
-    cmd += " -v $(pwd):/src -e PACKAGE=${PACKAGE} perspective/emsdk node scripts/test.js --private-emsdk " + args.join(" ");
+    cmd += ` -v ${process.cwd()}:/src ${process.env.PACKAGE ? `-e PACKAGE=${process.env.PACKAGE}` : ""} perspective/emsdk node scripts/test.js --private-emsdk ` + args.join(" ");
     return cmd;
 }