Merge pull request #363 from jpmorganchase/python-build-fix

Python build fix
finos · Jan 6, 2019 · 6ac68a9 · 6ac68a9
2 parents 2cc9c76 + 70a02a6
commit 6ac68a9
Show file tree

Hide file tree

Showing 10 changed files with 118 additions and 24 deletions.
diff --git a/.travis.yml b/.travis.yml
@@ -42,6 +42,7 @@ install:
 script:
   - if [ "$TEST" = "WASM" ]; then yarn lint; fi
   - if [ "$TEST" = "WASM" ]; then PSP_DOCKER=1 yarn build; fi
-  - if [ "$TEST" = "WASM" ]; then yarn test --docker --quiet; fi
-  - if [ "$TEST" = "CPP" ]; then docker build -f docker/cpp/Dockerfile .; fi
-  - if [ "$TEST" = "PYTHON" ]; then docker build -f docker/python/Dockerfile .; fi
+  - if [ "$TEST" = "WASM" ]; then PSP_DOCKER=1 yarn test --quiet; fi
+  - if [ "$TEST" = "CPP" ]; then PSP_DOCKER=1 yarn build_cpp; fi
+  - if [ "$TEST" = "CPP" ]; then PSP_DOCKER=1 yarn test_cpp; fi
+  - if [ "$TEST" = "PYTHON" ]; then PSP_DOCKER=1 yarn build_python; fi
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -57,13 +57,13 @@ else()
 endif()
 
 if (PSP_CPP_BUILD_TESTS)
-	message(WARNING "${Cyan}Building CPP tests${ColorReset}")
+	message(WARNING "${Cyan}Building C++ tests${ColorReset}")
 else()
-	message(WARNING "${Cyan}Skipping CPP tests${ColorReset}")
+	message(WARNING "${Cyan}Skipping C++ tests${ColorReset}")
 endif()
 
 if (NOT PSP_CPP_BUILD_STRICT)
-	message(WARNING "${Cyan}Building CPP without strict warnings${ColorReset}")
+	message(WARNING "${Cyan}Building C++ without strict warnings${ColorReset}")
 endif()
 
 #######################

diff --git a/docker/cpp/Dockerfile b/docker/cpp/Dockerfile
@@ -12,7 +12,4 @@ WORKDIR /usr/src/app
 ADD . /usr/src/app
 
 RUN apt-get update
-RUN apt-get -y install apt-transport-https libtbb-dev cmake libboost-all-dev
-
-RUN yarn
-RUN yarn build_cpp
+RUN apt-get -y install apt-transport-https libtbb-dev cmake libboost-all-dev
diff --git a/docker/python/Dockerfile b/docker/python/Dockerfile
@@ -28,6 +28,4 @@ RUN ln -s /usr/local/lib/libboost_python37.so /usr/local/lib/libboost_python.so
 RUN ln -s /usr/local/lib/libboost_numpy37.so /usr/local/lib/libboost_numpy.so
 
 RUN python3 -m pip install codecov nose2 mock flake8
-RUN python3 -m pip install -r python/requirements.txt
-
-RUN DOCKER=true cd python && make build && make test
+RUN python3 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0'
diff --git a/package.json b/package.json
@@ -38,13 +38,13 @@
         "build,test,bench": "npm run --silent build && npm run --silent test && npm run --silent bench",
         "build,bench": "npm run --silent build && npm run --silent bench",
         "build": "node scripts/build.js",
-        "build_cpp": "lerna run build:cpp ${PACKAGE:+--scope=@jpmorganchase/perspective} --stream",
-        "build_python": "cd python && python3 setup.py build",
+        "build_cpp": "node scripts/build_cpp.js",
+        "build_python": "node scripts/build_python.js",
         "install_python": "cd python && python3 setup.py install",
         "bench": "node scripts/bench.js",
         "docs": "lerna run docs --silent --stream",
         "test": "node scripts/test.js",
-        "test_cpp": "lerna run test:cpp ${PACKAGE:+--scope=@jpmorganchase/perspective} --stream",
+        "test_cpp": "node scripts/test_cpp.js",
         "clean": "find obj -mindepth 1 -delete && lerna run clean ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}} --stream",
         "preclean:screenshots": "lerna exec -- mkdir -p screenshots",
         "clean:screenshots": "lerna run clean:screenshots ${PACKAGE:+--scope=@jpmorganchase/${PACKAGE}}",

diff --git a/packages/perspective/package.json b/packages/perspective/package.json
@@ -18,15 +18,12 @@
     "scripts": {
         "bench": "node ../../scripts/bench.js",
         "prebuild": "mkdir -p build && mkdir -p obj",
-        "compile:cpp": "mkdir -p ../../cppbuild && cd ../../cppbuild && cmake ../ -DPSP_WASM_BUILD=0 -DPSP_CPP_BUILD=1 -DPSP_CPP_BUILD_TESTS=1 && make -j${PSP_CPU_COUNT-8}",
         "cpp": "npm-run-all build:cpp test:cpp",
         "build": "npm-run-all build:webpack",
-        "build:cpp": "npm-run-all compile:cpp",
         "build:webpack": "npm-run-all -p build:webpack:*",
         "build:webpack:umd": "webpack --color --config src/config/perspective.config.js",
         "build:webpack:node": "webpack --color --config src/config/perspective.node.config.js",
         "docs": "documentation build src/js/perspective.js -f md --shallow > README.md && cp README.md ../../docs/perspective.md",
-        "test:cpp": "npm-run-all build:cpp && ../../cppbuild/test/psp_test",
         "test:build": "cp test/html/* build",
         "test:run": "jest --color --ci",
         "test": "npm-run-all test:build test:run",

diff --git a/scripts/build.js b/scripts/build.js
@@ -81,7 +81,7 @@ function compileRuntime({inputFile, inputWasmFile, format, packageName}) {
 
     if (inputWasmFile) {
         console.log("-- Copying WASM file %s", inputWasmFile);
-        fs.copyFileSync(path.join(BUILD_DIRECTORY, inputWasmFile), path.join(path.join(OUTPUT_DIRECTORY, "build"), inputWasmFile));
+        fs.copyFileSync(path.join(BUILD_DIRECTORY, inputWasmFile), path.join(OUTPUT_DIRECTORY, "build", inputWasmFile));
     }
 
     console.debug("-- Creating wrapped js runtime");
@@ -101,23 +101,23 @@ function compileRuntime({inputFile, inputWasmFile, format, packageName}) {
         });
     }
 
-    fs.writeFileSync(path.join(path.join(OUTPUT_DIRECTORY, "obj"), inputFile), source);
+    fs.writeFileSync(path.join(OUTPUT_DIRECTORY, "obj", inputFile), source);
 }
 
-function emsdk() {
+function docker(image = "emsdk") {
     console.log("-- Creating emsdk docker image");
     let cmd = "docker run --rm -it";
     if (process.env.PSP_CPU_COUNT) {
         cmd += ` --cpus="${parseInt(process.env.PSP_CPU_COUNT)}.0"`;
     }
-    cmd += " -v $(pwd):/src -e PACKAGE=${PACKAGE} perspective/emsdk";
+    cmd += ` -v $(pwd):/src -e PACKAGE=${process.env.PACKAGE} perspective/${image}`;
     return cmd;
 }
 
 function compileCPP() {
     let cmd = `emcmake cmake ../ && emmake make -j${process.env.PSP_CPU_COUNT || os.cpus().length}`;
     if (process.env.PSP_DOCKER) {
-        cmd = `${emsdk()} bash -c 'cd obj && ${cmd}'`;
+        cmd = `${docker()} bash -c 'cd obj && ${cmd}'`;
     } else {
         cmd = `cd ${BASE_DIRECTORY} && ${cmd}`;
     }

diff --git a/scripts/build_cpp.js b/scripts/build_cpp.js
@@ -0,0 +1,35 @@
+/******************************************************************************
+ *
+ * Copyright (c) 2017, the Perspective Authors.
+ *
+ * This file is part of the Perspective library, distributed under the terms of
+ * the Apache License 2.0.  The full license can be found in the LICENSE file.
+ *
+ */
+
+const execSync = require("child_process").execSync;
+
+const execute = cmd => execSync(cmd, {stdio: "inherit"});
+
+function docker(image = "emsdk") {
+    console.log(`-- Creating ${image} docker image`);
+    let cmd = "docker run --rm -it";
+    if (process.env.PSP_CPU_COUNT) {
+        cmd += ` --cpus="${parseInt(process.env.PSP_CPU_COUNT)}.0"`;
+    }
+    cmd += ` -v $(pwd):/usr/src/app/cpp -w /usr/src/app/cpp/cppbuild perspective/${image}`;
+    return cmd;
+}
+
+try {
+    execute("mkdir -p cppbuild");
+    if (process.env.PSP_DOCKER) {
+        execute(docker("cpp") + " cmake ../ -DPSP_WASM_BUILD=0 -DPSP_CPP_BUILD=1 -DPSP_CPP_BUILD_TESTS=1");
+        execute(docker("cpp") + " make -j${PSP_CPU_COUNT-8}");
+    } else {
+        execute("cmake ../ -DPSP_WASM_BUILD=0 -DPSP_CPP_BUILD=1 -DPSP_CPP_BUILD_TESTS=1");
+        execute("make -j${PSP_CPU_COUNT-8}");
+    }
+} catch (e) {
+    process.exit(1);
+}
diff --git a/scripts/build_python.js b/scripts/build_python.js
@@ -0,0 +1,33 @@
+/******************************************************************************
+ *
+ * Copyright (c) 2017, the Perspective Authors.
+ *
+ * This file is part of the Perspective library, distributed under the terms of
+ * the Apache License 2.0.  The full license can be found in the LICENSE file.
+ *
+ */
+
+const execSync = require("child_process").execSync;
+
+const execute = cmd => execSync(cmd, {stdio: "inherit"});
+
+function docker(image = "emsdk") {
+    console.log(`-- Creating ${image} docker image`);
+    let cmd = "docker run --rm -it";
+    if (process.env.PSP_CPU_COUNT) {
+        cmd += ` --cpus="${parseInt(process.env.PSP_CPU_COUNT)}.0"`;
+    }
+    cmd += ` -v $(pwd):/usr/src/app/python -w /usr/src/app/python perspective/${image}`;
+    return cmd;
+}
+
+try {
+    let cmd = "python3 setup.py build";
+    if (process.env.PSP_DOCKER) {
+        execute(docker("python") + " " + cmd);
+    } else {
+        execute(cmd);
+    }
+} catch (e) {
+    process.exit(1);
+}
diff --git a/scripts/test_cpp.js b/scripts/test_cpp.js
@@ -0,0 +1,33 @@
+/******************************************************************************
+ *
+ * Copyright (c) 2017, the Perspective Authors.
+ *
+ * This file is part of the Perspective library, distributed under the terms of
+ * the Apache License 2.0.  The full license can be found in the LICENSE file.
+ *
+ */
+
+const execSync = require("child_process").execSync;
+
+const execute = cmd => execSync(cmd, {stdio: "inherit"});
+
+function docker(image = "emsdk") {
+    console.log(`-- Creating ${image} docker image`);
+    let cmd = "docker run --rm -it";
+    if (process.env.PSP_CPU_COUNT) {
+        cmd += ` --cpus="${parseInt(process.env.PSP_CPU_COUNT)}.0"`;
+    }
+    cmd += ` -v $(pwd):/usr/src/app/cpp -w /usr/src/app/cpp/cppbuild perspective/${image}`;
+    return cmd;
+}
+
+try {
+    execute("mkdir -p cppbuild");
+    if (process.env.PSP_DOCKER) {
+        execute(docker("cpp") + " ./test/psp_test");
+    } else {
+        execute("./test/psp_test");
+    }
+} catch (e) {
+    process.exit(1);
+}