build minimal arrow from source for linux - finish upgrade to 0.15

finos · Oct 18, 2019 · a520b17 · a520b17
1 parent 35c9c08
commit a520b17
Show file tree

Hide file tree

Showing 5 changed files with 28 additions and 14 deletions.
diff --git a/cmake/modules/FindPyArrow.cmake b/cmake/modules/FindPyArrow.cmake
@@ -47,7 +47,11 @@ find_path(PYTHON_PYARROW_INCLUDE_DIR arrow/python/api.h
   HINTS "${__pyarrow_path}" "${PYTHON_INCLUDE_PATH}" NO_DEFAULT_PATH)
 
 set(PYTHON_PYARROW_LIBRARY_DIR ${__pyarrow_library_dirs})
-set(PYTHON_PYARROW_SHARED_LIBRARY ${__pyarrow_library_dirs}/${CMAKE_SHARED_LIBRARY_PREFIX}arrow.15${CMAKE_SHARED_LIBRARY_SUFFIX})
+
+if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
+  # Link against pre-built libarrow on MacOS
+  set(PYTHON_PYARROW_SHARED_LIBRARY ${__pyarrow_library_dirs}/${CMAKE_SHARED_LIBRARY_PREFIX}arrow.15${CMAKE_SHARED_LIBRARY_SUFFIX})
+endif()
 
 # DONT USE, could conflict on arrow lib
 # set(PYTHON_PYARROW_LIBRARIES ${__pyarrow_libraries})

diff --git a/cpp/perspective/CMakeLists.txt b/cpp/perspective/CMakeLists.txt
@@ -297,11 +297,14 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
 		include_directories( ${PYTHON_NUMPY_INCLUDE_DIR})
 
 		find_package(PyArrow 0.15 REQUIRED)
+
 		if(NOT PYTHON_PYARROW_FOUND)
 			message(FATAL_ERROR "${Red}PyArrow could not be located${ColorReset}")
 		else()
 			message(WARNING "${Cyan}PyArrow found: ${PYTHON_PYARROW_INCLUDE_DIR}${ColorReset}")
-			message(WARNING "${Cyan}Using pre-built libarrow from: ${PYTHON_PYARROW_INCLUDE_DIR}${ColorReset}")
+			if (CMAKE_SYSTEM_NAME MATCHES "Darwin")
+				message(WARNING "${Cyan}Using pre-built libarrow from: ${PYTHON_PYARROW_INCLUDE_DIR}${ColorReset}")
+			endif()
 		endif()
 		include_directories(${PYTHON_PYARROW_INCLUDE_DIR})
 		link_directories(${PYTHON_PYARROW_LIBRARY_DIR})
@@ -313,7 +316,8 @@ endif()
 psp_build_dep("hopscotch" "../../cmake/hopscotch.txt.in")
 psp_build_dep("ordered-map" "../../cmake/ordered-map.txt.in")
 
-if (NOT PSP_PYTHON_BUILD)
+# For all non-MacOS builds, or if building WASM/CPP, build minimal arrow from source
+if (NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin" OR NOT PSP_PYTHON_BUILD)
 	# build arrow + dependencies from source for Emscripten and C++
 	message("${Cyan}Building minimal Apache Arrow${ColorReset}")
 
@@ -462,8 +466,15 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
 
 		target_compile_definitions(psp PRIVATE PSP_ENABLE_PYTHON=1)
 
-		# Link against libarrow 0.15.0 in pyarrow directory
-		target_link_libraries(psp ${PYTHON_PYARROW_SHARED_LIBRARY})
+		# On MacOS, link against libarrow 0.15.0 in pyarrow directory
+		# This does not work for Linux or Windows - prebuilt binaries for Linux must be backward compatible with GCC 4.8,
+		# which means they use the old ABI: https://github.com/blue-yonder/turbodbc/issues/102#issuecomment-312222949
+		if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+			target_link_libraries(psp ${PYTHON_PYARROW_SHARED_LIBRARY})
+		else()
+			target_link_libraries(psp arrow)
+			target_link_libraries(psp ${Boost_FILESYSTEM_LIBRARY})
+		endif()
 
 		target_link_libraries(psp ${PYTHON_LIBRARIES})
 		set(CMAKE_SHARED_LIBRARY_SUFFIX .so)
@@ -476,6 +487,11 @@ elseif(PSP_CPP_BUILD OR PSP_PYTHON_BUILD)
 		target_link_libraries(binding tbb)
 
 		# Link again libarrow from pyarrow
+		if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+			target_link_libraries(binding ${PYTHON_PYARROW_SHARED_LIBRARY})
+		else()
+			target_link_libraries(psp ${PYTHON_PYARROW_LIBRARIES})
+		endif()
 		target_link_libraries(binding ${PYTHON_PYARROW_SHARED_LIBRARY})
 		target_link_libraries(binding ${PYTHON_LIBRARIES})
 		########################

diff --git a/docker/python/Dockerfile b/docker/python/Dockerfile
@@ -28,7 +28,7 @@ RUN ln -s /usr/local/lib/libboost_python37.so /usr/local/lib/libboost_python.so
 RUN ln -s /usr/local/lib/libboost_numpy37.so /usr/local/lib/libboost_numpy.so
 
 RUN python3 -m pip install codecov nose2 mock flake8 pytest pytest-cov traitlets ipywidgets faker psutil zerorpc
-RUN python3 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow==0.15.0'
+RUN python3 -m pip install 'numpy>=1.13.1' 'pandas>=0.22.0' 'pyarrow==0.14.1'
 
 RUN npm install --global yarn
 RUN yarn --version

diff --git a/scripts/build_python.js b/scripts/build_python.js
@@ -16,7 +16,7 @@ const IS_DOCKER = process.env.PSP_DOCKER;
 const VALID_TARGETS = ["node", "table"];
 const HAS_TARGET = args.indexOf("--target") != -1;
 
-function docker(target = "perspective", image = "emsdk") {
+function docker(target = "perspective", image = "python") {
     console.log(`-- Creating ${image} docker image`);
     let cmd = "docker run --rm -it";
     if (process.env.PSP_CPU_COUNT) {
@@ -38,12 +38,6 @@ try {
     }
 
     let cmd;
-    let pip_target = "";
-
-    if (IS_DOCKER) {
-        pip_target = " --target=`pwd`";
-    }
-
     let build_cmd = `python3 setup.py build`;
 
     if (IS_DOCKER) {

diff --git a/scripts/test_python.js b/scripts/test_python.js
@@ -16,7 +16,7 @@ const VALID_TARGETS = ["node", "table"];
 const HAS_TARGET = args.indexOf("--target") != -1;
 const VERBOSE = args.indexOf("--verbose") != -1;
 
-function docker(target = "perspective", image = "emsdk") {
+function docker(target = "perspective", image = "python") {
     console.log(`-- Creating ${image} docker image`);
     let cmd = "docker run --rm -it";
     if (process.env.PSP_CPU_COUNT) {