Improvements to the python module generation CMake code (nv-morpheus#298

) This is the corresponding update to incorporate changes from nv-morpheus/utilities#17. Mainly this reorganizes the python CMake checks so they only get executed once. Significantly speeding up the configure step. Authors: - Michael Demoret (https://github.com/mdemoret-nv) Approvers: - David Gardner (https://github.com/dagardner-nv) - Devin Robison (https://github.com/drobison00) - Christopher Harris (https://github.com/cwharris) URL: nv-morpheus#298
AyodeAwe · Mar 15, 2023 · d0c9ae6 · d0c9ae6
1 parent a5be441
commit d0c9ae6
Show file tree

Hide file tree

Showing 10 changed files with 28 additions and 53 deletions.
diff --git a/.gitmodules b/.gitmodules
@@ -1,4 +1,4 @@
 [submodule "morpheus_utils"]
 	path = external/utilities
 	url = https://github.com/nv-morpheus/utilities.git
-	branch = branch-23.01
+	branch = branch-23.03
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -78,7 +78,7 @@ morpheus_utils_initialize_package_manager(
 morpheus_utils_initialize_cuda_arch(mrc)
 
 project(mrc
-  VERSION 23.01.00
+  VERSION 23.03.00
   LANGUAGES C CXX
 )
 

diff --git a/docs/quickstart/CMakeLists.txt b/docs/quickstart/CMakeLists.txt
@@ -17,23 +17,29 @@ list(APPEND CMAKE_MESSAGE_CONTEXT "quickstart")
 
 cmake_minimum_required(VERSION 3.24 FATAL_ERROR)
 
+set(MRC_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
+mark_as_advanced(MRC_CACHE_DIR)
+
 # Add the Conda environment to the prefix path and add the CMake files
 list(PREPEND CMAKE_PREFIX_PATH "$ENV{CONDA_PREFIX}")
 list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../cmake")
+list(PREPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../external/utilities/cmake")
 
-morpheus_utils_python_modules_ensure_python3()
+include(morpheus_utils/load)
 
 project(mrc-quickstart
-  VERSION 23.01
+  VERSION 23.03
   LANGUAGES C CXX
 )
 
+morpheus_utils_initialize_cpm(MRC_CACHE_DIR)
+
 # Ensure CPM is initialized
 rapids_cpm_init()
 
 # Set the option prefix to match the outer project before including. Must be before find_package(mrc)
 set(OPTION_PREFIX "MRC")
-include(morpheus_utils/python/register_api)
+morpheus_utils_python_ensure_loaded()
 
 rapids_find_package(mrc REQUIRED)
 rapids_find_package(CUDAToolkit REQUIRED)

diff --git a/docs/quickstart/compile.sh b/docs/quickstart/compile.sh
@@ -24,8 +24,8 @@ echo "Runing CMake configure..."
 cmake -B ${BUILD_DIR} -GNinja \
    -DCMAKE_MESSAGE_CONTEXT_SHOW=ON \
    -DMRC_PYTHON_INPLACE_BUILD:BOOL=ON \
-   -DMRC_PYTHON_PERFORM_INSTALL:BOOL=ON `# Ensure all of the libraries are installed` \
-   ${CMAKE_CONFIGURE_EXTRA_ARGS:-""} .
+   -DMRC_PYTHON_PERFORM_INSTALL:BOOL=ON # Ensure all of the libraries are installed` \
+   ${CMAKE_CONFIGURE_EXTRA_ARGS:+CMAKE_CONFIGURE_EXTRA_ARGS} .
 
 echo "Running CMake build..."
 cmake --build ${BUILD_DIR} -j "$@"
diff --git a/docs/quickstart/environment_cpp.yml b/docs/quickstart/environment_cpp.yml
@@ -31,7 +31,7 @@ dependencies:
   - python=3.8
   - scikit-build>=0.12
   - spdlog=1.8.5
-  - mrc=23.01
+  - mrc=23.03
   - sysroot_linux-64=2.17
   - pip:
     - cython

diff --git a/docs/quickstart/hybrid/mrc_qs_hybrid/common/CMakeLists.txt b/docs/quickstart/hybrid/mrc_qs_hybrid/common/CMakeLists.txt
@@ -15,8 +15,6 @@
 
 morpheus_utils_add_pybind11_library(
   data
-  # MODULE_ROOT
-  # ${QUICKSTART_HYBRID_HOME}
   SOURCE_FILES
     data.cpp
   LINK_TARGETS
@@ -30,8 +28,6 @@ target_include_directories(${common_data_target}
     ./include
 )
 
-morpheus_utils_inplace_build_copy(${common_data_target} ${CMAKE_CURRENT_SOURCE_DIR})
-
 morpheus_utils_add_pybind11_library(
   nodes
   SOURCE_FILES
@@ -42,8 +38,6 @@ morpheus_utils_add_pybind11_library(
     nodes_data_target
 )
 
-morpheus_utils_inplace_build_copy(${nodes_data_target} ${CMAKE_CURRENT_SOURCE_DIR})
-
 # Set this variable in the parent scope so other examples can link to it
 set(common_data_target ${common_data_target} PARENT_SCOPE)
 set(nodes_data_target ${nodes_data_target} PARENT_SCOPE)
diff --git a/docs/quickstart/hybrid/mrc_qs_hybrid/ex00_wrap_data_objects/CMakeLists.txt b/docs/quickstart/hybrid/mrc_qs_hybrid/ex00_wrap_data_objects/CMakeLists.txt
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-morpheus_utils_add_pybind11_module(
+mrc_quickstart_add_pybind11_module(
   data
   MODULE_ROOT
     ${QUICKSTART_HYBRID_HOME}
@@ -24,5 +24,3 @@ morpheus_utils_add_pybind11_module(
   OUTPUT_TARGET
     data_target
 )
-
-morpheus_utils_inplace_build_copy(${data_target} ${CMAKE_CURRENT_SOURCE_DIR})
diff --git a/docs/quickstart/hybrid/mrc_qs_hybrid/ex01_wrap_nodes/CMakeLists.txt b/docs/quickstart/hybrid/mrc_qs_hybrid/ex01_wrap_nodes/CMakeLists.txt
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-morpheus_utils_add_pybind11_module(
+mrc_quickstart_add_pybind11_module(
   nodes
   MODULE_ROOT
     ${QUICKSTART_HYBRID_HOME}
@@ -24,5 +24,3 @@ morpheus_utils_add_pybind11_module(
   OUTPUT_TARGET
     nodes_target
 )
-
-morpheus_utils_inplace_build_copy(${nodes_target} ${CMAKE_CURRENT_SOURCE_DIR})
diff --git a/external/utilities b/external/utilities
diff --git a/python/CMakeLists.txt b/python/CMakeLists.txt
@@ -17,17 +17,10 @@ list(APPEND CMAKE_MESSAGE_CONTEXT "python")
 
 find_package(CUDAToolkit REQUIRED)
 
-# Get the project name in uppercase if OPTION_PREFIX is not defined
-if(NOT DEFINED OPTION_PREFIX)
-  string(TOUPPER "${PROJECT_NAME}" OPTION_PREFIX)
-endif()
 
-option(${OPTION_PREFIX}_PYTHON_INPLACE_BUILD "Whether or not to copy built python modules back to the source tree for debug purposes." OFF)
-option(${OPTION_PREFIX}_PYTHON_PERFORM_INSTALL "Whether or not to automatically `pip install` any built python library. WARNING: This may overwrite any existing installation of the same name." OFF)
-option(${OPTION_PREFIX}_PYTHON_BUILD_STUBS "Whether or not to generated .pyi stub files for C++ Python modules. Disable to avoid requiring loading the NVIDIA GPU Driver during build" ON)
+# Ensure python is configured
+morpheus_utils_python_ensure_loaded()
 
-set(Python3_FIND_VIRTUALENV "FIRST")
-set(Python3_FIND_STRATEGY "LOCATION")
 
 morpheus_utils_print_python_info()
 
@@ -42,29 +35,13 @@ morpheus_utils_add_python_sources(
     ${pymrc_test_files}
 )
 
+# Build the pymrc library
+add_subdirectory(mrc/_pymrc)
 
-# Save the root of the python for relative paths
-set(MRC_PY_ROOT ${CMAKE_CURRENT_SOURCE_DIR})
-
-# A common macro for adding some default arguments to add_pybind11_module
-macro(mrc_add_pybind11_module)
-  # Build up the common arguments for add_pybind11_module
-  set(_common_args)
-  list(APPEND _common_args "LINK_TARGETS" "pymrc")
-
-  if(MRC_PYTHON_INPLACE_BUILD)
-    list(APPEND _common_args "COPY_INPLACE")
-  endif()
-
-  if(MRC_PYTHON_BUILD_STUBS)
-    list(APPEND _common_args "BUILD_STUBS")
-  endif()
-
-  # Forward all common arguments plus any arguments passed in
-  morpheus_utils_add_pybind11_module(${ARGN} ${_common_args})
-endmacro()
+# Set the default link targets to avoid repeating this
+morpheus_utils_python_package_set_default_link_targets(pymrc)
 
-add_subdirectory(mrc/_pymrc)
+# Now add the python bindings
 add_subdirectory(mrc/core)
 
 # ##################################################################################################
@@ -78,12 +55,14 @@ if(MRC_BUILD_TESTS)
 endif()
 
 # Complete the python package
-set(extra_args "")
-
 if(MRC_PYTHON_INPLACE_BUILD)
   list(APPEND extra_args "IS_INPLACE")
 endif()
 
+if(MRC_PYTHON_BUILD_WHEEL)
+  list(APPEND extra_args "BUILD_WHEEL")
+endif()
+
 if(MRC_PYTHON_PERFORM_INSTALL)
   list(APPEND extra_args "INSTALL_WHEEL")
 endif()
+1 −1		ci/runner/build_and_push.sh
+30 −0		cmake/morpheus_utils/environment_config/cpm/register_api.cmake
+3 −5		cmake/morpheus_utils/environment_config/rapids_cmake/register_api.cmake
+0 −2		cmake/morpheus_utils/environment_config/register_api.cmake
+2 −2		cmake/morpheus_utils/package_config/boost/Configure_boost.cmake
+1 −1		cmake/morpheus_utils/package_config/cudf/Configure_cudf.cmake
+0 −29		cmake/morpheus_utils/package_config/ensure_cpm_init.cmake
+1 −1		cmake/morpheus_utils/package_config/expected/Configure_expected.cmake
+1 −1		cmake/morpheus_utils/package_config/glog/Configure_glog.cmake
+1 −1		cmake/morpheus_utils/package_config/hwloc/Configure_hwloc.cmake
+1 −1		cmake/morpheus_utils/package_config/libcudacxx/Configure_libcudacxx.cmake
+6 −4		cmake/morpheus_utils/package_config/matx/Configure_matx.cmake
+4 −2		cmake/morpheus_utils/package_config/mrc/Configure_mrc.cmake
+1 −1		cmake/morpheus_utils/package_config/prometheus/Configure_prometheus.cmake
+2 −1		cmake/morpheus_utils/package_config/pybind11/Configure_pybind11.cmake
+1 −1		cmake/morpheus_utils/package_config/rdkafka/Configure_rdkafka.cmake
+9 −0		cmake/morpheus_utils/package_config/register_api.cmake
+1 −1		cmake/morpheus_utils/package_config/rmm/Configure_rmm.cmake
+1 −1		cmake/morpheus_utils/package_config/rxcpp/Configure_rxcpp.cmake
+1 −1		cmake/morpheus_utils/package_config/taskflow/Configure_taskflow.cmake
+1 −2		cmake/morpheus_utils/package_config/triton_client/Configure_triton_client.cmake
+1 −1		cmake/morpheus_utils/package_config/ucx/Configure_ucx.cmake
+68 −0		cmake/morpheus_utils/python/configure.cmake
+26 −0		cmake/morpheus_utils/python/ensure_cython.cmake
+24 −0		cmake/morpheus_utils/python/ensure_pybind11.cmake
+32 −12		cmake/morpheus_utils/python/ensure_python3.cmake
+37 −0		cmake/morpheus_utils/python/ensure_sk_build.cmake
+22 −8		cmake/morpheus_utils/python/pip_gen_depfile.py
+0 −70		cmake/morpheus_utils/python/python_config_macros.cmake
+47 −19		cmake/morpheus_utils/python/python_module_tools.cmake
+26 −7		cmake/morpheus_utils/python/register_api.cmake