Code coverage integration in SRF (nv-morpheus#105)

Steps to test/experiment: - Build in `debug` mode with `SRF_WITH_CODECOV=ON` - Run all tests/code that will touch libsrf/libpysrf `cd ./build && ctest && pytest ./python/tests` - Build the gcovr report `cmake --build ./build --target gcovr-html-report` - Open the HTML report `xdg-open ./build/gcovr-html-report/index.html` Authors: - Devin Robison (https://github.com/drobison00) Approvers: - David Gardner (https://github.com/dagardner-nv) - Ryan Olson (https://github.com/ryanolson) - Michael Demoret (https://github.com/mdemoret-nv) URL: nv-morpheus#105
AyodeAwe · Jul 13, 2022 · 88a2489 · 88a2489
1 parent 104fcbf
commit 88a2489
Show file tree

Hide file tree

Showing 13 changed files with 930 additions and 11 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -31,8 +31,9 @@ option(SRF_USE_CCACHE "Enable caching compilation results with ccache" OFF)
 option(SRF_USE_CLANG_TIDY "Enable running clang-tidy as part of the build process" OFF)
 option(SRF_USE_CONDA "Enables finding dependencies via conda instead of vcpkg. Note: This will disable vcpkg. All dependencies must be installed first in the conda environment" ON)
 option(SRF_USE_IWYU "Enable running include-what-you-use as part of the build process" OFF)
+option(SRF_ENABLE_CODECOV "Enable gcov code coverage" OFF)
 
-set(SRF_RAPIDS_VERSION "22.06" CACHE STRING "Which version of RAPIDs to build for. Sets default versions for RAPIDs CMake and RMM.")
+set(SRF_RAPIDS_VERSION "22.06" CACHE STRING "Which version of RAPIDS to build for. Sets default versions for RAPIDS CMake and RMM.")
 
 set(SRF_CACHE_DIR "${CMAKE_SOURCE_DIR}/.cache" CACHE PATH "Directory to contain all CPM and CCache data")
 mark_as_advanced(SRF_CACHE_DIR)
@@ -131,10 +132,13 @@ endif()
 
 add_subdirectory(protos)
 
-# ###################################
+####################################
 # - Post dependencies setup --------
 include(cmake/setup_compiler.cmake)
 
+# Setup code coverage components
+include(cmake/setup_coverage.cmake)
+
 # Setup IWYU if enabled
 include(cmake/setup_iwyu.cmake)
 

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -27,13 +27,25 @@ More information can be found at: [Contributor Code of Conduct](CODE_OF_CONDUCT.
 
 1. Find an issue to work on. The best way is to look for issues with the [good first issue](https://github.com/NVIDIA/SRF/issues) label.
 2. Comment on the issue stating that you are going to work on it.
-3. Code! Make sure to update unit tests! Ensure the [license headers are set properly](#Licensing).
+3. Code! Make sure to update unit tests and confirm that test coverage has not decreased (see below)! Ensure the 
+[license headers are set properly](#Licensing).
 4. When done, [create your pull request](https://github.com/NVIDIA/SRF/compare).
 5. Wait for other developers to review your code and update code as needed.
-6. Once reviewed and approved, a SRF developer will merge your pull request.
+6. Once reviewed and approved, an SRF developer will merge your pull request.
 
 Remember, if you are unsure about anything, don't hesitate to comment on issues and ask for clarifications!
 
+## Unit testing and Code Coverage
+Prior to submitting a pull request, you should ensure that all your contributed code is covered by unit tests, and that
+unit test coverage percentages have not decreased (even better if they've increased). To test, from the SRF root 
+directory:
+
+1. Generate a code coverage report and ensure your additions are covered.
+   1. In some environments you may need to set CUDAToolkit_ROOT
+      1. `export CUDAToolkit_ROOT=[YOUR_CUDA_ROOT]`
+   2. `./scripts/gen_coverage.sh`
+   3. open ./build/gcovr-html-report/index.html
+
 ## Seasoned developers
 
 Once you have gotten your feet wet and are more comfortable with the code, you can look at the prioritized issues for our next release in our [project boards](https://github.com/NVIDIA/SRF/projects).

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
@@ -38,7 +38,7 @@ pipeline {
         stage('Builds') {
             failFast true
             parallel {
-                stage('Build:linux:x86_64:gcc') {
+                stage('Build:linux:x86_64:gcc:release') {
                     options {
                         timeout(time: 1, unit: 'HOURS')
                     }
@@ -69,6 +69,38 @@ pipeline {
                         }
                     }
                 }
+                stage('Build:linux:x86_64:gcc:debug') {
+                    options {
+                        timeout(time: 1, unit: 'HOURS')
+                    }
+                    environment {
+                        BUILD_CC= "gcc-coverage"
+                        PARALLEL_LEVEL = '16'
+                        HOME = "${WORKSPACE}"
+                    }
+                    agent {
+                        docker {
+                            image 'gpuci/rapidsai-driver:21.10-cuda11.4-devel-ubuntu20.04-py3.8'
+                            label 'cpu4'
+                        }
+                    }
+                    steps {
+                        cleanWs(
+                            deleteDirs: true,
+                            externalDelete: 'sudo rm -rf %s'
+                        )
+                        checkout scm
+                        withCredentials([[
+                            $class: 'AmazonWebServicesCredentialsBinding',
+                            credentialsId: "aws-s3-gpuci",
+                            accessKeyVariable: 'AWS_ACCESS_KEY_ID',
+                            secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
+                        ]])
+                        {
+                            sh "${WORKSPACE}/ci/scripts/jenkins/build.sh"
+                        }
+                    }
+                }
                 stage('Build:linux:x86_64:clang') {
                     options {
                         timeout(time: 1, unit: 'HOURS')
@@ -132,13 +164,44 @@ pipeline {
                         }
                     }
                 }
-
             }
         }
         stage('Tests') {
             failFast true
             parallel {
-                stage('Test') {
+                stage('TestDebug') {
+                     options {
+                         timeout(time: 1, unit: 'HOURS')
+                     }
+                     agent {
+                         docker {
+                             image 'gpuci/rapidsai:21.10-cuda11.4-devel-ubuntu20.04-py3.8'
+                             label 'driver-495'
+                             args '--cap-add=sys_nice --runtime "nvidia" -e "NVIDIA_VISIBLE_DEVICES=$EXECUTOR_NUMBER"'
+                         }
+                     }
+                     environment {
+                         BUILD_TYPE = "Debug"
+                         HOME = "${WORKSPACE}"
+                     }
+                     steps {
+                         cleanWs(
+                             deleteDirs: true,
+                             externalDelete: 'sudo rm -rf %s'
+                         )
+                         checkout scm
+                         withCredentials([[
+                             $class: 'AmazonWebServicesCredentialsBinding',
+                             credentialsId: "aws-s3-gpuci",
+                             accessKeyVariable: 'AWS_ACCESS_KEY_ID',
+                             secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
+                         ]])
+                         {
+                             sh "${WORKSPACE}/ci/scripts/jenkins/test.sh"
+                         }
+                     }
+                }
+                stage('TestRelease') {
                     options {
                         timeout(time: 1, unit: 'HOURS')
                     }
@@ -150,6 +213,7 @@ pipeline {
                         }
                     }
                     environment {
+                        BUILD_TYPE = "Release"
                         HOME = "${WORKSPACE}"
                     }
                     steps {

diff --git a/ci/conda/environments/dev_env.yml b/ci/conda/environments/dev_env.yml
@@ -15,6 +15,7 @@ dependencies:
   - doxygen=1.9.2
   - flatbuffers=2.0
   - gcc_linux-64=9.4
+  - gcovr=5.0
   - gflags=2.2
   - git>=2.35.3 # Needed for wildcards on safe.directory
   - glog=0.6
@@ -23,6 +24,8 @@ dependencies:
   - grpc-cpp=1.45
   - gtest=1.10
   - gxx_linux-64=9.4
+  - jinja2=3.0
+  - lcov=1.15
   - libhwloc=2.5
   - librmm=22.06
   - libtool

diff --git a/ci/scripts/copyright.py b/ci/scripts/copyright.py
@@ -33,7 +33,7 @@
     re.compile(r"[.]flake8[.]cython$"),
     re.compile(r"meta[.]yaml$")
 ]
-ExemptFiles = ['.cache', 'build', 'cmake/LibFindMacros.cmake', 'versioneer.py']
+ExemptFiles = ['.cache', 'build', 'cmake/deps/Configure_gcov.cmake', 'cmake/LibFindMacros.cmake', 'versioneer.py']
 
 # this will break starting at year 10000, which is probably OK :)
 CheckSimple = re.compile(r"Copyright *(?:\(c\))? *(\d{4}),? *NVIDIA C(?:ORPORATION|orporation)")

diff --git a/ci/scripts/jenkins/build.sh b/ci/scripts/jenkins/build.sh
@@ -39,6 +39,11 @@ if [[ "${BUILD_CC}" == "gcc" ]]; then
     gcc --version
     g++ --version
     CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES} ${CMAKE_CACHE_FLAGS}"
+elif [[ "${BUILD_CC}" == "gcc-coverage" ]]; then
+    gpuci_logger "Building with GCC with gcov profile '-g -fprofile-arcs -ftest-coverage"
+    gcc --version
+    g++ --version
+    CMAKE_FLAGS="${CMAKE_BUILD_ALL_FEATURES} ${CMAKE_BUILD_WITH_CODECOV} ${CMAKE_CACHE_FLAGS}"
 else
     gpuci_logger "Installing Clang"
     mamba env update -q -n srf --file ${SRF_ROOT}/ci/conda/environments/clang_env.yml

diff --git a/ci/scripts/jenkins/common.sh b/ci/scripts/jenkins/common.sh
@@ -31,6 +31,7 @@ export NUM_PROC=${PARALLEL_LEVEL:-$(nproc)}
 export CONDA_ENV_YML="${SRF_ROOT}/ci/conda/environments/dev_env.yml"
 
 export CMAKE_BUILD_ALL_FEATURES="-DCMAKE_MESSAGE_CONTEXT_SHOW=ON -DSRF_BUILD_BENCHMARKS=ON -DSRF_BUILD_EXAMPLES=ON -DSRF_BUILD_PYTHON=ON -DSRF_BUILD_TESTS=ON -DSRF_USE_CONDA=ON"
+export CMAKE_BUILD_WITH_CODECOV="-DCMAKE_BUILD_TYPE=Debug -DSRF_ENABLE_CODECOV=ON"
 
 # Set the depth to allow git describe to work
 export GIT_DEPTH=1000

diff --git a/ci/scripts/jenkins/test.sh b/ci/scripts/jenkins/test.sh
@@ -34,7 +34,12 @@ REPORTS_DIR="${WORKSPACE_TMP}/reports"
 mkdir -p ${WORKSPACE_TMP}/reports
 
 # ctest requires cmake to be configured in order to locate tests
-cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} .
+
+if [[ "${BUILD_TYPE}" == "Debug" ]]; then
+  cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} ${CMAKE_BUILD_WITH_CODECOV} .
+else
+  cmake -B build -G Ninja ${CMAKE_BUILD_ALL_FEATURES} .
+fi
 
 gpuci_logger "Running C++ Tests"
 cd ${SRF_ROOT}/build
@@ -59,13 +64,22 @@ pytest -v --junit-xml=${WORKSPACE_TMP}/report_pytest.xml
 PYTEST_RESULTS=$?
 set -e
 
+if [[ "${BUILD_TYPE}" == "Debug" ]]; then
+  gpuci_logger "Generating codecov report"
+  cd ${SRF_ROOT}
+  cmake --build build --target gcovr-html-report
+
+  gpuci_logger "Archiving codecov report"
+  tar cfj ${WORKSPACE_TMP}/coverage_reports.tar.bz ${SRF_ROOT}/build/gcovr-html-report
+  aws s3 cp ${WORKSPACE_TMP}/coverage_reports.tar.bz "${ARTIFACT_URL}/coverage_reports.tar.bz"
+fi
+
 gpuci_logger "Archiving test reports"
 cd $(dirname ${REPORTS_DIR})
 tar cfj ${WORKSPACE_TMP}/test_reports.tar.bz $(basename ${REPORTS_DIR})
 
 gpuci_logger "Pushing results to ${DISPLAY_ARTIFACT_URL}/"
 aws s3 cp ${WORKSPACE_TMP}/test_reports.tar.bz "${ARTIFACT_URL}/test_reports.tar.bz"
 
-
 TEST_RESULTS=$(($CTEST_RESULTS+$PYTEST_RESULTS))
 exit ${TEST_RESULTS}