Script for uploading code coverage data to dashboard (microsoft#1209)

- Added Python script to post the code coverage data to the MySQL table used for dashboard - Added a build job to run a windows cpu debug build on every merge on master, and run the script - Removed the code coverage step from the CI build
iiSeymour · Jun 16, 2019 · 5f21eed · 5f21eed
1 parent 0873158
commit 5f21eed
Show file tree

Hide file tree

Showing 4 changed files with 165 additions and 5 deletions.
diff --git a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml
@@ -0,0 +1,41 @@
+# Jobs to run on master after every successful merge is committed
+jobs:
+- job: Windows_Debug_CPU_x64
+  pool: 'Win-CPU'
+  variables:
+    - group: dashboard_mysql_secret
+    - name: buildDirectory
+      value: '$(Build.BinariesDirectory)'
+    - name: buildConfig
+      value: 'Debug'
+    - name: buildArch
+      value: 'x64'
+
+  steps:
+    - template: templates/set-test-data-variables-step.yml
+    - template: templates/set-version-number-variables-step.yml
+
+    - template: templates/windows-build-tools-setup-steps.yml
+      parameters:
+        EnvSetupScript: 'setup_env.bat'
+        buildArch: 'amd64'  # amd64 is needed for vcvars target arch 
+        setVcvars: false
+
+    - template: templates/windows-build-and-test-steps.yml
+      parameters:
+        buildAdditionalParams: ' --use_openmp '
+        # --use_openmp is added, similar to the build config of the C-api package distribution.
+        # MLAS(default) with OpenMP is known to produce better performance (latency) than without OpenMP. 
+        buildArch: $(buildArch)
+        msbuildPlatform: $(buildArch)
+        buildConfig: $(buildConfig)
+
+    - template: templates/windows-code-coverage-steps.yml
+      parameters:
+        OpenCppCoverageExe: '$(Build.BinariesDirectory)\OpenCppCoverage\OpenCppCoverage.exe'
+        GitCommitHash: $(OnnxRuntimeGitCommitHash)    
+        PublishReport: true
+        PostToDashboard: true
+
+    - template: templates/clean-agent-build-directory-step.yml
+
diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-code-coverage-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-code-coverage-steps.yml
@@ -1,20 +1,42 @@
-# sets variables $(TestDataUrl) and $(TestDataChecksum)
+# Runs steps to generate and publish code coverage data
 
 parameters:
   OpenCppCoverageExe: '$(Build.BinariesDirectory)\OpenCppCoverage\OpenCppCoverage.exe'
-
+  GitCommitHash: $(OnnxRuntimeGitCommitHash)
+  PublishReport: true
+  PostToDashBoard: false
+  ContinueOnError: false
 steps:
 
 - task: PowerShell@2
   displayName: 'Run Test Coverage'
   inputs:
     filePath: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\run_OpenCppCoverage.ps1'
     arguments: '-OpenCppCoverageExe:"${{parameters.OpenCppCoverageExe}}" -SourceRoot:"$(Build.SourcesDirectory)" -BuildRoot:"$(Build.BinariesDirectory)" -LocalBuild:$false'
-  continueOnError: true  
+  continueOnError: ${{parameters.ContinueOnError}}  
 
 - task: PublishCodeCoverageResults@1
   inputs:
     codeCoverageTool: 'Cobertura'
     summaryFileLocation: '$(Build.BinariesDirectory)/**/cobertura.xml'
     reportDirectory: '$(Build.BinariesDirectory)/**/OpenCppCoverageResults'
-  continueOnError: true
+  continueOnError: ${{parameters.ContinueOnError}}
+  condition: ${{parameters.PublishReport}}
+
+- task: CmdLine@1
+  displayName: 'Install mysql-connector-python conda module'
+  inputs:
+    filename: '$(Build.BinariesDirectory)\packages\python\scripts\conda.exe'
+    arguments: 'install -q --insecure -y mysql-connector-python'
+  timeoutInMinutes: 10
+
+- task: BatchScript@1
+  displayName: 'Post code coverage data to the Dashboard'
+  inputs:
+    filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
+    arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\post_code_coverage_to_dashboard.py --commit_hash=${{parameters.GitCommitHash}} --report_file="$(Build.BinariesDirectory)/Debug/Debug/cobertura.xml" --report_url="https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)"'
+    workingFolder: "$(Build.BinariesDirectory)"
+  continueOnError: ${{parameters.ContinueOnError}}
+  condition: ${{parameters.PostToDashBoard}}
+  env: 
+    DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password)
diff --git a/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py b/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py
@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+
+# command line arguments
+# --report_url=<string>
+# --report_file=<string, local file path>
+# --commit_hash=<string, full git commit hash>
+
+import argparse
+import mysql.connector
+import xml.etree.ElementTree as ET
+import sys
+import os
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description="ONNXRuntime test coverge report uploader for dashboard")
+    parser.add_argument("--report_url", help="URL to the Cobertura XML report")
+    parser.add_argument("--report_file", help="Path to the local cobertura XML report")
+    parser.add_argument("--commit_hash", help="Full Git commit hash")
+    return parser.parse_args()
+
+def parse_xml_report(report_file):
+    tree = ET.parse(report_file) # may throw exception
+    root = tree.getroot()
+    result = {}
+
+    result['coverage'] = float(root.get('line-rate'))
+    result['lines_covered'] = int(root.get('lines-covered'))
+    result['lines_valid'] = int(root.get('lines-valid'))
+    return result
+
+def write_to_db(coverage_data, args):
+    # connect to database
+
+    cnx = mysql.connector.connect(
+        user='ort@onnxruntimedashboard', 
+        password=os.environ.get('DASHBOARD_MYSQL_ORT_PASSWORD'), 
+        host='onnxruntimedashboard.mysql.database.azure.com', 
+        database='onnxruntime')
+
+    try:
+        cursor = cnx.cursor()
+
+        #delete old records
+        delete_query = ('DELETE FROM onnxruntime.test_coverage '
+            'WHERE UploadTime < DATE_SUB(Now(), INTERVAL 30 DAY);'
+        )
+
+        cursor.execute(delete_query)
+
+        #insert current record
+        insert_query = ('INSERT INTO onnxruntime.test_coverage '
+            '(UploadTime, CommitId, Coverage, LinesCovered, TotalLines, ReportURL) '
+            'VALUES (Now(), "%s", %f, %d, %d, "%s") '
+            'ON DUPLICATE KEY UPDATE '
+            'UploadTime=Now(), Coverage=%f, LinesCovered=%d, TotalLines=%d, ReportURL="%s";'
+        ) % (args.commit_hash, 
+             coverage_data['coverage'], 
+             coverage_data['lines_covered'],  
+             coverage_data['lines_valid'],
+             args.report_url,
+             coverage_data['coverage'], 
+             coverage_data['lines_covered'],  
+             coverage_data['lines_valid'],
+             args.report_url
+        )
+
+        cursor.execute(insert_query) 
+        cnx.commit()
+
+        # # Use below for debugging:
+        # cursor.execute('select * from onnxruntime.test_coverage')
+        # for r in cursor:
+        #     print(r)
+
+        cursor.close()
+        cnx.close()
+    except BaseException as e:
+        cnx.close()
+        raise e
+
+
+if __name__ == "__main__":
+    try:
+        args = parse_arguments()
+        coverage_data = parse_xml_report(args.report_file)
+        write_to_db(coverage_data, args)
+    except BaseException as e:
+        print(str(e))
+        sys.exit(1)
+
+
+
diff --git a/tools/ci_build/github/windows/run_OpenCppCoverage.ps1 b/tools/ci_build/github/windows/run_OpenCppCoverage.ps1
@@ -72,7 +72,9 @@ RunTest $mlas_test @() ("binary:" + (Join-Path $buildDir "onnxruntime_mlas_test.
 
 # Lotus unit tests
 # need to copy the tvm.dll, since it is not in the buildDir path
-Copy-Item -Path $BuildRoot\Debug\external\tvm\Debug\tvm.dll -Destination $buildDir
+if (Test-Path -Path $BuildRoot\Debug\external\tvm\Debug\tvm.dll -PathType Leaf) {
+    Copy-Item -Path $BuildRoot\Debug\external\tvm\Debug\tvm.dll -Destination $buildDir
+}
 
 $onnxruntime_test_all = Join-Path $buildDir "onnxruntime_test_all.exe"
 RunTest $onnxruntime_test_all @() ("cobertura:$outputXml","html:$outputDir") ("onnxruntime_shared_lib_test.cov","onnx_test_runner.cov","onnxruntime_mlas_test.cov")