From 1936d141a7c35bcaa1262de8906fe7ba850b2ab4 Mon Sep 17 00:00:00 2001 From: Raymond Yang Date: Thu, 11 Apr 2019 22:06:18 -0700 Subject: [PATCH] Create nightly build for python packages (#817) * Enable nightly build * Update fetch file names * Fix * Update setup.py * Update run_dockerbuild.sh * Resolve comments --- setup.py | 19 +++++++++++ tools/ci_build/build.py | 33 +++++++++++++------ .../azure-pipelines-py-packaging.yml | 10 +++--- .../ci_build/github/linux/run_dockerbuild.sh | 7 ++++ 4 files changed, 54 insertions(+), 15 deletions(-) diff --git a/setup.py b/setup.py index f7e6d7e6cd4bf..1dea3f2945177 100644 --- a/setup.py +++ b/setup.py @@ -7,14 +7,29 @@ from os import path, getcwd import platform import sys +import datetime +nightly_build = False package_name = 'onnxruntime' if '--use_tensorrt' in sys.argv: package_name = 'onnxruntime-gpu-tensorrt' sys.argv.remove('--use_tensorrt') + if '--nightly_build' in sys.argv: + package_name = 'ort-trt-nightly' + nightly_build = True + sys.argv.remove('--nightly_build') elif '--use_cuda' in sys.argv: package_name = 'onnxruntime-gpu' sys.argv.remove('--use_cuda') + if '--nightly_build' in sys.argv: + package_name = 'ort-gpu-nightly' + nightly_build = True + sys.argv.remove('--nightly_build') + +if '--nightly_build' in sys.argv: + package_name = 'ort-nightly' + nightly_build = True + sys.argv.remove('--nightly_build') try: from wheel.bdist_wheel import bdist_wheel as _bdist_wheel @@ -52,9 +67,13 @@ def finalize_options(self): with open(README) as f: long_description = f.read() + version_number = '' with open('VERSION_NUMBER') as f: version_number = f.readline().strip() +if nightly_build: + date_suffix = str(datetime.datetime.now().date().strftime("%m%d")) + version_number = version_number + ".dev" + date_suffix # Setup setup( diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index c7c2a43665f6d..028a94f7b91b2 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -135,6 +135,7 @@ def parse_arguments(): parser.add_argument("--tensorrt_home", help="Path to TensorRT installation dir") parser.add_argument("--use_full_protobuf", action='store_true', help="Use the full protobuf library") parser.add_argument("--disable_contrib_ops", action='store_true', help="Disable contrib ops (reduces binary size)") + parser.add_argument("--skip_onnx_tests", action='store_true', help="Explicitly disable all onnx related tests") return parser.parse_args() def resolve_executable_path(command_or_path): @@ -556,17 +557,26 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_para else: run_subprocess([exe] + cmd, cwd=cwd) -def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_tensorrt): +def build_python_wheel(source_dir, build_dir, configs, use_cuda, use_tensorrt, nightly_build = False): for config in configs: cwd = get_config_build_dir(build_dir, config) + if is_windows(): cwd = os.path.join(cwd, config) - if use_tensorrt: - run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_tensorrt'], cwd=cwd) - elif use_cuda: - run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_cuda'], cwd=cwd) + if nightly_build: + if use_tensorrt: + run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_tensorrt', '--nightly_build'], cwd=cwd) + elif use_cuda: + run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_cuda', '--nightly_build'], cwd=cwd) + else: + run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--nightly_build'], cwd=cwd) else: - run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel'], cwd=cwd) + if use_tensorrt: + run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_tensorrt'], cwd=cwd) + elif use_cuda: + run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel', '--use_cuda'], cwd=cwd) + else: + run_subprocess([sys.executable, os.path.join(source_dir, 'setup.py'), 'bdist_wheel'], cwd=cwd) if is_ubuntu_1604(): run_subprocess([os.path.join(source_dir, 'rename_manylinux.sh')], cwd=cwd+'/dist') @@ -716,10 +726,12 @@ def main(): if (args.build): build_targets(cmake_path, build_dir, configs, args.parallel) - if (args.test): - run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, args.enable_pybind, args.use_tvm, args.use_tensorrt) + if args.test : + run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs, + args.enable_pybind if not args.skip_onnx_tests else False, + args.use_tvm, args.use_tensorrt) # run the onnx model tests if requested explicitly. - if (args.enable_onnx_tests): + if args.enable_onnx_tests and not args.skip_onnx_tests: # directory from ONNX submodule with ONNX test data onnx_test_data_dir = '/data/onnx' if is_windows() or not os.path.exists(onnx_test_data_dir): @@ -741,7 +753,8 @@ def main(): if args.build: if args.build_wheel: - build_python_wheel(source_dir, build_dir, configs, args.use_cuda, args.use_tensorrt) + nightly_build = bool(os.getenv('NIGHTLY_BUILD') == '1') + build_python_wheel(source_dir, build_dir, configs, args.use_cuda, args.use_tensorrt, nightly_build) if args.gen_doc: generate_documentation(source_dir, build_dir, configs) diff --git a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml index 48555b6446e1a..458511538f5b1 100644 --- a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml +++ b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml @@ -22,7 +22,7 @@ jobs: displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: SourceFolder: '$(Build.BinariesDirectory)' - Contents: 'Release/dist/onnxruntime-*-manylinux1_x86_64.whl' + Contents: 'Release/dist/*-manylinux1_x86_64.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - task: PublishBuildArtifacts@1 @@ -59,7 +59,7 @@ jobs: displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: SourceFolder: '$(Build.BinariesDirectory)' - Contents: 'Release/dist/onnxruntime*-manylinux1_x86_64.whl' + Contents: 'Release/dist/*-manylinux1_x86_64.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - task: PublishBuildArtifacts@1 @@ -104,7 +104,7 @@ jobs: displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: SourceFolder: '$(buildDirectory)' - Contents: '**\dist\onnxruntime-*.whl' + Contents: '**\dist\*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - task: PublishBuildArtifacts@1 @@ -171,7 +171,7 @@ jobs: displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: SourceFolder: '$(buildDirectory)' - Contents: '**\dist\onnxruntime_gpu-*.whl' + Contents: '**\dist\*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - task: PublishBuildArtifacts@1 @@ -221,7 +221,7 @@ jobs: displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: SourceFolder: '$(Build.SourcesDirectory)' - Contents: '**/dist/onnxruntime-*.whl' + Contents: '**/dist/*.whl' TargetFolder: '$(Build.ArtifactStagingDirectory)' - task: PublishBuildArtifacts@1 diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh index 629eeb78a266b..1830858091093 100755 --- a/tools/ci_build/github/linux/run_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_dockerbuild.sh @@ -51,6 +51,11 @@ fi set +e mkdir -p ~/.cache/onnxruntime mkdir -p ~/.onnx + +if [ -z "$NIGHTLY_BUILD" ]; then +set NIGHTLY_BUILD=0 +fi + if [ $BUILD_DEVICE = "cpu" ]; then docker rm -f "onnxruntime-$BUILD_DEVICE" || true docker run -h $HOSTNAME \ @@ -59,6 +64,7 @@ if [ $BUILD_DEVICE = "cpu" ]; then --volume "$BUILD_DIR:/build" \ --volume "$HOME/.cache/onnxruntime:/home/onnxruntimedev/.cache/onnxruntime" \ --volume "$HOME/.onnx:/home/onnxruntimedev/.onnx" \ + -e NIGHTLY_BUILD \ "onnxruntime-$IMAGE" \ /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \ -d $BUILD_DEVICE -x "$BUILD_EXTR_PAR" & @@ -71,6 +77,7 @@ else --volume "$BUILD_DIR:/build" \ --volume "$HOME/.cache/onnxruntime:/home/onnxruntimedev/.cache/onnxruntime" \ --volume "$HOME/.onnx:/home/onnxruntimedev/.onnx" \ + -e NIGHTLY_BUILD \ "onnxruntime-$IMAGE" \ /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \ -d $BUILD_DEVICE -x "$BUILD_EXTR_PAR" &