Standalone TVM Executor Provider (microsoft#10019)

* squashed commit for standalone tvm execution provider * critical fix for correct python build with stvm ep * get tuning log file from ep options. It has priority over AUTOTVM_TUNING_LOG * updates and fixes * update parsing of stvm provider options * add support of external data for onnx model * add conditional dump of subgraphs * remove unused code * get input tensor shapes through provider options. get output shapes for fixed input ones by TVM API * support AUTO_TVM tuning log file inside ORT. Selector for Ansor and Auto_TVM is provider option (tuning_type) * add fp16 * add functionality of conversion of model layout to NHWC if need. Necessary parameter was added to STVM provider options * fix license text in header. fix log format * small fixes * fix issues from flake8 * remove model proto construction from GetCapability * reserve memory for vector of DLTensors * add simple tutorial for STVM EP * STVM docs * jroesch/tvm -> apache/tvm * remove dead code, unneccessary logs and comments * fix in readme * improve tutorial notebook * tvm update * update STVM_EP.md * fix default value * update STVM_EP.md * some TODOs for the future development * shorten long lines * add hyperlink to STVM_EP.md * fix Linux CI error * fix error in csharp test Co-authored-by: Jared Roesch <jroesch@octoml.ai> Co-authored-by: Valery Chernov <valery.chernov@deelvin.com> Co-authored-by: KJlaccHoeUM9l <wotpricol@mail.ru>
garymm · Dec 16, 2021 · b327e89 · b327e89
1 parent 16274be
commit b327e89
Show file tree

Hide file tree

Showing 48 changed files with 2,251 additions and 7 deletions.
diff --git a/.gitignore b/.gitignore
@@ -15,6 +15,7 @@ gen
 *~
 .vs
 .DS_Store
+*.DS_Store
 TestResults/
 .idea/
 onnxruntime.egg-info

diff --git a/.gitmodules b/.gitmodules
@@ -79,3 +79,6 @@
 [submodule "cmake/external/onnx-tensorrt"]
 	path = cmake/external/onnx-tensorrt
 	url = https://github.com/onnx/onnx-tensorrt.git
+[submodule "cmake/external/tvm_update"]
+	path = cmake/external/tvm_update
+	url = https://github.com/apache/tvm
diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
@@ -59,8 +59,8 @@ option(onnxruntime_BUILD_OBJC "Build Objective-C library" OFF)
 option(onnxruntime_USE_PREINSTALLED_EIGEN "Use pre-installed EIGEN. Need to provide eigen_SOURCE_PATH if turn this on." OFF)
 option(onnxruntime_BUILD_BENCHMARKS "Build ONNXRuntime micro-benchmarks" OFF)
 
-option(onnxruntime_USE_TVM "Build tvm for code-gen" OFF)
-option(onnxruntime_USE_LLVM "Build tvm with LLVM" OFF)
+option(onnxruntime_USE_TVM "Build TVM for code generation." OFF)
+option(onnxruntime_USE_LLVM "Build TVM with LLVM" OFF)
 
 option(onnxruntime_BUILD_FOR_NATIVE_MACHINE "Enable this option for turning on optimization specific to this machine" OFF)
 option(onnxruntime_USE_AVX "Use AVX instructions" OFF)
@@ -103,6 +103,7 @@ option(onnxruntime_USE_MIMALLOC "Override new/delete and arena allocator with mi
 #It's default OFF because it's experimental now.
 option(onnxruntime_PREFER_SYSTEM_LIB "Experimental: Build with the preinstalled libraries in your system" OFF)
 option(onnxruntime_USE_ROCM "Build with AMD GPU support" OFF)
+option(onnxruntime_USE_STVM "Build with standalone TVM support" OFF)
 
 # Options related to reducing the binary size produced by the build
 option(onnxruntime_DISABLE_CONTRIB_OPS "Disable contrib ops" OFF)
@@ -1088,6 +1089,11 @@ if (onnxruntime_USE_NUPHAR)
     list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_NUPHAR=1)
     list(APPEND ONNXRUNTIME_PROVIDER_NAMES nuphar)
 endif()
+if (onnxruntime_USE_STVM)
+    list(APPEND ORT_PROVIDER_FLAGS  -DUSE_STVM=1)
+    list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_STVM=1)
+    list(APPEND ONNXRUNTIME_PROVIDER_NAMES stvm)
+endif()
 if (onnxruntime_USE_WINML)
   list(APPEND ORT_PROVIDER_FLAGS  -DUSE_WINML=1)
   list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_WINML=1)

diff --git a/cmake/external/tvm_update b/cmake/external/tvm_update
diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake
@@ -167,6 +167,7 @@ set(onnxruntime_INTERNAL_LIBRARIES
   ${PROVIDERS_MIGRAPHX}
   ${PROVIDERS_NNAPI}
   ${PROVIDERS_NUPHAR}
+  ${PROVIDERS_STVM}
   ${PROVIDERS_RKNPU}
   ${PROVIDERS_ROCM}
   ${PROVIDERS_VITISAI}

diff --git a/cmake/onnxruntime_csharp.cmake b/cmake/onnxruntime_csharp.cmake
@@ -34,6 +34,10 @@ if (onnxruntime_USE_NUPHAR)
   STRING(APPEND CSHARP_PREPROCESSOR_DEFINES "USE_NUPHAR;")
 endif()
 
+if (onnxruntime_USE_STVM)
+  STRING(APPEND CSHARP_PREPROCESSOR_DEFINES "USE_STVM,")
+endif()
+
 if (onnxruntime_USE_OPENVINO)
   STRING(APPEND CSHARP_PREPROCESSOR_DEFINES "USE_OPENVINO;")
 endif()

diff --git a/cmake/onnxruntime_java.cmake b/cmake/onnxruntime_java.cmake
@@ -62,7 +62,6 @@ file(GLOB onnxruntime4j_native_src
 onnxruntime_add_shared_library_module(onnxruntime4j_jni ${onnxruntime4j_native_src})
 set_property(TARGET onnxruntime4j_jni PROPERTY CXX_STANDARD 11)
 
-
 # depend on java sources. if they change, the JNI should recompile
 add_dependencies(onnxruntime4j_jni onnxruntime4j)
 onnxruntime_add_include_to_target(onnxruntime4j_jni onnxruntime_session)

diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake
@@ -97,6 +97,10 @@ endif()
 if(onnxruntime_USE_ROCM)
   set(PROVIDERS_ROCM onnxruntime_providers_rocm)
 endif()
+if (onnxruntime_USE_STVM)
+  set(PROVIDERS_STVM onnxruntime_providers_stvm)
+endif()
+
 
 source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_common_srcs} ${onnxruntime_providers_srcs})
 
@@ -1168,3 +1172,46 @@ if (onnxruntime_USE_ROCM)
           RUNTIME  DESTINATION ${CMAKE_INSTALL_BINDIR})
 
 endif()
+
+if (onnxruntime_USE_STVM)
+  add_definitions(-DUSE_STVM=1)
+
+  file (GLOB_RECURSE onnxruntime_providers_stvm_cc_srcs CONFIGURE_DEPENDS
+    "${ONNXRUNTIME_ROOT}/core/providers/stvm/*.h"
+    "${ONNXRUNTIME_ROOT}/core/providers/stvm/*.cc"
+    )
+  source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_stvm_cc_srcs})
+  onnxruntime_add_static_library(onnxruntime_providers_stvm ${onnxruntime_providers_stvm_cc_srcs})
+
+  if ( CMAKE_COMPILER_IS_GNUCC )
+    target_compile_options(onnxruntime_providers_stvm PRIVATE -Wno-unused-parameter -Wno-missing-field-initializers)
+  endif()
+
+  target_include_directories(onnxruntime_providers_stvm PRIVATE
+          ${onnxruntime_STVM_HOME}/include
+          ${onnxruntime_STVM_HOME}/3rdparty/dlpack/include
+          ${onnxruntime_STVM_HOME}/3rdparty/dmlc-core/include
+          ${PYTHON_INLCUDE_DIRS})
+  onnxruntime_add_include_to_target(onnxruntime_providers_stvm onnxruntime_common onnx)
+
+  add_dependencies(onnxruntime_providers_stvm ${onnxruntime_EXTERNAL_DEPENDENCIES})
+
+  target_link_libraries(onnxruntime_providers_stvm PRIVATE
+      onnx
+      onnxruntime_common
+      onnxruntime_framework
+  )
+  if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+    target_link_libraries(onnxruntime_providers_stvm PRIVATE ${onnxruntime_STVM_HOME}/build/libtvm.dylib)
+  else()
+    target_link_libraries(onnxruntime_providers_stvm PRIVATE ${onnxruntime_STVM_HOME}/build/libtvm.so)
+  endif()
+
+  set_target_properties(onnxruntime_providers_stvm PROPERTIES FOLDER "ONNXRuntime")
+  set_target_properties(onnxruntime_providers_stvm PROPERTIES LINKER_LANGUAGE CXX)
+
+  target_compile_options(onnxruntime_providers_stvm PRIVATE -Wno-error=type-limits)
+  target_compile_definitions(onnxruntime_providers_stvm PUBLIC DMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>)
+
+  install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/stvm  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
+endif()
diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake
@@ -138,6 +138,7 @@ target_link_libraries(onnxruntime_pybind11_state PRIVATE
     ${onnxruntime_libs}
     ${PROVIDERS_MIGRAPHX}
     ${PROVIDERS_NUPHAR}
+    ${PROVIDERS_STVM}
     ${PROVIDERS_VITISAI}
     ${PROVIDERS_NNAPI}
     ${PROVIDERS_COREML}
@@ -654,6 +655,23 @@ if (onnxruntime_USE_NUPHAR)
   )
 endif()
 
+if (onnxruntime_USE_STVM)
+  file(GLOB onnxruntime_python_providers_stvm_srcs CONFIGURE_DEPENDS
+    "${ONNXRUNTIME_ROOT}/python/providers/stvm/*.py"
+  )
+  add_custom_command(
+    TARGET onnxruntime_pybind11_state POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/providers
+    COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/providers/stvm
+    COMMAND ${CMAKE_COMMAND} -E copy
+        ${onnxruntime_python_providers_stvm_srcs}
+        $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/providers/stvm
+    COMMAND ${CMAKE_COMMAND} -E copy
+        $<TARGET_FILE:onnxruntime_providers_stvm>
+        $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/capi/
+    )
+endif()
+
 if (onnxruntime_USE_DML)
   add_custom_command(
     TARGET onnxruntime_pybind11_state POST_BUILD

diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake
@@ -483,6 +483,7 @@ set(ONNXRUNTIME_TEST_LIBS
     ${PROVIDERS_ACL}
     ${PROVIDERS_ARMNN}
     ${PROVIDERS_COREML}
+    # ${PROVIDERS_STVM}
     onnxruntime_optimizer
     onnxruntime_providers
     onnxruntime_util
@@ -537,6 +538,14 @@ if(onnxruntime_USE_COREML)
   endif()
 endif()
 
+if (onnxruntime_USE_STVM)
+  file (GLOB_RECURSE onnxruntime_test_stvm_src CONFIGURE_DEPENDS
+    "${ONNXRUNTIME_ROOT}/test/stvm/*.h"
+    "${ONNXRUNTIME_ROOT}/test/stvm/*.cc"
+  )
+
+  list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_stvm)
+endif()
 
 if(WIN32)
   if (onnxruntime_USE_TVM)
@@ -626,6 +635,11 @@ endif()
 if (onnxruntime_USE_TVM)
   list(APPEND all_tests ${onnxruntime_test_tvm_src})
 endif()
+
+if (onnxruntime_USE_STVM)
+    list(APPEND all_tests ${onnxruntime_test_stvm_src})
+endif()
+
 if (onnxruntime_USE_OPENVINO)
   list(APPEND all_tests ${onnxruntime_test_openvino_src})
 endif()
@@ -1202,4 +1216,12 @@ if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD
   endif()
 endif()
 
+if (onnxruntime_USE_STVM)
+  # find_library(STVM_LIBS NAMES libtvm.so PATHS ${onnxruntime_STVM_HOME}/lib)
+  # link_directories(onnxruntime_test_all ${STVM_LIBS})
+  find_library(PYTHON_LIBS NAMES libpython PATHS /usr/local/lib)
+  #target_link_libraries(onnxruntime_test_all PRIVATE ${PYTHON_LIBRARIES} -lutil)
+  # set(CMAKE_SHARED_LINKER_FLAGS "-Wl,-rpath,${STVM_LIBS}")
+endif()
+
 include(onnxruntime_fuzz_test.cmake)
diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs
@@ -792,6 +792,9 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca
 
         [DllImport(NativeLib.DllName, CharSet = CharSet.Ansi)]
         public static extern IntPtr /*(OrtStatus*)*/ OrtSessionOptionsAppendExecutionProvider_Nuphar(IntPtr /*(OrtSessionOptions*) */ options, int allow_unaligned_buffers, IntPtr /*(char char*)*/ settings);
+
+        [DllImport(NativeLib.DllName, CharSet = CharSet.Ansi)]
+        public static extern IntPtr /*(OrtStatus*)*/ OrtSessionOptionsAppendExecutionProvider_Stvm(IntPtr /*(OrtSessionOptions*) */ options, IntPtr /*(char char*)*/ settings);
 #endif
         /// <summary>
         /// Append a TensorRT EP instance (configured based on given provider options) to the native OrtSessionOptions instance

diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs
@@ -127,6 +127,20 @@ public static SessionOptions MakeSessionOptionWithNupharProvider(String settings
             return options;
         }
 
+        /// <summary>
+        /// A helper method to construct a SessionOptions object for Stvm execution.
+        /// Use only if you have the onnxruntime package specific to this Execution Provider.
+        /// </summary>
+        /// <param name="settings">settings string, comprises of comma separated key:value pairs. default is empty</param>
+        /// <returns>A SessionsOptions() object configured for execution with Stvm</returns>
+        public static SessionOptions MakeSessionOptionWithStvmProvider(String settings = "")
+        {
+            SessionOptions options = new SessionOptions();
+            options.AppendExecutionProvider_Stvm(settings);
+
+            return options;
+        }
+
         /// <summary>
         /// A helper method to construct a SessionOptions object for ROCM execution.
         /// Use only if ROCM is installed and you have the onnxruntime package specific to this Execution Provider.
@@ -304,6 +318,23 @@ public void AppendExecutionProvider_Nuphar(string settings = "")
             {
                 NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Nuphar(handle, 1, pinnedSettingsName.Pointer));
             }
+#endif
+        }
+
+        /// <summary>
+        /// Use only if you have the onnxruntime package specific to this Execution Provider.
+        /// </summary>
+        /// <param name="settings">string with Stvm specific settings</param>
+        public void AppendExecutionProvider_Stvm(string settings = "")
+        {
+#if __MOBILE__
+            throw new NotSupportedException("The Stvm Execution Provider is not supported in this build");
+#else
+            var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);
+            using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
+            {
+                NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Stvm(handle, pinnedSettingsName.Pointer));
+            }
 #endif
         }
 #endregion //ExecutionProviderAppends

diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs
@@ -118,6 +118,10 @@ public void TestSessionOptions()
                 opt.AppendExecutionProvider_Nnapi(0);
 #endif
 
+#if USE_STVM
+                opt.AppendExecutionProvider_Stvm("Vulkan -device=amd_apu");
+#endif                
+
 #if USE_NUPHAR
                 opt.AppendExecutionProvider_Nuphar();
 #endif