[Python Misc] Use template to generate _parallel_compile_patch.py (gr…

…pc#34847) `_parallel_compile_patch.py` exist in multiple places, we should use template system to generate the content.  Closes grpc#34847 COPYBARA_INTEGRATE_REVIEW=grpc#34847 from XuanWang-Amos:use_template_for__parallel_compile_patch 15b98ea PiperOrigin-RevId: 590262888
maniksurtani · Dec 12, 2023 · 4bf0109 · 4bf0109
1 parent 7f2ecdb
commit 4bf0109
Show file tree

Hide file tree

Showing 5 changed files with 130 additions and 3 deletions.
diff --git a/src/python/grpcio/_parallel_compile_patch.py b/src/python/grpcio/_parallel_compile_patch.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The gRPC Authors
+# Copyright 2023 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,6 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+#
+# This file has been automatically generated from a template file.
+# Please make modifications to
+# `$REPO_ROOT/templates/src/python/grpcio/_parallel_compile_patch.py.template`
+# instead. This file can be regenerated from the template by running
+# `tools/buildgen/generate_projects.sh`.
+
 """Patches the compile() to allow enable parallel compilation of C/C++.
 
 build_ext has lots of C/C++ files and normally them one by one.

diff --git a/templates/src/python/_parallel_compile_patch.py.include b/templates/src/python/_parallel_compile_patch.py.include
@@ -0,0 +1,65 @@
+"""Patches the compile() to allow enable parallel compilation of C/C++.
+
+build_ext has lots of C/C++ files and normally them one by one.
+Enabling parallel build helps a lot.
+"""
+
+import os
+
+try:
+    BUILD_EXT_COMPILER_JOBS = int(
+        os.environ["GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS"]
+    )
+except KeyError:
+    import multiprocessing
+
+    BUILD_EXT_COMPILER_JOBS = multiprocessing.cpu_count()
+except ValueError:
+    BUILD_EXT_COMPILER_JOBS = 1
+
+
+# monkey-patch for parallel compilation
+def _parallel_compile(
+    self,
+    sources,
+    output_dir=None,
+    macros=None,
+    include_dirs=None,
+    debug=0,
+    extra_preargs=None,
+    extra_postargs=None,
+    depends=None,
+):
+    # setup the same way as distutils.ccompiler.CCompiler
+    # https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564
+    macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
+        str(output_dir), macros, include_dirs, sources, depends, extra_postargs
+    )
+    cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)
+
+    def _compile_single_file(obj):
+        try:
+            src, ext = build[obj]
+        except KeyError:
+            return
+        self._compile(obj, src, ext, cc_args, extra_postargs, pp_opts)
+
+    # run compilation of individual files in parallel
+    import multiprocessing.pool
+
+    multiprocessing.pool.ThreadPool(BUILD_EXT_COMPILER_JOBS).map(
+        _compile_single_file, objects
+    )
+    return objects
+
+
+def monkeypatch_compile_maybe():
+    """
+    Monkeypatching is dumb, but the build speed gain is worth it.
+    After python 3.12, we won't find distutils if SETUPTOOLS_USE_DISTUTILS=stdlib.
+    """
+    use_distutils = os.environ.get("SETUPTOOLS_USE_DISTUTILS", "")
+    if BUILD_EXT_COMPILER_JOBS > 1 and use_distutils != "stdlib":
+        import distutils.ccompiler  # pylint: disable=wrong-import-position
+
+        distutils.ccompiler.CCompiler.compile = _parallel_compile
diff --git a/templates/src/python/grpcio/_parallel_compile_patch.py.template b/templates/src/python/grpcio/_parallel_compile_patch.py.template
@@ -0,0 +1,23 @@
+%YAML 1.2
+--- |
+  # Copyright 2023 The gRPC Authors
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+  #
+  # This file has been automatically generated from a template file.
+  # Please make modifications to
+  # `$REPO_ROOT/templates/src/python/grpcio/_parallel_compile_patch.py.template`
+  # instead. This file can be regenerated from the template by running
+  # `tools/buildgen/generate_projects.sh`.
+
+  <%include file="../_parallel_compile_patch.py.include" />
diff --git a/templates/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py.template b/templates/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py.template
@@ -0,0 +1,23 @@
+%YAML 1.2
+--- |
+  # Copyright 2023 The gRPC Authors
+  #
+  # Licensed under the Apache License, Version 2.0 (the "License");
+  # you may not use this file except in compliance with the License.
+  # You may obtain a copy of the License at
+  #
+  #     http://www.apache.org/licenses/LICENSE-2.0
+  #
+  # Unless required by applicable law or agreed to in writing, software
+  # distributed under the License is distributed on an "AS IS" BASIS,
+  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  # See the License for the specific language governing permissions and
+  # limitations under the License.
+  #
+  # This file has been automatically generated from a template file.
+  # Please make modifications to
+  # `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py.template`
+  # instead. This file can be regenerated from the template by running
+  # `tools/buildgen/generate_projects.sh`.
+
+  <%include file="../../../../src/python/_parallel_compile_patch.py.include" />
diff --git a/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py b/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py
@@ -1,4 +1,4 @@
-# Copyright 2018 The gRPC Authors
+# Copyright 2023 The gRPC Authors
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -11,6 +11,13 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+#
+# This file has been automatically generated from a template file.
+# Please make modifications to
+# `$REPO_ROOT/templates/tools/distrib/python/grpcio_tools/_parallel_compile_patch.py.template`
+# instead. This file can be regenerated from the template by running
+# `tools/buildgen/generate_projects.sh`.
+
 """Patches the compile() to allow enable parallel compilation of C/C++.
 
 build_ext has lots of C/C++ files and normally them one by one.
@@ -27,6 +34,8 @@
     import multiprocessing
 
     BUILD_EXT_COMPILER_JOBS = multiprocessing.cpu_count()
+except ValueError:
+    BUILD_EXT_COMPILER_JOBS = 1
 
 
 # monkey-patch for parallel compilation
@@ -44,7 +53,7 @@ def _parallel_compile(
     # setup the same way as distutils.ccompiler.CCompiler
     # https://github.com/python/cpython/blob/31368a4f0e531c19affe2a1becd25fc316bc7501/Lib/distutils/ccompiler.py#L564
     macros, objects, extra_postargs, pp_opts, build = self._setup_compile(
-        output_dir, macros, include_dirs, sources, depends, extra_postargs
+        str(output_dir), macros, include_dirs, sources, depends, extra_postargs
     )
     cc_args = self._get_cc_args(pp_opts, debug, extra_preargs)