Assorted python wheel build improvements (grpc#25602)

* cleanup unnecessary deps from grpc_artifact_python_manylinux2014_aarch64 * cleanup in grpc_artifact_python dockerfiles * enable boringssl assembly optimizations when crosscompiling aarch64 linux wheels * pin manylinux2010 images to keep python27 build * avoid pip install --upgrade cython when not necessary
zkkxu · Mar 4, 2021 · 1dce57f · 1dce57f
1 parent cb8114d
commit 1dce57f
Show file tree

Hide file tree

Showing 8 changed files with 45 additions and 29 deletions.
diff --git a/setup.py b/setup.py
@@ -109,6 +109,14 @@
 BUILD_WITH_BORING_SSL_ASM = os.environ.get('GRPC_BUILD_WITH_BORING_SSL_ASM',
                                            True)
 
+# Export this environment variable to override the platform variant that will
+# be chosen for boringssl assembly optimizations. This option is useful when
+# crosscompiling and the host platform as obtained by distutils.utils.get_platform()
+# doesn't match the platform we are targetting.
+# Example value: "linux-aarch64"
+BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM = os.environ.get(
+    'GRPC_BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM', '')
+
 # Environment variable to determine whether or not the Cython extension should
 # *use* Cython or use the generated C files. Note that this requires the C files
 # to have been generated by building first *with* Cython support. Even if this
@@ -314,20 +322,22 @@ def check_linker_need_libatomic():
 
 asm_key = ''
 if BUILD_WITH_BORING_SSL_ASM and not BUILD_WITH_SYSTEM_OPENSSL:
+    boringssl_asm_platform = BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM if BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM else util.get_platform(
+    )
     LINUX_X86_64 = 'linux-x86_64'
     LINUX_ARM = 'linux-arm'
     LINUX_AARCH64 = 'linux-aarch64'
-    if LINUX_X86_64 == util.get_platform():
+    if LINUX_X86_64 == boringssl_asm_platform:
         asm_key = 'crypto_linux_x86_64'
-    elif LINUX_ARM == util.get_platform():
+    elif LINUX_ARM == boringssl_asm_platform:
         asm_key = 'crypto_linux_arm'
-    elif LINUX_AARCH64 == util.get_platform():
+    elif LINUX_AARCH64 == boringssl_asm_platform:
         asm_key = 'crypto_linux_aarch64'
-    elif "mac" in util.get_platform() and "x86_64" in util.get_platform():
+    elif "mac" in boringssl_asm_platform and "x86_64" in boringssl_asm_platform:
         asm_key = 'crypto_mac_x86_64'
     else:
         print("ASM Builds for BoringSSL currently not supported on:",
-              util.get_platform())
+              boringssl_asm_platform)
 if asm_key:
     asm_files = grpc_core_dependencies.ASM_SOURCE_FILES[asm_key]
 else:

diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2010_x64/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2010_x64/Dockerfile
@@ -13,13 +13,14 @@
 # limitations under the License.
 
 # Docker file for building gRPC manylinux Python artifacts.
-# Updated: 2020-10-08
 
-FROM quay.io/pypa/manylinux2010_x86_64
+# python2.7 was removed from the manylinux2010 image.
+# Use the latest version that still has python2.7
+# TODO(jtattermusch): Stop building python2.7 wheels.
+FROM quay.io/pypa/manylinux2010_x86_64:2021-02-06-3d322a5
 
-# Update the package manager
-RUN yum update -y
-RUN yum install -y curl-devel expat-devel gettext-devel linux-headers openssl-devel zlib-devel gcc
+# TODO(jtattermusch): revisit which of the deps are really required
+RUN yum update -y && yum install -y curl-devel expat-devel gettext-devel linux-headers openssl-devel zlib-devel gcc
 
 ###################################
 # Install Python build requirements

diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2010_x86/Dockerfile
@@ -13,13 +13,14 @@
 # limitations under the License.
 
 # Docker file for building gRPC manylinux Python artifacts.
-# Updated: 2020-10-08
 
-FROM quay.io/pypa/manylinux2010_i686
+# python2.7 was removed from the manylinux2010 image.
+# Use the latest version that still has python2.7
+# TODO(jtattermusch): Stop building python2.7 wheels.
+FROM quay.io/pypa/manylinux2010_i686:2021-02-06-3d322a5
 
-# Update the package manager
-RUN yum update -y
-RUN yum install -y curl-devel expat-devel gettext-devel linux-headers openssl-devel zlib-devel gcc
+# TODO(jtattermusch): revisit which of the deps are really required
+RUN yum update -y && yum install -y curl-devel expat-devel gettext-devel linux-headers openssl-devel zlib-devel gcc
 
 ###################################
 # Install Python build requirements

diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2014_aarch64/Dockerfile
@@ -19,9 +19,6 @@
 # before https://github.com/dockcross/dockcross/pull/449
 FROM dockcross/manylinux2014-aarch64:20200929-608e6ac
 
-# Update the package manager
-RUN yum update -y && yum install -y curl-devel expat-devel gettext-devel openssl-devel zlib-devel
-
 ###################################
 # Install Python build requirements
 RUN /opt/python/cp35-cp35m/bin/pip install --upgrade cython

diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2014_x64/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2014_x64/Dockerfile
@@ -17,9 +17,8 @@
 
 FROM quay.io/pypa/manylinux2014_x86_64
 
-# Update the package manager
-RUN yum update -y
-RUN yum install -y curl-devel expat-devel gettext-devel openssl-devel zlib-devel
+# TODO(jtattermusch): revisit which of the deps are really required
+RUN yum update -y && yum install -y curl-devel expat-devel gettext-devel openssl-devel zlib-devel
 
 ###################################
 # Install Python build requirements

diff --git a/tools/dockerfile/grpc_artifact_python_manylinux2014_x86/Dockerfile b/tools/dockerfile/grpc_artifact_python_manylinux2014_x86/Dockerfile
@@ -17,9 +17,8 @@
 
 FROM quay.io/pypa/manylinux2014_i686
 
-# Update the package manager
-RUN yum update -y
-RUN yum install -y curl-devel expat-devel gettext-devel openssl-devel zlib-devel
+# TODO(jtattermusch): revisit which of the deps are really required
+RUN yum update -y && yum install -y curl-devel expat-devel gettext-devel openssl-devel zlib-devel
 
 ###################################
 # Install Python build requirements

diff --git a/tools/run_tests/artifacts/artifact_targets.py b/tools/run_tests/artifacts/artifact_targets.py
@@ -144,6 +144,7 @@ def build_jobspec(self):
             environ['PYTHON'] = '/opt/python/{}/bin/python'.format(
                 self.py_version)
             environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version)
+            environ['GRPC_SKIP_PIP_CYTHON_UPGRADE'] = 'TRUE'
             if self.arch == 'aarch64':
                 environ['GRPC_SKIP_TWINE_CHECK'] = 'TRUE'
                 # when crosscompiling, we need to force statically linking libstdc++

diff --git a/tools/run_tests/artifacts/build_artifact_python.sh b/tools/run_tests/artifacts/build_artifact_python.sh
@@ -22,8 +22,17 @@ export PYTHON=${PYTHON:-python}
 export PIP=${PIP:-pip}
 export AUDITWHEEL=${AUDITWHEEL:-auditwheel}
 
-# Install Cython to avoid source wheel build failure.
-"${PIP}" install --upgrade cython
+if [ "$GRPC_SKIP_PIP_CYTHON_UPGRADE" == "" ]
+then
+  # Install Cython to avoid source wheel build failure.
+  # This only needs to be done when not running under docker (=on MacOS)
+  # since the docker images used for building python wheels
+  # already have a new-enough version of cython pre-installed.
+  # Any installation step is a potential source of breakages,
+  # so we are trying to perform as few download-and-install operations
+  # as possible.
+  "${PIP}" install --upgrade cython
+fi
 
 # Allow build_ext to build C/C++ files in parallel
 # by enabling a monkeypatch. It speeds up the build a lot.
@@ -46,9 +55,8 @@ then
   GRPC_PYTHON_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')"
   export GRPC_PYTHON_OVERRIDE_EXT_SUFFIX
 
-  # Set to empty string to disable the option (see https://github.com/grpc/grpc/issues/24498)
-  # TODO: enable ASM optimizations for crosscompiled wheels
-  export GRPC_BUILD_WITH_BORING_SSL_ASM=""
+  # since we're crosscompiling, we need to explicitly choose the right platform for boringssl assembly optimizations
+  export GRPC_BUILD_OVERRIDE_BORING_SSL_ASM_PLATFORM="linux-aarch64"
 fi
 
 # Build the source distribution first because MANIFEST.in cannot override