Get rid of RuntimeWarning when using process pool

Restructure process_pool implementation code in a way that resolves `RuntimeWarning: 'petastorm.workers_pool.exec_in_new_process' found in sys.modules after import of package 'petastorm.workers_pool', but prior to execution of 'petastorm.workers_pool.exec_in_new_process'; this may result in unpredictable behaviou when using process pool` warning. In this diff we break main entry point from `exec_in_new_process.py` file into two a separate `exec_in_new_process_entrypoint.py`. Resolves #585
uber · selitvin · Jan 8, 2022 · Jan 7, 2022 · Jan 7, 2022 · Jan 7, 2022
commit 345aabb9d2570a097b65e4585535d8331e2a3263
diff --git a/docs/release-notes.rst b/docs/release-notes.rst
@@ -6,6 +6,8 @@ Release notes
 
 Release 0.11.4 (unreleased)
 ===========================
+- `PR 732 <https://github.com/uber/petastorm/pull/732>`_ (resolves `PR 585 <https://github.com/uber/petastorm/issues/585>`_): Restructure process_pool implementation code in a way that resolves ``RuntimeWarning: 'petastorm.workers_pool.exec_in_new_process' found in sys.modules after
+    import of package 'petastorm.workers_pool', but prior to execution of 'petastorm.workers_pool.exec_in_new_process'; this may result in unpredictable behaviou when using process pool`` warning.
 
 
 Release 0.11.3

diff --git a/petastorm/workers_pool/exec_in_new_process.py b/petastorm/workers_pool/exec_in_new_process.py
@@ -11,7 +11,6 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-
 import logging
 import os
 import subprocess
@@ -42,28 +41,7 @@ def exec_in_new_process(func, *args, **kargs):
     # Popen this script (__main__) below will be an entry point
     process = subprocess.Popen(args=[sys.executable,
                                      '-m',
-                                     bootstrap_package_name,
+                                     bootstrap_package_name + "_entrypoint",
                                      new_process_runnable_file],
                                executable=sys.executable)
     return process
-
-
-if __name__ == '__main__':
-    # An entry point to the newely executed process.
-    # Will unpickle function handle and arguments and call the function.
-    try:
-        logging.basicConfig()
-        if len(sys.argv) != 2:
-            raise RuntimeError('Expected a single command line argument')
-        new_process_runnable_file = sys.argv[1]
-
-        with open(new_process_runnable_file, 'rb') as f:
-            func, args, kargs = dill.load(f)  # pylint: disable=unpacking-non-sequence
-
-        # Don't need the pickle file with the runable. Cleanup.
-        os.remove(new_process_runnable_file)
-
-        func(*args, **kargs)
-    except Exception as e:
-        logger.error('Unhandled exception in the function launched by exec_in_new_process: %s', str(e))
-        raise
diff --git a/petastorm/workers_pool/exec_in_new_process_entrypoint.py b/petastorm/workers_pool/exec_in_new_process_entrypoint.py
@@ -0,0 +1,40 @@
+#  Copyright (c) 2017-2018 Uber Technologies, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import logging
+import os
+import sys
+
+import dill
+
+logger = logging.getLogger(__name__)
+
+if __name__ == '__main__':
+    # An entry point to the newely executed process.
+    # Will unpickle function handle and arguments and call the function.
+    try:
+        logging.basicConfig()
+        if len(sys.argv) != 2:
+            raise RuntimeError('Expected a single command line argument')
+        new_process_runnable_file = sys.argv[1]
+
+        with open(new_process_runnable_file, 'rb') as f:
+            func, args, kargs = dill.load(f)  # pylint: disable=unpacking-non-sequence
+
+        # Don't need the pickle file with the runable. Cleanup.
+        os.remove(new_process_runnable_file)
+
+        func(*args, **kargs)
+    except Exception as e:
+        logger.error('Unhandled exception in the function launched by exec_in_new_process: %s', str(e))
+        raise