Add _pyodide_core C extension module (pyodide#1090)

jmsmdy · Jan 11, 2021 · 3548683 · 3548683
1 parent e8e341f
commit 3548683
Show file tree

Hide file tree

Showing 8 changed files with 111 additions and 40 deletions.
diff --git a/src/core/jsimport.c b/src/core/jsimport.c
@@ -63,17 +63,17 @@ JsImport_init()
 {
   PyObject* module_dict = PyImport_GetModuleDict();
   if (module_dict == NULL) {
-    return 1;
+    return -1;
   }
 
   js_module = PyModule_Create(&JsModule);
   if (js_module == NULL) {
-    return 1;
+    return -1;
   }
 
   if (PyDict_SetItemString(module_dict, "js", js_module)) {
     Py_DECREF(js_module);
-    return 1;
+    return -1;
   }
 
   return 0;

diff --git a/src/core/jsproxy.c b/src/core/jsproxy.c
@@ -772,32 +772,49 @@ JsException_AsJs(PyObject* err)
   return hiwire_incref(js_error->js);
 }
 
+// Copied from Python 3.9
+// TODO: remove once we update to Python 3.9
+static int
+PyModule_AddType(PyObject* module, PyTypeObject* type)
+{
+  if (PyType_Ready(type) < 0) {
+    return -1;
+  }
+
+  const char* name = _PyType_Name(type);
+  assert(name != NULL);
+
+  Py_INCREF(type);
+  if (PyModule_AddObject(module, name, (PyObject*)type) < 0) {
+    Py_DECREF(type);
+    return -1;
+  }
+
+  return 0;
+}
+
 int
-JsProxy_init()
+JsProxy_init(PyObject* core_module)
 {
   bool success = false;
 
   PyObject* asyncio_module = NULL;
   PyObject* pyodide_module = NULL;
 
-  PyExc_BaseException_Type = (PyTypeObject*)PyExc_BaseException;
-  _Exc_JsException.tp_base = (PyTypeObject*)PyExc_Exception;
-
   asyncio_module = PyImport_ImportModule("asyncio");
   FAIL_IF_NULL(asyncio_module);
 
   asyncio_get_event_loop =
     _PyObject_GetAttrId(asyncio_module, &PyId_get_event_loop);
   FAIL_IF_NULL(asyncio_get_event_loop);
 
+  PyExc_BaseException_Type = (PyTypeObject*)PyExc_BaseException;
+  _Exc_JsException.tp_base = (PyTypeObject*)PyExc_Exception;
+
   // Add JsException to the pyodide module so people can catch it if they want.
-  pyodide_module = PyImport_ImportModule("pyodide");
-  FAIL_IF_NULL(pyodide_module);
-  FAIL_IF_MINUS_ONE(
-    PyObject_SetAttrString(pyodide_module, "JsException", Exc_JsException));
-  FAIL_IF_MINUS_ONE(PyType_Ready(&JsProxyType));
-  FAIL_IF_MINUS_ONE(PyType_Ready(&JsBoundMethodType));
-  FAIL_IF_MINUS_ONE(PyType_Ready(&_Exc_JsException));
+  FAIL_IF_MINUS_ONE(PyModule_AddType(core_module, &JsProxyType));
+  FAIL_IF_MINUS_ONE(PyModule_AddType(core_module, &JsBoundMethodType));
+  FAIL_IF_MINUS_ONE(PyModule_AddType(core_module, &_Exc_JsException));
 
   success = true;
 finally:

diff --git a/src/core/jsproxy.h b/src/core/jsproxy.h
@@ -54,6 +54,6 @@ JsException_AsJs(PyObject* x);
 
 /** Initialize global state for the JsProxy functionality. */
 int
-JsProxy_init();
+JsProxy_init(PyObject* core_module);
 
 #endif /* JSPROXY_H */
diff --git a/src/core/main.c b/src/core/main.c
@@ -20,8 +20,8 @@
     if (PyErr_Occurred()) {                                                    \
       printf("Error was triggered by Python exception:\n");                    \
       PyErr_Print();                                                           \
-      return 1;                                                                \
     }                                                                          \
+    return -1;                                                                 \
   } while (0)
 
 #define TRY_INIT(mod)                                                          \
@@ -31,46 +31,76 @@
     }                                                                          \
   } while (0)
 
+#define TRY_INIT_WITH_CORE_MODULE(mod)                                         \
+  do {                                                                         \
+    if (mod##_init(core_module)) {                                             \
+      FATAL_ERROR("Failed to initialize module %s.\n", #mod);                  \
+    }                                                                          \
+  } while (0)
+
+static struct PyModuleDef core_module_def = {
+  PyModuleDef_HEAD_INIT,
+  .m_name = "_pyodide_core",
+  .m_doc = "Pyodide C builtins",
+  .m_size = -1,
+};
+
 int
 main(int argc, char** argv)
 {
-  if (alignof(JsRef) != alignof(int)) {
-    FATAL_ERROR("JsRef doesn't have the same alignment as int.");
-  }
-  if (sizeof(JsRef) != sizeof(int)) {
-    FATAL_ERROR("JsRef doesn't have the same size as int.");
-  }
+  PyObject* sys = NULL;
+  PyObject* core_module = NULL;
   TRY_INIT(hiwire);
 
   setenv("PYTHONHOME", "/", 0);
-
-  Py_InitializeEx(0);
-
   // This doesn't seem to work anymore, but I'm keeping it for good measure
   // anyway The effective way to turn this off is below: setting
   // sys.dont_write_bytecode = True
   setenv("PYTHONDONTWRITEBYTECODE", "1", 0);
 
-  PyObject* sys = PyImport_ImportModule("sys");
+  Py_InitializeEx(0);
+
+  sys = PyImport_ImportModule("sys");
   if (sys == NULL) {
     FATAL_ERROR("Failed to import sys module.");
   }
 
   if (PyObject_SetAttrString(sys, "dont_write_bytecode", Py_True)) {
     FATAL_ERROR("Failed to set attribute on sys module.");
   }
-  Py_DECREF(sys);
+
+  if (alignof(JsRef) != alignof(int)) {
+    FATAL_ERROR("JsRef doesn't have the same alignment as int.");
+  }
+
+  if (sizeof(JsRef) != sizeof(int)) {
+    FATAL_ERROR("JsRef doesn't have the same size as int.");
+  }
+
+  core_module = PyModule_Create(&core_module_def);
+  if (core_module == NULL) {
+    FATAL_ERROR("Failed to create core module.");
+  }
 
   TRY_INIT(error_handling);
   TRY_INIT(js2python);
   TRY_INIT(JsImport);
-  TRY_INIT(JsProxy);
+  TRY_INIT_WITH_CORE_MODULE(JsProxy);
   TRY_INIT(pyproxy);
   TRY_INIT(python2js);
+
+  PyObject* module_dict = PyImport_GetModuleDict(); // borrowed
+  if (PyDict_SetItemString(module_dict, "_pyodide_core", core_module)) {
+    FATAL_ERROR("Failed to add '_pyodide_core' module to modules dict.");
+  }
+
+  // pyodide.py imported for these two.
+  // They should appear last so that core_module is ready.
   TRY_INIT(runpython);
 
+  Py_CLEAR(sys);
+  Py_CLEAR(core_module);
   printf("Python initialization complete\n");
-
   emscripten_exit_with_live_runtime();
   return 0;
 }
diff --git a/src/core/runpython.c b/src/core/runpython.c
@@ -69,6 +69,12 @@ runpython_init()
   JsRef pyodide_py_proxy = NULL;
   JsRef globals_proxy = NULL;
 
+  // I'm a bit confused about this deal with globals and builtins:
+  // 1. Why are we using __main__.__dict__ as globals? Shouldn't we make a fresh
+  // dictionary?
+  // 2. Why do we dump "builtins" directly into the dict? Normally we would
+  // leave the builtins where they belong as globals().__builtins__.
+
   // borrowed
   PyObject* builtins = PyImport_AddModule("builtins");
   FAIL_IF_NULL(builtins);

diff --git a/src/pyodide-py/pyodide/__init__.py b/src/pyodide-py/pyodide/__init__.py
@@ -1,4 +1,6 @@
-from ._base import open_url, eval_code, find_imports, as_nested_list, JsException
+from ._base import open_url, eval_code, find_imports, as_nested_list
+from ._core import JsException  # type: ignore
+
 
 __version__ = "0.16.1"
 

diff --git a/src/pyodide-py/pyodide/_base.py b/src/pyodide-py/pyodide/_base.py
@@ -12,16 +12,6 @@
 import tokenize
 
 
-class JsException(Exception):
-    """
-    A wrapper around a Javascript Error to allow the Error to be thrown in Python.
-    """
-
-    # This gets overwritten in jsproxy.c, it is just here for autodoc and humans
-    # reading this file.
-    pass
-
-
 def open_url(url: str) -> StringIO:
     """
     Fetches a given URL

diff --git a/src/pyodide-py/pyodide/_core.py b/src/pyodide-py/pyodide/_core.py
@@ -0,0 +1,26 @@
+# type: ignore
+import platform
+
+if platform.system() == "Emscripten":
+    from _pyodide_core import JsProxy, JsBoundMethod, JsException
+else:
+    # Can add shims here if we are so inclined.
+    class JsException(Exception):
+        """
+        A wrapper around a Javascript Error to allow the Error to be thrown in Python.
+        """
+
+        # Defined in jsproxy.c
+
+    class JsProxy:
+        """A proxy to make a Javascript object behave like a Python object"""
+
+        # Defined in jsproxy.c
+
+    class JsBoundMethod:
+        """A proxy to make it possible to call Javascript bound methods from Python."""
+
+        # Defined in jsproxy.c
+
+
+__all__ = [JsProxy, JsBoundMethod, JsException]