Flexible jsimport (pyodide#1146)

This allows users to bind arbitrary JS objects as python modules. This is used to implement the `js` module. Fixes pyodide#960
jmsmdy · Jan 17, 2021 · 7374be5 · 7374be5
1 parent a961851
commit 7374be5
Show file tree

Hide file tree

Showing 13 changed files with 335 additions and 87 deletions.
diff --git a/Makefile b/Makefile
@@ -55,7 +55,7 @@ all: check \
 	echo -e "\nSUCCESS!"
 
 
-build/pyodide.asm.js: src/core/main.o src/core/jsimport.o \
+build/pyodide.asm.js: src/core/main.o  \
 	        src/core/jsproxy.o src/core/js2python.o \
 		src/core/error_handling.o \
 		src/core/pyproxy.o \

diff --git a/docs/api_reference.md b/docs/api_reference.md
@@ -16,6 +16,8 @@ Backward compatibility of the API is not guaranteed at this point.
    pyodide.find_imports
    pyodide.open_url
    pyodide.JsException
+   pyodide.register_js_module
+   pyodide.unregister_js_module
 ```
 
 
@@ -28,10 +30,12 @@ Backward compatibility of the API is not guaranteed at this point.
 | **{ref}`js_api_pyodide_globals`**        | An alias to the global Python namespace                        |
 | **{ref}`pyodide.loadPackage(names, ...) <js_api_pyodide_loadPackage>`**    | Load a package or a list of packages over the network          |
 | **{ref}`js_api_pyodide_loadedPackages`** | `Object` with loaded packages.                                 |
+| **{ref}`pyodide.registerJsPackage(name, js_object) <js_api_pyodide_registerJsModule>`**   | Registers a javascript object as a Python module.        |
+| **{ref}`pyodide.unregisterJsPackage(name) <js_api_pyodide_unregisterJsModule>`** | Unregisters a module previously registered with `js_api_pyodide_registerJsPackage`.        |
 | **{ref}`js_api_pyodide_pyimport`**       | Access a Python object in the global namespace from Javascript |
 | **{ref}`js_api_pyodide_runPython`**      | Runs Python code from Javascript.                              |
 | **{ref}`pyodide.runPythonAsync(code, ...) <js_api_pyodide_runPythonAsync>`** | Runs Python code with automatic preloading of imports.         |
-| **{ref}`js_api_pyodide_version`**        | Returns the pyodide version.                                   |
+| **{ref}`js_api_pyodide_version`**        | The pyodide version string.                                    |
 
 
 ```{eval-rst}

diff --git a/docs/changelog.md b/docs/changelog.md
@@ -40,6 +40,8 @@
   [#1083](https://github.com/iodide-project/pyodide/pull/1083)
 - An InteractiveConsole to ease the integration of Pyodide REPL in
   webpages (used in console.html) [#1125](https://github.com/iodide-project/pyodide/pull/1125)
+- Flexible jsimports: it now possible to add custom Python "packages" backed by Javascript code, like the js package.
+  The js package is now implemented using this system. [#1146](https://github.com/iodide-project/pyodide/pull/1146)
 
 ### Fixed
 - getattr and dir on JsProxy now report consistent results and include all names defined on the Python dictionary backing JsProxy. [#1017](https://github.com/iodide-project/pyodide/pull/1017)
@@ -49,6 +51,9 @@
 - JsBoundMethod is now a subclass of JsProxy, which fixes nested attribute access and various other strange bugs.
   [#1124](https://github.com/iodide-project/pyodide/pull/1124)
 - In console.html: sync behavior, full stdout/stderr support, clean namespace and bigger font [#1125](https://github.com/iodide-project/pyodide/pull/1125)
+- Javascript functions imported like `from js import fetch` no longer trigger "invalid invocation" errors (issue [#461](https://github.com/iodide-project/pyodide/issues/461)) and `js.fetch("some_url")` also works now (issue [#768](https://github.com/iodide-project/pyodide/issues/461)).
+[#1126](https://github.com/iodide-project/pyodide/pull/1126)
+- Javascript bound method calls now work correctly with keyword arguments. [#1138](https://github.com/iodide-project/pyodide/pull/1138)
 
 ## Version 0.16.1
 *December 25, 2020*

diff --git a/docs/faq.md b/docs/faq.md
@@ -63,7 +63,7 @@ If you want to change which packages `loadPackagesFromImports` loads, you can
 monkey patch `pyodide-py.find_imports` which takes `code` as an argument
 and returns a list of packages imported.
 
-# How can I execute code in a custom namespace?
+## How can I execute code in a custom namespace?
 The second argument to `eval_code` is a namespace to execute the code in.
 The namespace is a python dictionary. So you can use:
 ```javascript
@@ -116,3 +116,35 @@ if "PYODIDE" in os.environ:
 ```
 We used to use the environment variable `PYODIDE_BASE_URL` for this purpose,
 but this usage is deprecated.
+
+
+## How do I create custom python packages from javascript?
+
+Put a collection of functions into a javascript object and use `pyodide.registerJsModule`:
+Javascript:
+```javascript
+let my_module = {
+  f : function(x){
+    return x*x + 1;
+  },
+  g : function(x){
+    console.log(`Calling g on argument ${x}`);
+    return x;
+  },
+  submodule : {
+    h : function(x) {
+      return x*x - 1;
+    },
+    c  : 2,
+  },  
+};
+pyodide.registerJsModule("my_js_module", my_module);
+```
+You can import your package like a normal Python package:
+```
+import my_js_module
+from my_js_module.submodule import h, c
+assert my_js_module.f(7) == 50
+assert h(9) == 80
+assert c == 2
+```
diff --git a/docs/js-api/pyodide_registerJsModule.md b/docs/js-api/pyodide_registerJsModule.md
@@ -0,0 +1,12 @@
+(js_api_pyodide_registerJsModule)=
+# pyodide.registerJsModule(name, module)
+
+Registers the Js object ``module`` as a Js module with ``name``. This module can then be imported from Python using the standard Python import system. If another module by the same name has already been imported, this won't have much effect unless you also delete the imported module from ``sys.modules``. This calls the ``pyodide_py`` api ``pyodide_py.register_js_module``.
+
+
+**Parameters**
+
+| name      | type   | description                          |
+|-----------|--------|--------------------------------------|
+| *name*    | String | Name of js module                    |
+| *module*  | object | Javascript object backing the module |
diff --git a/docs/js-api/pyodide_unregisterJsModule.md b/docs/js-api/pyodide_unregisterJsModule.md
@@ -0,0 +1,11 @@
+(js_api_pyodide_unregisterJsModule)=
+# pyodide.unregisterJsModule(name)
+
+Unregisters a Js module with given name that has been previously registered with `js_api_pyodide_registerJsModule` or ``pyodide.register_js_module``. If a Js module with that name does not already exist, will throw an error. Note that if the module has already been imported, this won't have much effect unless you also delete the imported module from ``sys.modules``. This calls the ``pyodide_py`` api ``pyodide_py.unregister_js_module``.
+
+**Parameters**
+
+| name    | type   | description                    |
+|---------|--------|--------------------------------|
+| *name*  | String | Name of js module              |
+
diff --git a/src/core/jsimport.c b/src/core/jsimport.c
diff --git a/src/core/jsimport.h b/src/core/jsimport.h
diff --git a/src/core/main.c b/src/core/main.c
@@ -7,7 +7,6 @@
 #include "error_handling.h"
 #include "hiwire.h"
 #include "js2python.h"
-#include "jsimport.h"
 #include "jsproxy.h"
 #include "pyproxy.h"
 #include "python2js.h"
@@ -97,7 +96,6 @@ main(int argc, char** argv)
   TRY_INIT(error_handling);
   TRY_INIT(js2python);
   TRY_INIT_WITH_CORE_MODULE(JsProxy); // JsProxy needs to be before JsImport
-  TRY_INIT(JsImport);
   TRY_INIT(pyproxy);
   TRY_INIT(python2js);
 

diff --git a/src/pyodide-py/pyodide/__init__.py b/src/pyodide-py/pyodide/__init__.py
@@ -1,5 +1,12 @@
 from ._base import open_url, eval_code, find_imports, as_nested_list
 from ._core import JsException  # type: ignore
+from ._importhooks import JsFinder
+import sys
+
+jsfinder = JsFinder()
+register_js_module = jsfinder.register_js_module
+unregister_js_module = jsfinder.unregister_js_module
+sys.meta_path.append(jsfinder)  # type: ignore
 
 
 __version__ = "0.16.1"
@@ -10,4 +17,6 @@
     "find_imports",
     "as_nested_list",
     "JsException",
+    "register_js_module",
+    "unregister_js_module",
 ]
diff --git a/src/pyodide-py/pyodide/_importhooks.py b/src/pyodide-py/pyodide/_importhooks.py
@@ -0,0 +1,96 @@
+from ._core import JsProxy
+from importlib.abc import MetaPathFinder, Loader
+from importlib.util import spec_from_loader
+import sys
+
+
+class JsFinder(MetaPathFinder):
+    def __init__(self):
+        self.jsproxies = {}
+
+    def find_spec(self, fullname, path, target=None):
+        [parent, _, child] = fullname.rpartition(".")
+        if parent:
+            parent_module = sys.modules[parent]
+            if not isinstance(parent_module, JsProxy):
+                # Not one of us.
+                return None
+            try:
+                jsproxy = getattr(parent_module, child)
+            except AttributeError:
+                raise ModuleNotFoundError(
+                    f"No module named {fullname!r}", name=fullname
+                ) from None
+            if not isinstance(jsproxy, JsProxy):
+                raise ModuleNotFoundError(
+                    f"No module named {fullname!r}", name=fullname
+                )
+        else:
+            try:
+                jsproxy = self.jsproxies[fullname]
+            except KeyError:
+                return None
+        loader = JsLoader(jsproxy)
+        return spec_from_loader(fullname, loader, origin="javascript")
+
+    def register_js_module(self, name, jsproxy):
+        """
+        Registers the Js object ``module`` as a Js module with ``name``. The module
+        can then be imported from Python using the standard Python import system.
+        If another module by the same name has already been imported, this won't
+        have much effect unless you also delete the imported module from
+        ``sys.modules``. This is called by the javascript API
+        ``pyodide.registerJsModule``.
+
+        Parameters
+        ----------
+        name : str
+            Name of js module
+        jsproxy : JsProxy
+            Javascript object backing the module
+        """
+        if not isinstance(name, str):
+            raise TypeError(
+                f"Argument 'name' must be a str, not {type(name).__name__!r}"
+            )
+        if not isinstance(jsproxy, JsProxy):
+            raise TypeError(
+                f"Argument 'jsproxy' must be a JsProxy, not {type(jsproxy).__name__!r}"
+            )
+        self.jsproxies[name] = jsproxy
+
+    def unregister_js_module(self, name):
+        """
+        Unregisters a Js module with given name that has been previously registered
+        with `js_api_pyodide_registerJsModule` or ``pyodide.register_js_module``. If
+        a Js module with that name does not already exist, will raise an error. Note
+        that if the module has already been imported, this won't have much effect
+        unless you also delete the imported module from ``sys.modules``. This is
+        called by the javascript API ``pyodide.unregisterJsModule``.
+
+        Parameters
+        ----------
+        name : str
+            Name of js module
+        """
+        try:
+            del self.jsproxies[name]
+        except KeyError:
+            raise ValueError(
+                f"Cannot unregister {name!r}: no javascript module with that name is registered"
+            ) from None
+
+
+class JsLoader(Loader):
+    def __init__(self, jsproxy):
+        self.jsproxy = jsproxy
+
+    def create_module(self, spec):
+        return self.jsproxy
+
+    def exec_module(self, module):
+        pass
+
+    # used by importlib.util.spec_from_loader
+    def is_package(self, fullname):
+        return True
diff --git a/src/pyodide.js b/src/pyodide.js
@@ -313,6 +313,8 @@ globalThis.languagePluginLoader = new Promise((resolve, reject) => {
     'runPython',
     'runPythonAsync',
     'version',
+    'registerJsModule',
+    'unregisterJsModule',
   ];
 
   function makePublicAPI(module, public_api) {
@@ -365,6 +367,11 @@ globalThis.languagePluginLoader = new Promise((resolve, reject) => {
     return Module.runPython(code);
   };
 
+  Module.registerJsModule = function(
+      name, module) { Module.pyodide_py.register_js_module(name, module); };
+  Module.unregisterJsModule = function(
+      name) { Module.pyodide_py.unregister_js_module(name); };
+
   Module.function_supports_kwargs = function(funcstr) {
     // This is basically a finite state machine (except for paren counting)
     // Start at beginning of argspec
@@ -461,6 +468,7 @@ globalThis.languagePluginLoader = new Promise((resolve, reject) => {
     let response = await fetch(`${baseURL}packages.json`);
     let json = await response.json();
     fixRecursionLimit(self.pyodide);
+    self.pyodide.registerJsModule("js", globalThis);
     self.pyodide = makePublicAPI(self.pyodide, PUBLIC_API);
     self.pyodide._module.packages = json;
     resolve();