Clapack as so (pyodide#1236)

jmsmdy · Feb 26, 2021 · 451924b · 451924b
1 parent 47018e0
commit 451924b
Show file tree

Hide file tree

Showing 9 changed files with 96 additions and 90 deletions.
diff --git a/Makefile b/Makefile
@@ -29,6 +29,7 @@ LDFLAGS=\
 	$(CPYTHONROOT)/installs/python-$(PYVERSION)/lib/libpython$(PYMINOR).a \
 	-s TOTAL_MEMORY=20971520 \
 	-s ALLOW_MEMORY_GROWTH=1 \
+    --use-preload-plugins \
 	-s MAIN_MODULE=1 \
 	-s EMULATE_FUNCTION_POINTER_CASTS=1 \
 	-s LINKABLE=1 \

diff --git a/docs/development/new-packages.md b/docs/development/new-packages.md
@@ -148,6 +148,10 @@ Extra arguments to pass to the linker when building for WebAssembly.
 
 Should be set to true for library packages. Library packages are packages that are needed for other packages but are not Python packages themselves. For library packages, the script specified in the `build/script` section is run to compile the library. See the [zlib meta.yaml](https://github.com/iodide-project/pyodide/blob/master/packages/zlib/meta.yaml) for an example of a library package specification.
 
+#### `build/sharedlibrary`
+
+Should be set to true for shared library packages. Shared library packages are packages that are needed for other packages, but are loaded dynamically when pyodide is run. For shared library packages, the script specified in the `build/script` section is run to compile the library. The script should build the shared library and copy into into a subfolder of the source folder called `install`. Files or folders in this install folder will be packaged to make the pyodide package. See the [CLAPACK meta.yaml](https://github.com/iodide-project/pyodide/blob/master/packages/CLAPACK/meta.yaml) for an example of a shared library specification.
+
 #### `build/script`
 
 The script section is required for a library package (`build/library` set to true). For a Python package this section is optional. If it is specified for a Python package, the script section will be run before the build system runs `setup.py`. This script is run by `bash` in the directory where the tarball was extracted.

diff --git a/docs/project/changelog.md b/docs/project/changelog.md
@@ -12,6 +12,11 @@ substitutions:
 
 ## Version [Unreleased]
 
+### Improvements to package loading and dynamic linking
+- {{Enhancement}} Uses the emscripten preload plugin system to preload .so files in packages
+- {{Enhancement}} Support for shared library packages. This is used for CLAPACK which makes scipy a lot smaller.
+  [#1236] https://github.com/iodide-project/pyodide/pull/1236
+
 ### Python / JS type conversions
 - {{ Feature }} A `JsProxy` of a Javascript `Promise` or other awaitable object is now a
   Python awaitable.

diff --git a/packages/CLAPACK/meta.yaml b/packages/CLAPACK/meta.yaml
@@ -17,7 +17,7 @@ source:
     - [make.inc, make.inc]
 
 build:
-  library: true
+  sharedlibrary: true
   script: |
     # The archive's contents have default permission 0750. If we use docker
     # to build, then we will not own the contents in the host, which means
@@ -37,3 +37,5 @@ build:
     # blas_WA.a, lapack_WA.a which are linked statically in scipy
     # in each module that needs them.
     emmake make -j ${PYODIDE_JOBS:-3} blaslib lapacklib
+    mkdir -p install/lib
+    emcc blas_WA.a  lapack_WA.a F2CLIBS/libf2c.a -sSIDE_MODULE -sEMULATE_FUNCTION_POINTER_CASTS -s BINARYEN_EXTRA_PASSES="--pass-arg=max-func-params@61" -o install/lib/clapack_all.so 
diff --git a/pyodide_build/buildall.py b/pyodide_build/buildall.py
@@ -32,6 +32,9 @@ def __init__(self, pkgdir: Path):
         self.meta: dict = parse_package_config(pkgpath)
         self.name: str = self.meta["package"]["name"]
         self.library: bool = self.meta.get("build", {}).get("library", False)
+        self.shared_library: bool = self.meta.get("build", {}).get(
+            "sharedlibrary", False
+        )
 
         assert self.name == pkgdir.stem
 
@@ -215,14 +218,16 @@ def build_packages(packages_dir: Path, outputdir: Path, args) -> None:
     package_data: dict = {
         "dependencies": {"test": []},
         "import_name_to_package_name": {},
+        "shared_library": {},
     }
 
     libraries = [pkg.name for pkg in pkg_map.values() if pkg.library]
 
     for name, pkg in pkg_map.items():
         if pkg.library:
             continue
-
+        if pkg.shared_library:
+            package_data["shared_library"][name] = True
         package_data["dependencies"][name] = [
             x for x in pkg.dependencies if x not in libraries
         ]

diff --git a/pyodide_build/buildpkg.py b/pyodide_build/buildpkg.py
@@ -280,7 +280,11 @@ def build_package(path: Path, args):
         if pkg.get("build", {}).get("script"):
             run_script(buildpath, srcpath, pkg)
         if not pkg.get("build", {}).get("library", False):
-            compile(path, srcpath, pkg, args)
+            # shared libraries get built by the script and put into install
+            # subfolder, then packaged into a pyodide module
+            # i.e. they need package running, but not compile
+            if not pkg.get("build", {}).get("sharedlibrary"):
+                compile(path, srcpath, pkg, args)
             package_files(buildpath, srcpath, pkg, args)
     finally:
         os.chdir(orig_path)

diff --git a/pyodide_build/io.py b/pyodide_build/io.py
@@ -24,6 +24,7 @@
         "cxxflags": str,
         "ldflags": str,
         "library": bool,
+        "sharedlibrary": bool,
         "script": str,
         "post": str,
         "replace-libs": list,

diff --git a/pyodide_build/pywasmcross.py b/pyodide_build/pywasmcross.py
@@ -318,43 +318,6 @@ def handle_command(line, args, dryrun=False):
         ):
             continue
 
-        # Fix for scipy to link to the correct BLAS/LAPACK files
-        if arg.startswith("-L") and "CLAPACK" in arg:
-            out_idx = line.index("-o")
-            out_idx += 1
-            module_name = line[out_idx]
-            module_name = Path(module_name).name.split(".")[0]
-
-            lapack_dir = arg.replace("-L", "")
-            # For convenience we determine needed scipy link libraries
-            # here, instead of in patch files
-            link_libs = ["F2CLIBS/libf2c.a", "blas_WA.a"]
-            if module_name in [
-                "_flapack",
-                "_flinalg",
-                "_calc_lwork",
-                "cython_lapack",
-                "_iterative",
-                "_arpack",
-            ]:
-                link_libs.append("lapack_WA.a")
-
-            for lib_name in link_libs:
-                arg = os.path.join(lapack_dir, f"{lib_name}")
-                new_args.append(arg)
-
-            new_args.extend(["-s", "INLINING_LIMIT=5"])
-            continue
-
-        # Use -Os for files that are statically linked to CLAPACK
-        if (
-            arg.startswith("-O")
-            and "CLAPACK" in " ".join(line)
-            and "-L" in " ".join(line)
-        ):
-            new_args.append("-Os")
-            continue
-
         if new_args[-1].startswith("-B") and "compiler_compat" in arg:
             # conda uses custom compiler search paths with the compiler_compat folder.
             # Ignore it.

diff --git a/src/pyodide.js b/src/pyodide.js
@@ -28,49 +28,6 @@ globalThis.languagePluginLoader = (async () => {
     }
   };
 
-  // clang-format off
-  let preloadWasm = () => {
-    // On Chrome, we have to instantiate wasm asynchronously. Since that
-    // can't be done synchronously within the call to dlopen, we instantiate
-    // every .so that comes our way up front, caching it in the
-    // `preloadedWasm` dictionary.
-
-    let promise = new Promise((resolve) => resolve());
-    let FS = pyodide._module.FS;
-
-    function recurseDir(rootpath) {
-      let dirs;
-      try {
-        dirs = FS.readdir(rootpath);
-      } catch {
-        return;
-      }
-      for (let entry of dirs) {
-        if (entry.startsWith('.')) {
-          continue;
-        }
-        const path = rootpath + entry;
-        if (entry.endsWith('.so')) {
-          if (Module['preloadedWasm'][path] === undefined) {
-            promise = promise
-              .then(() => Module['loadWebAssemblyModule'](
-                FS.readFile(path), {loadAsync: true,allowUndefined: true}))
-              .then((module) => {
-                Module['preloadedWasm'][path] = module;
-              });
-          }
-        } else if (FS.isDir(FS.lookupPath(path).node.mode)) {
-          recurseDir(path + '/');
-        }
-      }
-    }
-
-    recurseDir('/');
-
-    return promise;
-  }
-  // clang-format on
-
   let loadScript;
   if (self.document) { // browser
     loadScript = (url) => new Promise((res, rej) => {
@@ -88,9 +45,11 @@ globalThis.languagePluginLoader = (async () => {
     throw new Error("Cannot determine runtime environment");
   }
 
-  function recursiveDependencies(names, _messageCallback, errorCallback) {
-    const packages = Module.packages.dependencies;
+  function recursiveDependencies(names, _messageCallback, errorCallback,
+                                 sharedLibsOnly) {
+    const packages = self.pyodide._module.packages.dependencies;
     const loadedPackages = self.pyodide.loadedPackages;
+    const sharedLibraries = self.pyodide._module.packages.shared_library;
     const toLoad = new Map();
 
     const addPackage = (pkg) => {
@@ -123,6 +82,15 @@ globalThis.languagePluginLoader = (async () => {
         errorCallback(`Skipping unknown package '${name}'`);
       }
     }
+    if (sharedLibsOnly) {
+      onlySharedLibs = new Map();
+      for (let c of toLoad) {
+        if (c[0] in sharedLibraries) {
+          onlySharedLibs.set(c[0], toLoad.get(c[0]));
+        }
+      }
+      return onlySharedLibs;
+    }
     return toLoad;
   }
 
@@ -249,10 +217,8 @@ globalThis.languagePluginLoader = (async () => {
       resolveMsg = 'No packages loaded';
     }
 
-    if (!isFirefox) {
-      await preloadWasm();
-      Module.reportUndefinedSymbols();
-    }
+    Module.reportUndefinedSymbols();
+
     messageCallback(resolveMsg);
 
     // We have to invalidate Python's import caches, or it won't
@@ -291,7 +257,61 @@ globalThis.languagePluginLoader = (async () => {
     if (!Array.isArray(names)) {
       names = [ names ];
     }
+    // get shared library packages and load those first
+    // otherwise bad things happen with linking them in firefox.
+    sharedLibraryNames = [];
+    try {
+      sharedLibraryPackagesToLoad =
+          recursiveDependencies(names, messageCallback, errorCallback, true);
+      for (pkg of sharedLibraryPackagesToLoad) {
+        sharedLibraryNames.push(pkg[0]);
+      }
+    } catch (e) {
+      // do nothing - let the main load throw any errors
+    }
+    // override the load plugin so that it imports any dlls also
+    // this only needs to be done for shared library packages because
+    // we assume that if a package depends on a shared library
+    // it needs to have access to it.
+    // not needed for so in standard module because those are linked together
+    // correctly, it is only where linking goes across modules that it needs to
+    // be done. Hence we only put this extra preload plugin in during the shared
+    // library load
+    let oldPlugin;
+    for (let p in Module.preloadPlugins) {
+      if (Module.preloadPlugins[p].canHandle("test.so")) {
+        oldPlugin = Module.preloadPlugins[p];
+        break;
+      }
+    }
+    let dynamicLoadHandler = {
+      get : function(obj, prop) {
+        if (prop === 'handle') {
+          return function(bytes, name) {
+            obj[prop].apply(obj, arguments);
+            this["asyncWasmLoadPromise"] =
+                this["asyncWasmLoadPromise"].then(function() {
+                  Module.loadDynamicLibrary(name,
+                                            {global : true, nodelete : true})
+                });
+          }
+        } else {
+          return obj[prop];
+        }
+      }
+    };
+    var loadPluginOverride = new Proxy(oldPlugin, dynamicLoadHandler);
+    // restore the preload plugin
+    Module.preloadPlugins.unshift(loadPluginOverride);
+
     let promise = loadPackageChain.then(
+        () => _loadPackage(sharedLibraryNames, messageCallback || console.log,
+                           errorCallback || console.error));
+    loadPackageChain = loadPackageChain.then(() => promise.catch(() => {}));
+    await promise;
+    Module.preloadPlugins.shift(loadPluginOverride);
+
+    promise = loadPackageChain.then(
         () => _loadPackage(names, messageCallback || console.log,
                            errorCallback || console.error));
     loadPackageChain = loadPackageChain.then(() => promise.catch(() => {}));
@@ -360,7 +380,8 @@ globalThis.languagePluginLoader = (async () => {
 
   Module.noImageDecoding = true;
   Module.noAudioDecoding = true;
-  Module.noWasmDecoding = true;
+  Module.noWasmDecoding =
+      false; // we preload wasm using the built in plugin now
   Module.preloadedWasm = {};
   let isFirefox = navigator.userAgent.toLowerCase().indexOf('firefox') > -1;