tasks: configurable compilation for different targets

faasm · csegarragonz · Mar 12, 2024 · Feb 9, 2024 · Feb 9, 2024 · Feb 12, 2024
commit d69921e69fb5ab8930c8cac028b30827ab7cca15
diff --git a/faasmtools/build.py b/faasmtools/build.py
@@ -15,8 +15,6 @@
 FAASM_NATIVE_DIR = join(FAASM_LOCAL_DIR, "native")
 FAASM_CMAKE_ROOT = "/usr/share/cmake-3.24"
 WASM_SYSROOT = join(FAASM_LOCAL_DIR, "llvm-sysroot")
-WASM_HEADER_INSTALL = "{}/include/wasm32-wasi".format(WASM_SYSROOT)
-WASM_LIB_INSTALL = "{}/lib/wasm32-wasi".format(WASM_SYSROOT)
 WASM_TOOLCHAIN_ROOT = join(FAASM_LOCAL_DIR, "toolchain")
 WASM_TOOLCHAIN_TOOLS = join(WASM_TOOLCHAIN_ROOT, "tools")
 WASM_TOOLCHAIN_BIN = join(WASM_TOOLCHAIN_ROOT, "bin")
@@ -38,6 +36,7 @@
 WASM_LDXX = WASM_CXX
 
 # Host triple
+# TODO: remove this?
 WASM_BUILD = "wasm32"
 WASM_HOST = "wasm32-unknown-wasi"
 WASM_HOST_STATIC = "wasm32-wasi"
@@ -73,7 +72,7 @@
 # https://reviews.llvm.org/D59281
 WASM_CFLAGS = [
     "-O3",
-    "-mno-atomics",
+    # "-mno-atomics",
     "-msimd128",
     "--sysroot={}".format(WASM_SYSROOT),
     "-m32",
@@ -142,7 +141,6 @@
     "-Xlinker --export={}".format(FAASM_WASM_CTORS_FUNC_NAME),
     "-Xlinker --export=__stack_pointer",
     "-Xlinker --max-memory={}".format(FAASM_WASM_MAX_MEMORY),
-    "-Xlinker --features=bulk-memory,mutable-globals,sign-ext,simd128",
     "-Wl,-z,stack-size={} -Wl".format(FAASM_WASM_STACK_SIZE),
 ]
 
@@ -190,7 +188,7 @@
 # depending on the build type variables target
 # WARNING: do NOT import this method directly, instead use the getter method:
 # get_faasm_build_env_dict
-FAASM_BUILD_ENV_DICT = {
+_FAASM_BUILD_ENV_DICT = {
     "CMAKE_ROOT": FAASM_CMAKE_ROOT,
     "FAASM_NATIVE_INSTALL_DIR": FAASM_NATIVE_DIR,
     "FAASM_WASM_MAKE_TOOLCHAIN_FILE": MAKE_TOOLCHAIN_FILE,
@@ -203,8 +201,6 @@
     "FAASM_WASM_HOST_STATIC": WASM_HOST_STATIC,
     "FAASM_WASM_HOST_UNKNOWN": WASM_HOST_UNKNOWN,
     "FAASM_WASM_INSTALL_DIR": WASM_TOOLCHAIN_BIN,
-    "FAASM_WASM_HEADER_INSTALL_DIR": WASM_HEADER_INSTALL,
-    "FAASM_WASM_LIB_INSTALL_DIR": WASM_LIB_INSTALL,
     "FAASM_WASM_SYSROOT": WASM_SYSROOT,
     "FAASM_WASM_CFLAGS": " ".join(WASM_CFLAGS),
     "FAASM_WASM_CFLAGS_SHARED": " ".join(WASM_CFLAGS_SHARED),
@@ -233,16 +229,46 @@ def get_faasm_build_env_dict(is_threads=False):
     This method returns the right set of environment variables needed to use
     our toolchain file as well as most cross-compilation scripts in Faasm.
     """
-    build_env_dicts = FAASM_BUILD_ENV_DICT
+    build_env_dicts = _FAASM_BUILD_ENV_DICT
     if is_threads:
-        build_env_dicts["FAASM_WASM_TRIPLE"] = "wasm32-wasi-threads"
+        wasm_triple = "wasm32-wasi-threads"
+        build_env_dicts["FAASM_WASM_TRIPLE"] = wasm_triple
         build_env_dicts["FAASM_WASM_CFLAGS"] += " -pthread"
         build_env_dicts["FAASM_WASM_CXXFLAGS"] += " -pthread"
-        build_env_dicts["FAASM_WASM_EXE_LINKER_FLAGS"] += " -Wl,--import-memory"
-        build_env_dicts["FAASM_WASM_EXE_LINKER_FLAGS"] += " -Wl,--export-memory"
+        linker_features = [
+            "atomics",
+            "bulk-memory",
+            "mutable-globals",
+            "sign-ext",
+            "simd128",
+        ]
+        build_env_dicts[
+            "FAASM_WASM_EXE_LINKER_FLAGS"
+        ] += " -Wl,--import-memory"
+        build_env_dicts[
+            "FAASM_WASM_EXE_LINKER_FLAGS"
+        ] += " -Wl,--export-memory"
     else:
-        build_env_dicts["FAASM_WASM_TRIPLE"] = "wasm32-wasi"
-
+        wasm_triple = "wasm32-wasi"
+        build_env_dicts["FAASM_WASM_TRIPLE"] = wasm_triple
+        linker_features = [
+            "bulk-memory",
+            "mutable-globals",
+            "sign-ext",
+            "simd128",
+        ]
+
+    build_env_dicts["FAASM_WASM_HEADER_INSTALL_DIR"] = join(
+        WASM_SYSROOT, "include", wasm_triple
+    )
+    build_env_dicts["FAASM_WASM_LIB_INSTALL_DIR"] = join(
+        WASM_SYSROOT, "lib", wasm_triple
+    )
+    build_env_dicts[
+        "FAASM_WASM_EXE_LINKER_FLAGS"
+    ] += " -Xlinker --features={}".format(",".join(linker_features))
+
+    return build_env_dicts
 
 
 def get_dict_as_cmake_vars(env_dict):

diff --git a/faasmtools/compile_util.py b/faasmtools/compile_util.py
@@ -1,6 +1,6 @@
 from faasmtools.build import (
     CMAKE_TOOLCHAIN_FILE,
-    FAASM_BUILD_ENV_DICT,
+    get_faasm_build_env_dict,
 )
 from faasmtools.env import WASM_DIR
 from os import environ, makedirs
@@ -9,7 +9,9 @@
 from subprocess import run
 
 
-def wasm_cmake(src_dir, build_dir, target, clean=False, debug=False, is_threads=False):
+def wasm_cmake(
+    src_dir, build_dir, target, clean=False, debug=False, is_threads=False
+):
     cmake_build_type = "Debug" if debug else "Release"
 
     if exists(build_dir) and clean:
@@ -28,7 +30,7 @@ def wasm_cmake(src_dir, build_dir, target, clean=False, debug=False, is_threads=
     print(build_cmd)
 
     work_env = environ.copy()
-    work_env.update(FAASM_BUILD_ENV_DICT)
+    work_env.update(get_faasm_build_env_dict(is_threads))
 
     res = run(build_cmd, shell=True, cwd=build_dir, env=work_env)
     if res.returncode != 0:

diff --git a/tasks/clapack.py b/tasks/clapack.py
@@ -1,9 +1,8 @@
 from copy import copy
 from faasmtools.env import THIRD_PARTY_DIR, USABLE_CPUS
 from faasmtools.build import (
-    FAASM_BUILD_ENV_DICT,
     WASM_SYSROOT,
-    WASM_LIB_INSTALL,
+    get_faasm_build_env_dict,
 )
 from invoke import task
 from os import environ, remove
@@ -37,7 +36,7 @@ def build(ctx, clean=False, shared=False):
     # Set up environment to specify whether we're building static or shared
     env = copy(environ)
     env.update({"LIBEXT": ".so" if shared else ".a"})
-    env.update(FAASM_BUILD_ENV_DICT)
+    env.update(get_faasm_build_env_dict())
 
     # Make libf2c first (needed by others)
     run(
@@ -70,14 +69,20 @@ def uninstall(ctx):
     """
     Removes all installed files
     """
-    for headers_dir in HEADERS_DIRS:
+    build_env = get_faasm_build_env_dict()
+
+    for headers_dir in build_env["FAASM_WASM_HEADER_INSTALL_DIR"]:
         if exists(headers_dir):
             print("Removing headers {}".format(headers_dir))
             rmtree(headers_dir)
 
     for lib_name in INSTALLED_LIBS:
-        static_path = join(WASM_LIB_INSTALL, "{}.a".format(lib_name))
-        shared_path = join(WASM_LIB_INSTALL, "{}.so".format(lib_name))
+        static_path = join(
+            build_env["FAASM_WASM_LIB_INSTALL_DIR"], "{}.a".format(lib_name)
+        )
+        shared_path = join(
+            build_env["FAASM_WASM_LIB_INSTALL_DIR"], "{}.so".format(lib_name)
+        )
 
         if exists(static_path):
             print("Removing static lib {}".format(static_path))

diff --git a/tasks/func.py b/tasks/func.py
@@ -72,7 +72,14 @@ def compile(ctx, user, func, clean=False, debug=False, native=False):
         )
     else:
         # Build the function (gets written to the build dir)
-        wasm_cmake(FUNC_DIR, FUNC_BUILD_DIR, func, clean, debug, _is_threaded_func(user, func))
+        wasm_cmake(
+            FUNC_DIR,
+            FUNC_BUILD_DIR,
+            func,
+            clean,
+            debug,
+            _is_threaded_func(user, func),
+        )
 
         # Copy into place
         _copy_built_function(user, func)
@@ -153,7 +160,14 @@ def user(ctx, user, clean=False, debug=False):
     """
     # Build all funcs for this user (will fail if any builds fail)
     target = "{}_all_funcs".format(user)
-    wasm_cmake(FUNC_DIR, FUNC_BUILD_DIR, target, clean, debug, _is_threaded_func(user, ""))
+    wasm_cmake(
+        FUNC_DIR,
+        FUNC_BUILD_DIR,
+        target,
+        clean,
+        debug,
+        _is_threaded_func(user, ""),
+    )
 
     funcs = _get_all_user_funcs(user)
     for f in funcs:

diff --git a/tasks/lib.py b/tasks/lib.py
@@ -1,8 +1,7 @@
 from faasmtools.build import (
     CMAKE_TOOLCHAIN_FILE,
-    FAASM_BUILD_ENV_DICT,
     FAASM_NATIVE_DIR,
-    WASM_LIB_INSTALL,
+    get_faasm_build_env_dict,
 )
 from faasmtools.env import LLVM_VERSION, PROJ_ROOT
 from os.path import exists, join
@@ -11,12 +10,18 @@
 from subprocess import run
 
 
-def build_faasm_lib(subdir, clean=False, native=False, shared=False):
+def build_faasm_lib(
+    subdir, clean=False, native=False, shared=False, threads=False
+):
     """
     Builds one of the libraries included in this repo
     """
     work_dir = join(PROJ_ROOT, subdir)
-    install_dir = FAASM_NATIVE_DIR if native else WASM_LIB_INSTALL
+    install_dir = (
+        FAASM_NATIVE_DIR
+        if native
+        else get_faasm_build_env_dict(threads)["FAASM_WASM_LIB_INSTALL_DIR"]
+    )
 
     if native and shared:
         build_dir = "build-native-shared"
@@ -62,7 +67,7 @@ def build_faasm_lib(subdir, clean=False, native=False, shared=False):
 
     work_env = environ.copy()
     if not native:
-        work_env.update(FAASM_BUILD_ENV_DICT)
+        work_env.update(get_faasm_build_env_dict(is_threads=threads))
     run(build_cmd_str, shell=True, cwd=build_dir, check=True, env=work_env)
 
     run("ninja", shell=True, cwd=build_dir, check=True)

diff --git a/tasks/libfaasm.py b/tasks/libfaasm.py
@@ -3,8 +3,10 @@
 
 
 @task(default=True)
-def build(ctx, clean=False, native=False, shared=False):
+def build(ctx, clean=False, native=False, shared=False, threads=False):
     """
     Builds faasm C/C++ lib
     """
-    build_faasm_lib("libfaasm", clean=clean, native=native, shared=shared)
+    build_faasm_lib(
+        "libfaasm", clean=clean, native=native, shared=shared, threads=threads
+    )
diff --git a/tasks/libfake.py b/tasks/libfake.py
@@ -1,7 +1,7 @@
 from faasmtools.build import (
     CMAKE_TOOLCHAIN_FILE,
-    FAASM_BUILD_ENV_DICT,
     WASM_SYSROOT,
+    get_faasm_build_env_dict,
 )
 from faasmtools.env import PROJ_ROOT, FAASM_RUNTIME_ROOT
 from invoke import task
@@ -35,7 +35,7 @@ def fake(ctx, clean=False):
     ]
 
     work_env = environ.copy()
-    work_env.update(FAASM_BUILD_ENV_DICT)
+    work_env.update(get_faasm_build_env_dict())
     run(
         " ".join(build_cmd),
         shell=True,

diff --git a/tasks/libffi.py b/tasks/libffi.py
@@ -1,20 +1,18 @@
 from faasmtools.build import (
     build_config_cmd,
+    get_faasm_build_env_dict,
     run_autotools,
-    WASM_SYSROOT,
-    WASM_LIB_INSTALL,
 )
-from faasmtools.env import THIRD_PARTY_DIR, USABLE_CPUS, PROJ_ROOT
+from faasmtools.env import THIRD_PARTY_DIR, USABLE_CPUS
 from invoke import task
 from os.path import join, exists
 from subprocess import run
 
 LIBFFI_DIR = join(THIRD_PARTY_DIR, "libffi")
-LIBFFI_LIBS_DIR = join(LIBFFI_DIR, "wasm32-unknown-wasi")
 
 
 @task(default=True)
-def build(ctx, clean=False):
+def build(ctx, clean=False, threads=False):
     """
     Builds libffi
     """
@@ -25,10 +23,14 @@ def build(ctx, clean=False):
     run_autotools(LIBFFI_DIR)
 
     # Configure
+    build_env = get_faasm_build_env_dict(threads)
     configure_cmd = build_config_cmd(
         [
             "./configure",
-            "--prefix={}".format(WASM_SYSROOT),
+            "--includedir={}".format(
+                build_env["FAASM_WASM_HEADER_INSTALL_DIR"]
+            ),
+            "--libdir={}".format(build_env["FAASM_WASM_LIB_INSTALL_DIR"]),
             "--disable-multi-os-directory",
             "--disable-docs",
         ],
@@ -44,14 +46,11 @@ def build(ctx, clean=False):
     run("make install", shell=True, check=True, cwd=LIBFFI_DIR)
 
     # Move lib into correct install dir
-    for lib_name in ["libffi.a", "libffi.la"]:
-        src_lib = join(WASM_SYSROOT, "lib", lib_name)
-        dest_lib = join(WASM_LIB_INSTALL, lib_name)
-        print("Moving {} to {}".format(src_lib, dest_lib))
-        run("mv {} {}".format(src_lib, dest_lib), shell=True, check=True)
-
     # Copy imports into place
+    # TODO: do we need these?
+    """
     src_imports = join(PROJ_ROOT, "sysroot_extras", "libffi.imports")
     dest_imports = join(WASM_LIB_INSTALL, "libffi.imports")
     print("Copying {} to {}".format(src_imports, dest_imports))
     run("cp {} {}".format(src_imports, dest_imports), check=True, shell=True)
+    """