Skip to content

Commit

Permalink
Single-file standalone Bun executables (oven-sh#2879)
Browse files Browse the repository at this point in the history 
* Add LIEF

* Compile LIEF

* Implement support for embedding files on macOS

* proof of concept

* Add zstd

* Implement runtime support

* Move some code around

* Update .gitmodules

* Upgrade zig

ziglang/zig#15278

* leftover

* leftover

* delete dead code

* Fix extname

* Revert "Upgrade zig"

This reverts commit dd968f3.

* Revert "leftover"

This reverts commit 7664de7.

* Revert "leftover"

This reverts commit 498005b.

* various fixes

* it works!

* leftover

* Make `zig build` a little faster

* give up on code signing support

* Support Linux & macOS

* Finish removing LIEF

* few more

* Add zstd to list of deps

* make it pretty

---------

Co-authored-by: Jarred Sumner <709451+Jarred-Sumner@users.noreply.github.com>
  • Loading branch information
@Jarred-Sumner
Jarred-Sumner and Jarred-Sumner authored May 14, 2023
1 parent 7f25aa9 commit 893f70f
Show file tree
Hide file tree
Showing 28 changed files with 1,416 additions and 2,024 deletions.
4 changes: 4 additions & 0 deletions .gitmodules
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,3 +65,7 @@ fetchRecurseSubmodules = false
[submodule "src/deps/c-ares"]
path = src/deps/c-ares
url = https://github.com/c-ares/c-ares.git
[submodule "src/deps/zstd"]
path = src/deps/zstd
url = git@github.com:facebook/zstd.git
ignore = dirty
14 changes: 13 additions & 1 deletion .vscode/settings.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@
"src/deps/lol-html": true,
"src/deps/c-ares": true,
"src/deps/tinycc": true,
"src/deps/zstd": true,
"test/snippets/package-json-exports/_node_modules_copy": true
},
"C_Cpp.files.exclude": {
Expand Down Expand Up @@ -204,7 +205,18 @@
"compare": "cpp",
"concepts": "cpp",
"typeindex": "cpp",
"__verbose_abort": "cpp"
"__verbose_abort": "cpp",
"__std_stream": "cpp",
"any": "cpp",
"charconv": "cpp",
"csignal": "cpp",
"format": "cpp",
"forward_list": "cpp",
"future": "cpp",
"regex": "cpp",
"span": "cpp",
"valarray": "cpp",
"codecvt": "cpp"
},
"cmake.configureOnOpen": false,
"C_Cpp.errorSquiggles": "enabled",
Expand Down
28 changes: 28 additions & 0 deletions Dockerfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -503,6 +503,33 @@ ENV LIB_ICU_PATH=${WEBKIT_DIR}/lib

RUN --mount=type=cache,target=/ccache cd $BUN_DIR && make sqlite

FROM bun-base as zstd

ARG DEBIAN_FRONTEND
ARG GITHUB_WORKSPACE
ARG ZIG_PATH
# Directory extracts to "bun-webkit"
ARG WEBKIT_DIR
ARG BUN_RELEASE_DIR
ARG BUN_DEPS_OUT_DIR
ARG BUN_DIR

ARG CPU_TARGET
ENV CPU_TARGET=${CPU_TARGET}

ENV CCACHE_DIR=/ccache

COPY Makefile ${BUN_DIR}/Makefile
COPY src/deps/zstd ${BUN_DIR}/src/deps/zstd
COPY .prettierrc.cjs ${BUN_DIR}/.prettierrc.cjs

WORKDIR $BUN_DIR

ENV JSC_BASE_DIR=${WEBKIT_DIR}
ENV LIB_ICU_PATH=${WEBKIT_DIR}/lib

RUN --mount=type=cache,target=/ccache cd $BUN_DIR && make zstd

FROM scratch as build_release_cpp

COPY --from=compile_cpp /tmp/*.o /
Expand Down Expand Up @@ -535,6 +562,7 @@ COPY --from=lolhtml ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/
COPY --from=mimalloc ${BUN_DEPS_OUT_DIR}/*.o ${BUN_DEPS_OUT_DIR}/
COPY --from=picohttp ${BUN_DEPS_OUT_DIR}/*.o ${BUN_DEPS_OUT_DIR}/
COPY --from=sqlite ${BUN_DEPS_OUT_DIR}/*.o ${BUN_DEPS_OUT_DIR}/
COPY --from=zstd ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/
COPY --from=tinycc ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/
COPY --from=uws ${BUN_DEPS_OUT_DIR}/*.a ${BUN_DEPS_OUT_DIR}/
COPY --from=uws ${BUN_DEPS_OUT_DIR}/*.o ${BUN_DEPS_OUT_DIR}/
Expand Down
9 changes: 6 additions & 3 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -344,7 +344,7 @@ LINUX_INCLUDE_DIRS := $(ALL_JSC_INCLUDE_DIRS) \
UWS_INCLUDE_DIR := -I$(BUN_DEPS_DIR)/uws/uSockets/src -I$(BUN_DEPS_DIR)/uws/src -I$(BUN_DEPS_DIR)


INCLUDE_DIRS := $(UWS_INCLUDE_DIR) -I$(BUN_DEPS_DIR)/mimalloc/include -Isrc/napi -I$(BUN_DEPS_DIR)/boringssl/include -I$(BUN_DEPS_DIR)/c-ares/include
INCLUDE_DIRS := $(UWS_INCLUDE_DIR) -I$(BUN_DEPS_DIR)/mimalloc/include -I$(BUN_DEPS_DIR)/zstd/include -Isrc/napi -I$(BUN_DEPS_DIR)/boringssl/include -I$(BUN_DEPS_DIR)/c-ares/include


ifeq ($(OS_NAME),linux)
Expand Down Expand Up @@ -452,6 +452,7 @@ ARCHIVE_FILES_WITHOUT_LIBCRYPTO = $(MINIMUM_ARCHIVE_FILES) \
-ltcc \
-lusockets \
-lcares \
-lzstd \
$(BUN_DEPS_OUT_DIR)/libuwsockets.o

ARCHIVE_FILES = $(ARCHIVE_FILES_WITHOUT_LIBCRYPTO)
Expand Down Expand Up @@ -636,6 +637,9 @@ compile-ffi-test:

sqlite:

.PHONY: zstd
zstd:
cd $(BUN_DEPS_DIR)/zstd && rm -rf build-cmake-debug && cmake $(CMAKE_FLAGS) -DZSTD_BUILD_STATIC=ON -B build-cmake-debug -S build/cmake -G Ninja && ninja -C build-cmake-debug && cp build-cmake-debug/lib/libzstd.a $(BUN_DEPS_OUT_DIR)/libzstd.a

.PHONY: libarchive
libarchive:
Expand Down Expand Up @@ -908,7 +912,6 @@ bun-codesign-release-local:
bun-codesign-release-local-debug:



.PHONY: jsc
jsc: jsc-build jsc-copy-headers jsc-bindings
.PHONY: jsc-build
Expand Down Expand Up @@ -1857,7 +1860,7 @@ cold-jsc-start:
misctools/cold-jsc-start.cpp -o cold-jsc-start

.PHONY: vendor-without-npm
vendor-without-npm: node-fallbacks runtime_js fallback_decoder bun_error mimalloc picohttp zlib boringssl libarchive lolhtml sqlite usockets uws tinycc c-ares
vendor-without-npm: node-fallbacks runtime_js fallback_decoder bun_error mimalloc picohttp zlib boringssl libarchive lolhtml sqlite usockets uws tinycc c-ares zstd

.PHONY: vendor-without-check
vendor-without-check: npm-install vendor-without-npm
Expand Down
136 changes: 22 additions & 114 deletions build.zig
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,6 +71,24 @@ const BunBuildOptions = struct {
sizegen: bool = false,
base_path: [:0]const u8 = "",

runtime_js_version: u64 = 0,
fallback_html_version: u64 = 0,

pub fn updateRuntime(this: *BunBuildOptions) anyerror!void {
var runtime_out_file = try std.fs.cwd().openFile("src/runtime.out.js", .{ .mode = .read_only });
const runtime_hash = std.hash.Wyhash.hash(
0,
try runtime_out_file.readToEndAlloc(std.heap.page_allocator, try runtime_out_file.getEndPos()),
);
this.runtime_js_version = runtime_hash;
var fallback_out_file = try std.fs.cwd().openFile("src/fallback.out.js", .{ .mode = .read_only });
const fallback_hash = std.hash.Wyhash.hash(
0,
try fallback_out_file.readToEndAlloc(std.heap.page_allocator, try fallback_out_file.getEndPos()),
);
this.fallback_html_version = fallback_hash;
}

pub fn step(this: BunBuildOptions, b: anytype) *std.build.OptionsStep {
var opts = b.addOptions();
opts.addOption(@TypeOf(this.canary), "is_canary", this.canary);
Expand All @@ -79,6 +97,8 @@ const BunBuildOptions = struct {
opts.addOption(@TypeOf(this.bindgen), "bindgen", this.bindgen);
opts.addOption(@TypeOf(this.sizegen), "sizegen", this.sizegen);
opts.addOption(@TypeOf(this.base_path), "base_path", this.base_path);
opts.addOption(@TypeOf(this.runtime_js_version), "runtime_js_version", this.runtime_js_version);
opts.addOption(@TypeOf(this.fallback_html_version), "fallback_html_version", this.fallback_html_version);
return opts;
}
};
Expand All @@ -105,28 +125,6 @@ const fmt = struct {
}
};

fn updateRuntime() anyerror!void {
var runtime_out_file = try std.fs.cwd().openFile("src/runtime.out.js", .{ .mode = .read_only });
const runtime_hash = std.hash.Wyhash.hash(
0,
try runtime_out_file.readToEndAlloc(std.heap.page_allocator, try runtime_out_file.getEndPos()),
);
const runtime_version_file = std.fs.cwd().createFile("src/runtime.version", .{ .truncate = true }) catch std.debug.panic("Failed to create src/runtime.version", .{});
defer runtime_version_file.close();
runtime_version_file.writer().print("{any}", .{fmt.hexInt(runtime_hash)}) catch unreachable;
var fallback_out_file = try std.fs.cwd().openFile("src/fallback.out.js", .{ .mode = .read_only });
const fallback_hash = std.hash.Wyhash.hash(
0,
try fallback_out_file.readToEndAlloc(std.heap.page_allocator, try fallback_out_file.getEndPos()),
);

const fallback_version_file = std.fs.cwd().createFile("src/fallback.version", .{ .truncate = true }) catch std.debug.panic("Failed to create src/fallback.version", .{});

fallback_version_file.writer().print("{any}", .{fmt.hexInt(fallback_hash)}) catch unreachable;

fallback_version_file.close();
}

var x64 = "x64";
var optimize: std.builtin.OptimizeMode = undefined;

Expand Down Expand Up @@ -194,8 +192,6 @@ pub fn build(b: *Build) !void {
else
"root.zig";

updateRuntime() catch {};

const min_version: std.builtin.Version = if (target.getOsTag() != .freestanding)
target.getOsVersionMin().semver
else
Expand Down Expand Up @@ -271,6 +267,8 @@ pub fn build(b: *Build) !void {
obj.target.cpu_model = .{ .explicit = &std.Target.aarch64.cpu.generic };
}

try default_build_options.updateRuntime();

// we have to dump to stderr because stdout is read by zls
std.io.getStdErr().writer().print("Build {s} v{} - v{} ({s})\n", .{
triplet,
Expand Down Expand Up @@ -454,106 +452,16 @@ pub fn build(b: *Build) !void {
}

try configureObjectStep(b, headers_obj, @TypeOf(target), target, obj.main_pkg_path.?);
try linkObjectFiles(b, headers_obj, target);

headers_step.dependOn(&headers_obj.step);
headers_obj.addOptions("build_options", default_build_options.step(b));

// var iter = headers_obj.modules.iterator();
// while (iter.next()) |item| {
// const module = @ptrCast(*Module, item.value_ptr);
// }
// // while (headers_obj.modules.)
// for (headers_obj.packages.items) |pkg_| {
// const pkg: std.build.Pkg = pkg_;
// if (std.mem.eql(u8, pkg.name, "clap")) continue;
// var test_ = b.addTestSource(pkg.source);

// b
// .test_.setMainPkgPath(obj.main_pkg_path.?);
// try configureObjectStep(b, test_, @TypeOf(target), target, obj.main_pkg_path.?);
// try linkObjectFiles(b, test_, target);
// test_.addOptions("build_options", default_build_options.step(b));

// if (pkg.dependencies) |children| {
// test_.packages = std.ArrayList(std.build.Pkg).init(b.allocator);
// try test_.packages.appendSlice(children);
// }

// var before = b.addLog("\x1b[" ++ color_map.get("magenta").? ++ "\x1b[" ++ color_map.get("b").? ++ "[{s} tests]" ++ "\x1b[" ++ color_map.get("d").? ++ " ----\n\n" ++ "\x1b[0m", .{pkg.name});
// var after = b.addLog("\x1b[" ++ color_map.get("d").? ++ "–––---\n\n" ++ "\x1b[0m", .{});
// headers_step.dependOn(&before.step);
// headers_step.dependOn(&test_.step);
// headers_step.dependOn(&after.step);
// }
}

b.default_step.dependOn(obj_step);
}

pub var original_make_fn: ?*const fn (step: *std.build.Step) anyerror!void = null;

// Due to limitations in std.build.Builder
// we cannot use this with debugging
// so I am leaving this here for now, with the eventual intent to switch to std.build.Builder
// but it is dead code
pub fn linkObjectFiles(b: *Build, obj: *CompileStep, target: anytype) !void {
if (target.getOsTag() == .freestanding)
return;
var dirs_to_search = std.BoundedArray([]const u8, 32).init(0) catch unreachable;
const arm_brew_prefix: []const u8 = "/opt/homebrew";
const x86_brew_prefix: []const u8 = "/usr/local";
try dirs_to_search.append(b.env_map.get("BUN_DEPS_OUT_DIR") orelse b.env_map.get("BUN_DEPS_DIR") orelse @as([]const u8, b.pathFromRoot("src/deps")));
if (target.getOsTag() == .macos) {
if (target.getCpuArch().isAARCH64()) {
try dirs_to_search.append(comptime arm_brew_prefix ++ "/opt/icu4c/lib/");
} else {
try dirs_to_search.append(comptime x86_brew_prefix ++ "/opt/icu4c/lib/");
}
}

if (b.env_map.get("JSC_LIB")) |jsc| {
try dirs_to_search.append(jsc);
}

var added = std.AutoHashMap(u64, void).init(b.allocator);

const files_we_care_about = std.ComptimeStringMap([]const u8, .{
.{ "libmimalloc.o", "libmimalloc.o" },
.{ "libz.a", "libz.a" },
.{ "libarchive.a", "libarchive.a" },
.{ "libssl.a", "libssl.a" },
.{ "picohttpparser.o", "picohttpparser.o" },
.{ "libcrypto.boring.a", "libcrypto.boring.a" },
.{ "libicuuc.a", "libicuuc.a" },
.{ "libicudata.a", "libicudata.a" },
.{ "libicui18n.a", "libicui18n.a" },
.{ "libJavaScriptCore.a", "libJavaScriptCore.a" },
.{ "libWTF.a", "libWTF.a" },
.{ "libbmalloc.a", "libbmalloc.a" },
.{ "liblolhtml.a", "liblolhtml.a" },
.{ "uSockets.a", "uSockets.a" },
});

for (dirs_to_search.slice()) |deps_path| {
var deps_dir = std.fs.cwd().openIterableDir(deps_path, .{}) catch continue;
var iterator = deps_dir.iterate();
obj.addIncludePath(deps_path);
obj.addLibraryPath(deps_path);

while (iterator.next() catch null) |entr| {
const entry: std.fs.IterableDir.Entry = entr;
if (files_we_care_about.get(entry.name)) |obj_name| {
var has_added = try added.getOrPut(std.hash.Wyhash.hash(0, obj_name));
if (!has_added.found_existing) {
var paths = [_][]const u8{ deps_path, obj_name };
obj.addObjectFile(try std.fs.path.join(b.allocator, &paths));
}
}
}
}
}

pub fn configureObjectStep(b: *std.build.Builder, obj: *CompileStep, comptime Target: type, target: Target, main_pkg_path: []const u8) !void {
obj.setMainPkgPath(main_pkg_path);

Expand Down
2 changes: 1 addition & 1 deletion docs/bundler/migration.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ In Bun's CLI, simple boolean flags like `--minify` do not accept an argument. Ot

- `--bundle`
- n/a
- Bun always bundles, use `--transpile` to disable this behavior.
- Bun always bundles, use `--no-bundle` to disable this behavior.

---

Expand Down
5 changes: 5 additions & 0 deletions docs/project/licensing.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ Bun statically links these libraries:

---

- [`zstd`](https://github.com/facebook/zstd)
- dual-licensed under the BSD License or GPLv2 license

---

- [`simdutf`](https://github.com/simdutf/simdutf)
- Apache 2.0

Expand Down
Loading

0 comments on commit 893f70f

Please sign in to comment.