diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc18df32f6b351..84394f6548ca2e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,33 +47,17 @@ TODO: document this (see [`bindings.zig`](src/bun.js/bindings/bindings.zig) and Copy from examples like `Subprocess` or `Response`. -### ESM modules +### ESM Modules and Builtins JS Bun implements ESM modules in a mix of native code and JavaScript. Several Node.js modules are implemented in JavaScript and loosely based on browserify polyfills. -The ESM modules in Bun are located in [`src/bun.js/*.exports.js`](src/bun.js/). Unlike other code in Bun, these files are NOT transpiled. They are loaded directly into the JavaScriptCore VM. That means `require` does not work in these files. Instead, you must use `import.meta.require`, or ideally, not use require/import other files at all. +Builtin modules in Bun are located in [`src/js`](src/js/). These files are transpiled and support a JavaScriptCore-only syntax for internal slots, which is explained further in [`src/js/README.md`](src/js/README.md). -The module loader is in [`src/bun.js/module_loader.zig`](src/bun.js/module_loader.zig). - -### JavaScript Builtins - -TODO: update this with the new build process that uses TypeScript and `$` instead of `@`. - -JavaScript builtins are located in [`src/js/builtins/*.ts`](src/js/builtins). - -These files support a JavaScriptCore-only syntax for internal slots. `@` is used to access an internal slot. For example: `new @Array(123)` will create a new `Array` similar to `new Array(123)`, except if a library modifies the `Array` global, it will not affect the internal slot (`@Array`). These names must be allow-listed in `BunBuiltinNames.h` (though JavaScriptCore allowlists some names by default). +Native C++ modules are in `src/bun.js/modules/`. -They can not use or reference ESM-modules. The files that end with `*Internals.js` are automatically loaded globally. Most usage of internals right now are the stream implementations (which share a lot of code from Safari/WebKit) and ImportMetaObject (which is how `require` is implemented in the runtime) - -To regenerate the builtins: - -```sh -make clean-bindings && make generate-builtins && make bindings -j10 -``` - -It is recommended that you have ccache installed or else you will spend a lot of time waiting for the bindings to compile. +The module loader is in [`src/bun.js/module_loader.zig`](src/bun.js/module_loader.zig). ### Memory management in Bun's JavaScript runtime diff --git a/Makefile b/Makefile index 070d03ec46e5c8..96fd51ee6bf93a 100644 --- a/Makefile +++ b/Makefile @@ -555,7 +555,7 @@ tinycc: PYTHON=$(shell which python 2>/dev/null || which python3 2>/dev/null || which python2 2>/dev/null) .PHONY: esm -js: +js: # to rebundle js (rebuilding binary not needed to reload js code) NODE_ENV=production bun src/js/_codegen/index.ts esm-debug: @@ -660,8 +660,8 @@ else PKGNAME_NINJA := ninja-build endif -.PHONY: require -require: +.PHONY: assert-deps +assert-deps: @echo "Checking if the required utilities are available..." @if [ $(CLANG_VERSION) -lt "15" ]; then echo -e "ERROR: clang version >=15 required, found: $(CLANG_VERSION). Install with:\n\n $(POSIX_PKG_MANAGER) install llvm@15"; exit 1; fi @cmake --version >/dev/null 2>&1 || (echo -e "ERROR: cmake is required."; exit 1) @@ -673,6 +673,9 @@ require: @which $(LIBTOOL) > /dev/null || (echo -e "ERROR: libtool is required. Install with:\n\n $(POSIX_PKG_MANAGER) install libtool"; exit 1) @which ninja > /dev/null || (echo -e "ERROR: Ninja is required. Install with:\n\n $(POSIX_PKG_MANAGER) install $(PKGNAME_NINJA)"; exit 1) @which pkg-config > /dev/null || (echo -e "ERROR: pkg-config is required. Install with:\n\n $(POSIX_PKG_MANAGER) install pkg-config"; exit 1) + @which rustc > /dev/null || (echo -e "ERROR: rustc is required." exit 1) + @which cargo > /dev/null || (echo -e "ERROR: cargo is required." exit 1) + @test $(shell cargo --version | awk '{print $$2}' | cut -d. -f2) -gt 57 || (echo -e "ERROR: cargo version must be at least 1.57."; exit 1) @echo "You have the dependencies installed! Woo" # the following allows you to run `make submodule` to update or init submodules. but we will exclude webkit @@ -1106,9 +1109,6 @@ endif dev-obj-linux: $(ZIG) build obj -Dtarget=x86_64-linux-gnu -Dcpu="$(CPU_TARGET)" -.PHONY: dev -dev: mkdir-dev dev-obj link ## compile zig changes + link bun - mkdir-dev: mkdir -p $(DEBUG_PACKAGE_DIR) @@ -1898,26 +1898,44 @@ vendor-without-npm: node-fallbacks runtime_js fallback_decoder bun_error mimallo vendor-without-check: npm-install vendor-without-npm .PHONY: vendor -vendor: require submodule vendor-without-check +vendor: assert-deps submodule vendor-without-check .PHONY: vendor-dev -vendor-dev: require submodule npm-install-dev vendor-without-npm +vendor-dev: assert-deps submodule npm-install-dev vendor-without-npm .PHONY: bun bun: vendor identifier-cache build-obj bun-link-lld-release bun-codesign-release-local -.PHONY: regenerate-bindings -regenerate-bindings: ## compile src/js/builtins + all c++ code, does not link +.PHONY: cpp +cpp: ## compile src/js/builtins + all c++ code then link + @make clean-bindings js + @make bindings -j$(CPU_COUNT) + @make link + +.PHONY: cpp +cpp-no-link: @make clean-bindings js @make bindings -j$(CPU_COUNT) +.PHONY: zig +zig: ## compile zig code then link + @make mkdir-dev dev-obj link + +.PHONY: zig-no-link +zig-no-link: + @make mkdir-dev dev-obj + +.PHONY: dev +dev: # combo of `make cpp` and `make zig` + @make cpp-no-link zig-no-link -j2 + @make link + .PHONY: setup -setup: vendor-dev identifier-cache clean-bindings js - make jsc-check - make bindings -j$(CPU_COUNT) +setup: vendor-dev identifier-cache clean-bindings + make jsc-check cpp zig link @echo "" - @echo "Development environment setup complete" - @echo "Run \`make dev\` to build \`bun-debug\`" + @echo "First build complete!" + @echo "\"bun-debug\" is available at $(DEBUG_BIN)/bun-debug" @echo "" .PHONY: help diff --git a/docs/api/workers.md b/docs/api/workers.md index ba45d7cc19e098..73a92553195918 100644 --- a/docs/api/workers.md +++ b/docs/api/workers.md @@ -88,7 +88,7 @@ worker.addEventListener("message", event => { ## Terminating a worker -A `Worker` instance terminate automatically when Bun's process exits. To terminate a `Worker` sooner, call `worker.terminate()`. +A `Worker` instance terminates automatically once it's event loop has no work left to do. Attaching a `"message"` listener on the global or any `MessagePort`s will keep the event loop alive. To forcefully terminate a `Worker`, call `worker.terminate()`. ```ts const worker = new Worker(new URL("worker.ts", import.meta.url).href); @@ -97,18 +97,20 @@ const worker = new Worker(new URL("worker.ts", import.meta.url).href); worker.terminate(); ``` +This will cause the worker's to exit as soon as possible. + ### `process.exit()` -A worker can terminate itself with `process.exit()`. This does not terminate the main process. Like in Node.js, `process.on('beforeExit', callback)` and `process.on('exit', callback)` are emitted on the worker thread (and not on the main thread). +A worker can terminate itself with `process.exit()`. This does not terminate the main process. Like in Node.js, `process.on('beforeExit', callback)` and `process.on('exit', callback)` are emitted on the worker thread (and not on the main thread), and the exit code is passed to the `"close"` event. ### `"close"` -The `"close"` event is emitted when a worker has been terminated. It can take some time for the worker to actually terminate, so this event is emitted when the worker has been marked as terminated. +The `"close"` event is emitted when a worker has been terminated. It can take some time for the worker to actually terminate, so this event is emitted when the worker has been marked as terminated. The `CloseEvent` will contain the exit code passed to `process.exit()`, or 0 if closed for other reasons. ```ts const worker = new Worker(new URL("worker.ts", import.meta.url).href); -worker.addEventListener("close", () => { +worker.addEventListener("close", event => { console.log("worker is being closed"); }); ``` @@ -117,14 +119,27 @@ This event does not exist in browsers. ## Managing lifetime -By default, an active `Worker` will _not_ keep the main (spawning) process alive. Once the main script finishes, the main thread will terminate, shutting down any workers it created. +By default, an active `Worker` will keep the main (spawning) process alive, so async tasks like `setTimeout` and promises will keep the process alive. Attaching `message` listeners will also keep the `Worker` alive. + +### `worker.unref` + +To stop a running worker from keeping the process alive, call `worker.unref()`. This decouples the lifetime of the worker to the lifetime of the main process, and is equivlent to what Node.js' `worker_threads` does. + +```ts +const worker = new Worker(new URL("worker.ts", import.meta.url).href); +worker.unref(); +``` + +Note: `worker.unref()` is not available in browers. ### `worker.ref` -To keep the process alive until the `Worker` terminates, call `worker.ref()`. This couples the lifetime of the worker to the lifetime of the main process. +To keep the process alive until the `Worker` terminates, call `worker.ref()`. A ref'd worker is the default behavior, and still needs something going on in the event loop (such as a `"message"` listener) for the worker to continue running. ```ts const worker = new Worker(new URL("worker.ts", import.meta.url).href); +worker.unref(); +// later... worker.ref(); ``` @@ -132,22 +147,11 @@ Alternatively, you can also pass an `options` object to `Worker`: ```ts const worker = new Worker(new URL("worker.ts", import.meta.url).href, { - ref: true, + ref: false, }); ``` -### `worker.unref` - -To stop keeping the process alive, call `worker.unref()`. - -```ts -const worker = new Worker(new URL("worker.ts", import.meta.url).href); -worker.ref(); -// ...later on -worker.unref(); -``` - -Note: `worker.ref()` and `worker.unref()` do not exist in browsers. +Note: `worker.ref()` is not available in browers. ## Memory usage with `smol` diff --git a/docs/project/development.md b/docs/project/development.md index 2a3630a124ede3..c635595b348bca 100644 --- a/docs/project/development.md +++ b/docs/project/development.md @@ -115,18 +115,16 @@ $ bun install -g @oven/zig $ zigup 0.11.0-dev.4006+bf827d0b5 ``` -## Building - -After cloning the repository, run the following command. The runs +{% callout %} +We last updated Zig on **July 18th, 2023** +{% /callout %} -```bash -$ make setup -``` +## First Build -Then to build Bun: +After cloning the repository, run the following command to run the first build. This may take a while as it will clone submodules and build dependencies. ```bash -$ make dev +$ make setup ``` The binary will be located at `packages/debug-bun-{platform}-{arch}/bun-debug`. It is recommended to add this to your `$PATH`. To verify the build worked, lets print the version number on the development build of Bun. @@ -136,16 +134,78 @@ $ packages/debug-bun-*/bun-debug --version bun 0.x.y__dev ``` +Note: `make setup` is just an alias for the following: + +```bash +$ make assert-deps submodule npm-install-dev node-fallbacks runtime_js fallback_decoder bun_error mimalloc picohttp zlib boringssl libarchive lolhtml sqlite usockets uws tinycc c-ares zstd base64 cpp zig link +``` + +## Rebuilding + +Bun uses a series of make commands to rebuild parts of the codebase. The general rule for rebuilding is there is `make link` to rerun the linker, and then different make targets for different parts of the codebase. Do not pass `-j` to make as these scripts will break if run out of order, and multiple cores will be used when possible during the builds. + +| What changed | Run this command | +| ------------------------------------ | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| Zig Code | `make zig` | +| C++ Code | `make cpp` | +| Zig + C++ Code | `make dev` (combination of the above two) | +| JS/TS Code in `src/js` | `make js` (in bun-debug, js is loaded from disk without a recompile). If you change the names of any file or add/remove anything, you must also run `make dev`. | +| `*.classes.ts` | `make generate-classes dev` | +| JSSink | `make generate-sink cpp` | +| `src/node_fallbacks/*` | `make node-fallbacks zig` | +| `identifier_data.zig` | `make identifier-cache zig` | +| Code using `cppFn`/`JSC.markBinding` | `make headers` (TODO: explain explain what this is used for and why it's useful) | + +`make setup` cloned a bunch of submodules and built the subprojects. When a submodule is out of date, run `make submodule` to quickly reset/update all your submodules, then you can rebuild individual submodules with their respective command. + +| Dependency | Run this command | +| -------------- | ---------------------------------------- | +| WebKit | `bun install` (it is a prebuilt package) | +| uWebSockets | `make uws` | +| Mimalloc | `make mimalloc` | +| PicoHTTPParser | `make picohttp` | +| zlib | `make zlib` | +| BoringSSL | `make boringssl` | +| libarchive | `make libarchive` | +| lolhtml | `make lolhtml` | +| sqlite | `make sqlite` | +| TinyCC | `make tinycc` | +| c-ares | `make c-ares` | +| zstd | `make zstd` | +| Base64 | `make base64` | + +The above will probably also need Zig and/or C++ code rebuilt. + ## VSCode VSCode is the recommended IDE for working on Bun, as it has been configured. Once opening, you can run `Extensions: Show Recommended Extensions` to install the recommended extensions for Zig and C++. ZLS is automatically configured. +### ZLS + +ZLS is the language server for Zig. The latest binary that the extension auto-updates may not function with the version of Zig that Bun uses. It may be more reliable to build ZLS from source: + +```bash +$ git clone https://github.com/zigtools/zls +$ cd zls +$ git checkout f91ff831f4959efcb7e648dba4f0132c296d26c0 +$ zig build +``` + +Then add absolute paths to Zig and ZLS in your vscode config: + +```json +{ + "zig.zigPath": "/path/to/zig/install/zig", + "zig.zls.path": "/path/to/zls/zig-out/bin/zls" +} +``` + ## JavaScript builtins When you change anything in `src/js/builtins/*` or switch branches, run this: ```bash -$ make regenerate-bindings +$ make js cpp ``` That inlines the TypeScript code into C++ headers. @@ -154,6 +214,8 @@ That inlines the TypeScript code into C++ headers. Make sure you have `ccache` installed, otherwise regeneration will take much longer than it should. {% /callout %} +For more information on how `src/js` works, see `src/js/README.md` in the codebase. + ## Code generation scripts Bun leverages a lot of code generation scripts. @@ -193,7 +255,7 @@ Certain modules like `node:fs`, `node:stream`, `bun:sqlite`, and `ws` are implem When these are changed, run: ``` -$ make esm +$ make js ``` In debug builds, Bun automatically loads these from the filesystem, wherever it was compiled, so no need to re-run `make dev`. In release builds, this same behavior can be done via the environment variable `BUN_OVERRIDE_MODULE_PATH`. When set to the repository root, Bun will read from the bundled modules in the repository instead of the ones baked into the binary. @@ -244,7 +306,7 @@ For performance reasons, `make submodule` does not automatically update the WebK ```bash $ bun install -$ make regenerate-bindings +$ make cpp ```