From 93605ab267564f8de6769411797fc6e0b0b9431b Mon Sep 17 00:00:00 2001 From: Richard Lau Date: Mon, 29 Jul 2024 16:02:58 +0100 Subject: [PATCH 1/2] fix: restore externalized Node.js dep compatibility (#3421) * fix: restore externalized Node.js dep compatibility Restore the ability to build Undici compatible with Node.js' `configure --shared-builtin-undici/undici-path ...` build option. Scopes the `hasApk` conditional to only cover the part that requires `apk`. Makes the WASM optimizer (binaryen) optional to allow building on Linux distributions that do not package `binaryen` and must be able to rebuild everything (including tooling) from source. * ci: add workflow for externalized Node.js dep Add a workflow to test building Undici in a way compatible with Node.js built with `configure --shared-builtin-undici/undici-path ...`. This configuration is used by downstream Node.js packagers (e.g. Fedora) who require the ability to be able to build everything from source. --- .github/workflows/nodejs-shared.yml | 102 ++++++++++++++++++++++++++++ CONTRIBUTING.md | 3 +- build/wasm.js | 48 +++++++++---- 3 files changed, 138 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/nodejs-shared.yml diff --git a/.github/workflows/nodejs-shared.yml b/.github/workflows/nodejs-shared.yml new file mode 100644 index 00000000000..c13a793362e --- /dev/null +++ b/.github/workflows/nodejs-shared.yml @@ -0,0 +1,102 @@ +name: Node.js compiled --shared-builtin-undici/undici-path CI + +on: + push: + branches: + - main + - current + - next + - 'v*' + pull_request: + +permissions: + contents: read + +jobs: + test-shared-builtin: + name: Test with Node.js ${{ matrix.version }} compiled --shared-builtin-undici/undici-path + strategy: + fail-fast: false + max-parallel: 0 + matrix: + version: [20, 22] + runs-on: ubuntu-latest + timeout-minutes: 120 + steps: + # Checkout into a subdirectory otherwise Node.js tests will break due to finding Undici's package.json in a parent directory. + - name: Checkout + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 # v4.1.2 + with: + path: ./undici + persist-credentials: false + + # Setup node, install deps, and build undici prior to building node with `--shared-builtin-undici/undici-path` and testing + - name: Setup Node.js@${{ inputs.version }} + uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2 + with: + node-version: ${{ inputs.version }} + + - name: Install dependencies + working-directory: ./undici + run: npm install + + - name: Install wasi-libc + run: sudo apt-get install -y wasi-libc + + - name: Build WASM + working-directory: ./undici + run: | + export EXTERNAL_PATH=${{ github.workspace }}/undici + export WASM_CC=clang + export WASM_CFLAGS='--target=wasm32-wasi --sysroot=/usr' + export WASM_LDFLAGS='-nodefaultlibs' + export WASM_LDLIBS='-lc' + node build/wasm.js + + - name: Determine latest release + id: release + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + result-encoding: string + script: | + const req = await fetch('https://nodejs.org/download/release/index.json') + const releases = await req.json() + + const latest = releases.find((r) => r.version.startsWith('v${{ matrix.version }}')) + return latest.version + + - name: Download and extract source + run: curl https://nodejs.org/download/release/${{ steps.release.outputs.result }}/node-${{ steps.release.outputs.result }}.tar.xz | tar xfJ - + + - name: Install ninja + run: sudo apt-get install ninja-build + + - name: ccache + uses: hendrikmuhs/ccache-action@c92f40bee50034e84c763e33b317c77adaa81c92 #v1.2.13 + with: + key: node(external_undici)${{ matrix.version }} + + - name: Build node + working-directory: ./node-${{ steps.release.outputs.result }} + run: | + export CC="ccache gcc" + export CXX="ccache g++" + rm -rf deps/undici + ./configure --shared-builtin-undici/undici-path ${{ github.workspace }}/undici/loader.js --ninja --prefix=./final + make + make install + echo "$(pwd)/final/bin" >> $GITHUB_PATH + + - name: Print version information + run: | + echo OS: $(node -p "os.version()") + echo Node.js: $(node --version) + echo npm: $(npm --version) + echo git: $(git --version) + echo external config: $(node -e "console.log(process.config)" | grep NODE_SHARED_BUILTIN_UNDICI_UNDICI_PATH) + echo Node.js built-in undici version: $(node -p "process.versions.undici") # undefined for external Undici + + - name: Run tests + working-directory: ./node-${{ steps.release.outputs.result }} + run: tools/test.py -p dots --flaky-tests=dontcare + diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index febb3bcae02..6b6c01d2264 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -157,7 +157,8 @@ If you are packaging `undici` for a distro, this might help if you would like to an unbundled version instead of bundling one in `libnode.so`. To enable this, pass `EXTERNAL_PATH=/path/to/global/node_modules/undici` to `build/wasm.js`. -You shall also pass this path to `--shared-builtin-undici/undici-path` in Node.js's `configure.py`. +Pass this path with `loader.js` appended to `--shared-builtin-undici/undici-path` in Node.js's `configure.py`. +If building on a non-Alpine Linux distribution, you may need to also set the `WASM_CC`, `WASM_CFLAGS`, `WASM_LDFLAGS` and `WASM_LDLIBS` environment variables before running `build/wasm.js`. ### Benchmarks diff --git a/build/wasm.js b/build/wasm.js index 9c88427874a..0e86e11f195 100644 --- a/build/wasm.js +++ b/build/wasm.js @@ -15,6 +15,9 @@ let WASM_CFLAGS = process.env.WASM_CFLAGS || '--sysroot=/usr/share/wasi-sysroot let WASM_LDFLAGS = process.env.WASM_LDFLAGS || '' const WASM_LDLIBS = process.env.WASM_LDLIBS || '' +// For compatibility with Node.js' `configure --shared-builtin-undici/undici-path ...` +const EXTERNAL_PATH = process.env.EXTERNAL_PATH + // These are relevant for undici and should not be overridden WASM_CFLAGS += ' -Ofast -fno-exceptions -fvisibility=hidden -mexec-model=reactor' WASM_LDFLAGS += ' -Wl,-error-limit=0 -Wl,-O3 -Wl,--lto-O3 -Wl,--strip-all' @@ -73,6 +76,9 @@ if (process.argv[2] === '--docker') { const hasApk = (function () { try { execSync('command -v apk'); return true } catch (error) { return false } })() +const hasOptimizer = (function () { + try { execSync('./wasm-opt --version'); return true } catch (error) { return false } +})() if (hasApk) { // Gather information about the tools used for the build const buildInfo = execSync('apk info -v').toString() @@ -81,24 +87,38 @@ if (hasApk) { process.exit(-1) } console.log(buildInfo) +} - // Build wasm binary - execSync(`${WASM_CC} ${WASM_CFLAGS} ${WASM_LDFLAGS} \ - ${join(WASM_SRC, 'src')}/*.c \ - -I${join(WASM_SRC, 'include')} \ - -o ${join(WASM_OUT, 'llhttp.wasm')} \ - ${WASM_LDLIBS}`, { stdio: 'inherit' }) +// Build wasm binary +execSync(`${WASM_CC} ${WASM_CFLAGS} ${WASM_LDFLAGS} \ +${join(WASM_SRC, 'src')}/*.c \ +-I${join(WASM_SRC, 'include')} \ +-o ${join(WASM_OUT, 'llhttp.wasm')} \ +${WASM_LDLIBS}`, { stdio: 'inherit' }) +if (hasOptimizer) { execSync(`./wasm-opt ${WASM_OPT_FLAGS} -o ${join(WASM_OUT, 'llhttp.wasm')} ${join(WASM_OUT, 'llhttp.wasm')}`, { stdio: 'inherit' }) - writeWasmChunk('llhttp.wasm', 'llhttp-wasm.js') +} +writeWasmChunk('llhttp.wasm', 'llhttp-wasm.js') - // Build wasm simd binary - execSync(`${WASM_CC} ${WASM_CFLAGS} -msimd128 ${WASM_LDFLAGS} \ - ${join(WASM_SRC, 'src')}/*.c \ - -I${join(WASM_SRC, 'include')} \ - -o ${join(WASM_OUT, 'llhttp_simd.wasm')} \ - ${WASM_LDLIBS}`, { stdio: 'inherit' }) +// Build wasm simd binary +execSync(`${WASM_CC} ${WASM_CFLAGS} -msimd128 ${WASM_LDFLAGS} \ +${join(WASM_SRC, 'src')}/*.c \ +-I${join(WASM_SRC, 'include')} \ +-o ${join(WASM_OUT, 'llhttp_simd.wasm')} \ +${WASM_LDLIBS}`, { stdio: 'inherit' }) +if (hasOptimizer) { execSync(`./wasm-opt ${WASM_OPT_FLAGS} --enable-simd -o ${join(WASM_OUT, 'llhttp_simd.wasm')} ${join(WASM_OUT, 'llhttp_simd.wasm')}`, { stdio: 'inherit' }) - writeWasmChunk('llhttp_simd.wasm', 'llhttp_simd-wasm.js') +} +writeWasmChunk('llhttp_simd.wasm', 'llhttp_simd-wasm.js') + +// For compatibility with Node.js' `configure --shared-builtin-undici/undici-path ...` +if (EXTERNAL_PATH) { + writeFileSync(join(ROOT, 'loader.js'), ` +'use strict' +globalThis.__UNDICI_IS_NODE__ = true +module.exports = require('node:module').createRequire('${EXTERNAL_PATH}/loader.js')('./index-fetch.js') +delete globalThis.__UNDICI_IS_NODE__ +`) } From 8499c4bbdc9dcff3a5ce36fac41b4e9497814a16 Mon Sep 17 00:00:00 2001 From: Matteo Collina Date: Wed, 31 Jul 2024 12:32:26 +0200 Subject: [PATCH 2/2] Bumped v6.19.5 Signed-off-by: Matteo Collina --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 60679e07fa9..88aa14df839 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "undici", - "version": "6.19.4", + "version": "6.19.5", "description": "An HTTP/1.1 client, written from scratch for Node.js", "homepage": "https://undici.nodejs.org", "bugs": {