perf: use a worker for hard linking directories

pnpm · zkochan · Oct 19, 2023 · Oct 1, 2023 · Oct 1, 2023 · Oct 1, 2023
commit 77e8624fa8b1a419a4acfd17752bfc099f45c84b
@@ -1,70 +1,62 @@
 import path from 'path'
-import { promises as fs } from 'fs'
+import fs from 'fs'
 import { globalWarn } from '@pnpm/logger'
 
-export async function hardLinkDir (src: string, destDirs: string[]) {
+export function hardLinkDir (src: string, destDirs: string[]) {
   if (destDirs.length === 0) return
   // Don't try to hard link the source directory to itself
   destDirs = destDirs.filter((destDir) => path.relative(destDir, src) !== '')
-  await _hardLinkDir(src, destDirs, true)
+  _hardLinkDir(src, destDirs, true)
 }
 
-async function _hardLinkDir (src: string, destDirs: string[], isRoot?: boolean) {
+function _hardLinkDir (src: string, destDirs: string[], isRoot?: boolean) {
   let files: string[] = []
   try {
-    files = await fs.readdir(src)
+    files = fs.readdirSync(src)
   } catch (err: any) { // eslint-disable-line
     if (!isRoot || err.code !== 'ENOENT') throw err
     globalWarn(`Source directory not found when creating hardLinks for: ${src}. Creating destinations as empty: ${destDirs.join(', ')}`)
-    await Promise.all(
-      destDirs.map((dir) => fs.mkdir(dir, { recursive: true }))
-    )
+    destDirs.map((dir) => fs.mkdirSync(dir, { recursive: true }))
     return
   }
-  await Promise.all(
-    files.map(async (file) => {
-      if (file === 'node_modules') return
-      const srcFile = path.join(src, file)
-      if ((await fs.lstat(srcFile)).isDirectory()) {
-        const destSubdirs = await Promise.all(
-          destDirs.map(async (destDir) => {
-            const destSubdir = path.join(destDir, file)
-            try {
-              await fs.mkdir(destSubdir, { recursive: true })
-            } catch (err: any) { // eslint-disable-line
-              if (err.code !== 'EEXIST') throw err
-            }
-            return destSubdir
-          })
-        )
-        await _hardLinkDir(srcFile, destSubdirs)
-        return
+  for (const file of files) {
+    if (file === 'node_modules') continue
+    const srcFile = path.join(src, file)
+    if ((fs.lstatSync(srcFile)).isDirectory()) {
+      const destSubdirs = destDirs.map((destDir) => {
+        const destSubdir = path.join(destDir, file)
+        try {
+          fs.mkdirSync(destSubdir, { recursive: true })
+        } catch (err: any) { // eslint-disable-line
+          if (err.code !== 'EEXIST') throw err
+        }
+        return destSubdir
+      })
+      _hardLinkDir(srcFile, destSubdirs)
+      continue
+    }
+    for (const destDir of destDirs) {
+      const destFile = path.join(destDir, file)
+      try {
+        linkOrCopyFile(srcFile, destFile)
+      } catch (err: any) { // eslint-disable-line
+        if (err.code === 'ENOENT') {
+          // Ignore broken symlinks
+          continue
+        }
+        throw err
       }
-      await Promise.all(
-        destDirs.map(async (destDir) => {
-          const destFile = path.join(destDir, file)
-          try {
-            await linkOrCopyFile(srcFile, destFile)
-          } catch (err: any) { // eslint-disable-line
-            if (err.code === 'ENOENT') {
-              // Ignore broken symlinks
-              return
-            }
-            throw err
-          }
-        })
-      )
-    })
-  )
+    }
+  }
 }
 
-async function linkOrCopyFile (srcFile: string, destFile: string) {
+function linkOrCopyFile (srcFile: string, destFile: string) {
   try {
-    await linkOrCopy(srcFile, destFile)
+    linkOrCopy(srcFile, destFile)
   } catch (err: any) { // eslint-disable-line
     if (err.code === 'ENOENT') {
-      await fs.mkdir(path.dirname(destFile), { recursive: true })
-      await linkOrCopy(srcFile, destFile)
+      fs.mkdirSync(path.dirname(destFile), { recursive: true })
+      linkOrCopy(srcFile, destFile)
       return
     }
     if (err.code !== 'EEXIST') {
@@ -77,11 +69,11 @@ async function linkOrCopyFile (srcFile: string, destFile: string) {
  * This function could be optimized because we don't really need to try linking again
  * if linking failed once.
  */
-async function linkOrCopy (srcFile: string, destFile: string) {
+function linkOrCopy (srcFile: string, destFile: string) {
   try {
-    await fs.link(srcFile, destFile)
+    fs.linkSync(srcFile, destFile)
   } catch (err: any) { // eslint-disable-line
     if (err.code !== 'EXDEV') throw err
-    await fs.copyFile(srcFile, destFile)
+    fs.copyFileSync(srcFile, destFile)
   }
 }
@@ -3,7 +3,7 @@ import path from 'path'
 import { tempDir as createTempDir } from '@pnpm/prepare'
 import { hardLinkDir } from '@pnpm/fs.hard-link-dir'
 
-test('hardLinkDirectory()', async () => {
+test('hardLinkDirectory()', () => {
   const tempDir = createTempDir()
   const srcDir = path.join(tempDir, 'source')
   const dest1Dir = path.join(tempDir, 'dest1')
@@ -18,7 +18,7 @@ test('hardLinkDirectory()', async () => {
   fs.writeFileSync(path.join(srcDir, 'subdir/file.txt'), 'Hello World')
   fs.writeFileSync(path.join(srcDir, 'node_modules/file.txt'), 'Hello World')
 
-  await hardLinkDir(srcDir, [dest1Dir, dest2Dir])
+  hardLinkDir(srcDir, [dest1Dir, dest2Dir])
 
   // It should link the files from the root
   expect(fs.readFileSync(path.join(dest1Dir, 'file.txt'), 'utf8')).toBe('Hello World')
@@ -33,12 +33,12 @@ test('hardLinkDirectory()', async () => {
   expect(fs.existsSync(path.join(dest2Dir, 'node_modules/file.txt'))).toBe(false)
 })
 
-test("don't fail on missing source and dest directories", async () => {
+test("don't fail on missing source and dest directories", () => {
   const tempDir = createTempDir()
   const missingDirSrc = path.join(tempDir, 'missing_source')
   const missingDirDest = path.join(tempDir, 'missing_dest')
 
-  await hardLinkDir(missingDirSrc, [missingDirDest])
+  hardLinkDir(missingDirSrc, [missingDirDest])
 
   // It should create an empty dest dir if src does not exist
   expect(fs.existsSync(missingDirSrc)).toBe(false)

@@ -35,9 +35,10 @@
     "@pnpm/cafs-types": "workspace:*",
     "@pnpm/create-cafs-store": "workspace:*",
     "@pnpm/error": "workspace:*",
+    "@pnpm/fs.hard-link-dir": "workspace:*",
     "@pnpm/graceful-fs": "workspace:*",
-    "@pnpm/symlink-dependency": "workspace:*",
     "@pnpm/store.cafs": "workspace:*",
+    "@pnpm/symlink-dependency": "workspace:*",
     "@rushstack/worker-pool": "0.3.34",
     "load-json-file": "^6.2.0"
   },

@@ -9,6 +9,7 @@ import {
   type AddDirToStoreMessage,
   type LinkPkgMessage,
   type SymlinkAllModulesMessage,
+  HardLinkDirMessage,
 } from './types'
 
 let workerPool: WorkerPool | undefined
@@ -220,3 +221,25 @@ export async function symlinkAllModules (
     } as SymlinkAllModulesMessage)
   })
 }
+
+export async function hardLinkDir (src: string, destDirs: string[]): Promise<void> {
+  if (!workerPool) {
+    workerPool = createTarballWorkerPool()
+  }
+  const localWorker = await workerPool.checkoutWorkerAsync(true)
+  await new Promise<void>((resolve, reject) => {
+    localWorker.once('message', ({ status, error }: any) => { // eslint-disable-line @typescript-eslint/no-explicit-any
+      workerPool!.checkinWorker(localWorker)
+      if (status === 'error') {
+        reject(new PnpmError('HARDLINK_FAILED', error as string))
+        return
+      }
+      resolve()
+    })
+    localWorker.postMessage({
+      type: 'hardLinkDir',
+      src,
+      destDirs
+    } as HardLinkDirMessage)
+  })
+}
@@ -54,3 +54,9 @@ export interface ReadPkgFromCafsMessage {
   readManifest: boolean
   verifyStoreIntegrity: boolean
 }
+
+export interface HardLinkDirMessage {
+  type: 'hardLinkDir'
+  src: string
+  destDirs: string[]
+}
@@ -3,6 +3,7 @@ import fs from 'fs'
 import gfs from '@pnpm/graceful-fs'
 import * as crypto from 'crypto'
 import { createCafsStore } from '@pnpm/create-cafs-store'
+import { hardLinkDir } from '@pnpm/fs.hard-link-dir'
 import {
   checkPkgFilesIntegrity,
   createCafs,
@@ -22,6 +23,7 @@ import {
   type LinkPkgMessage,
   type SymlinkAllModulesMessage,
   type TarballExtractMessage,
+  HardLinkDirMessage,
 } from './types'
 
 const INTEGRITY_REGEX: RegExp = /^([^-]+)-([A-Za-z0-9+/=]+)$/
@@ -33,7 +35,7 @@ const cafsStoreCache = new Map<string, ReturnType<typeof createCafsStore>>()
 const cafsLocker = new Map<string, number>()
 
 async function handleMessage (
-  message: TarballExtractMessage | LinkPkgMessage | AddDirToStoreMessage | ReadPkgFromCafsMessage | SymlinkAllModulesMessage | false
+  message: TarballExtractMessage | LinkPkgMessage | AddDirToStoreMessage | ReadPkgFromCafsMessage | SymlinkAllModulesMessage | HardLinkDirMessage | false
 ): Promise<void> {
   if (message === false) {
     parentPort!.off('message', handleMessage)
@@ -94,6 +96,11 @@ async function handleMessage (
       parentPort!.postMessage(symlinkAllModules(message))
       break
     }
+    case 'hardLinkDir': {
+      hardLinkDir(message.src, message.destDirs)
+      parentPort!.postMessage({ status: 'success' })
+      break
+    }
     }
   } catch (e: any) { // eslint-disable-line
     parentPort!.postMessage({ status: 'error', error: e.toString() })
@@ -235,3 +242,4 @@ function writeJsonFile (filePath: string, data: unknown) {
 process.on('uncaughtException', (err) => {
   console.error(err)
 })
+