perf: faster repeat install (#8838)

pnpm · Dec 8, 2024 · d47c426 · d47c426
1 parent 6483b64
commit d47c426
Show file tree

Hide file tree

Showing 45 changed files with 587 additions and 277 deletions.
diff --git a/.changeset/plenty-mayflies-float.md b/.changeset/plenty-mayflies-float.md
@@ -0,0 +1,9 @@
+---
+"@pnpm/plugin-commands-installation": minor
+"pnpm": minor
+"@pnpm/pnpmfile": minor
+"@pnpm/workspace.state": major
+"@pnpm/deps.status": major
+---
+
+On repeat install perform a fast check if `node_modules` is up to date [#8838](https://github.com/pnpm/pnpm/pull/8838).
diff --git a/deps/status/package.json b/deps/status/package.json
@@ -40,6 +40,7 @@
     "@pnpm/lockfile.settings-checker": "workspace:*",
     "@pnpm/lockfile.verification": "workspace:*",
     "@pnpm/parse-overrides": "workspace:*",
+    "@pnpm/pnpmfile": "workspace:*",
     "@pnpm/resolver-base": "workspace:*",
     "@pnpm/types": "workspace:*",
     "@pnpm/workspace.find-packages": "workspace:*",

diff --git a/deps/status/src/checkDepsStatus.ts b/deps/status/src/checkDepsStatus.ts
@@ -28,6 +28,7 @@ import {
 } from '@pnpm/lockfile.verification'
 import { globalWarn, logger } from '@pnpm/logger'
 import { parseOverrides } from '@pnpm/parse-overrides'
+import { getPnpmfilePath } from '@pnpm/pnpmfile'
 import { type WorkspacePackages } from '@pnpm/resolver-base'
 import {
   type DependencyManifest,
@@ -39,7 +40,8 @@ import { findWorkspacePackages } from '@pnpm/workspace.find-packages'
 import { readWorkspaceManifest } from '@pnpm/workspace.read-manifest'
 import { loadWorkspaceState, updateWorkspaceState } from '@pnpm/workspace.state'
 import { assertLockfilesEqual } from './assertLockfilesEqual'
-import { statManifestFile } from './statManifestFile'
+import { safeStat, safeStatSync, statManifestFile } from './statManifestFile'
+import { type WorkspaceStateSettings } from '@pnpm/workspace.state/src/types'
 
 export type CheckDepsStatusOptions = Pick<Config,
 | 'allProjects'
@@ -55,9 +57,13 @@ export type CheckDepsStatusOptions = Pick<Config,
 | 'sharedWorkspaceLockfile'
 | 'virtualStoreDir'
 | 'workspaceDir'
->
+| 'patchesDir'
+| 'pnpmfile'
+> & {
+  ignoreFilteredInstallCache?: boolean
+} & WorkspaceStateSettings
 
-export async function checkDepsStatus (opts: CheckDepsStatusOptions): Promise<{ upToDate: boolean, issue?: string }> {
+export async function checkDepsStatus (opts: CheckDepsStatusOptions): Promise<{ upToDate: boolean | undefined, issue?: string }> {
   try {
     return await _checkDepsStatus(opts)
   } catch (error) {
@@ -71,7 +77,7 @@ export async function checkDepsStatus (opts: CheckDepsStatusOptions): Promise<{
   }
 }
 
-async function _checkDepsStatus (opts: CheckDepsStatusOptions): Promise<{ upToDate: boolean, issue?: string }> {
+async function _checkDepsStatus (opts: CheckDepsStatusOptions): Promise<{ upToDate: boolean | undefined, issue?: string }> {
   const {
     allProjects,
     autoInstallPeers,
@@ -89,17 +95,31 @@ async function _checkDepsStatus (opts: CheckDepsStatusOptions): Promise<{ upToDa
     ? getOptionsFromRootManifest(rootProjectManifestDir, rootProjectManifest)
     : undefined
 
-  if (allProjects && workspaceDir) {
-    const workspaceState = loadWorkspaceState(workspaceDir)
-    if (!workspaceState) {
+  const workspaceState = loadWorkspaceState(workspaceDir ?? rootProjectManifestDir)
+  if (!workspaceState) {
+    return {
+      upToDate: false,
+      issue: 'Cannot check whether dependencies are outdated',
+    }
+  }
+  if (opts.ignoreFilteredInstallCache && workspaceState.filteredInstall) {
+    return { upToDate: undefined }
+  }
+
+  for (const [settingName, settingValue] of Object.entries(workspaceState.settings)) {
+    if (settingName === 'catalogs') continue
+    // @ts-expect-error
+    if (!equals(settingValue, opts[settingName])) {
       return {
         upToDate: false,
-        issue: 'Cannot check whether dependencies are outdated',
+        issue: `The value of the ${settingName} setting has changed`,
       }
     }
+  }
 
+  if (allProjects && workspaceDir) {
     if (!equals(
-      filter(value => value != null, workspaceState.catalogs ?? {}),
+      filter(value => value != null, workspaceState.settings.catalogs ?? {}),
       filter(value => value != null, catalogs ?? {})
     )) {
       return {
@@ -108,8 +128,13 @@ async function _checkDepsStatus (opts: CheckDepsStatusOptions): Promise<{ upToDa
       }
     }
 
-    const currentProjectRootDirs = allProjects.map(project => project.rootDir).sort()
-    if (!equals(workspaceState.projectRootDirs, currentProjectRootDirs)) {
+    if (allProjects.length !== Object.keys(workspaceState.projects).length ||
+      !allProjects.every((currentProject) => {
+        const prevProject = workspaceState.projects[currentProject.rootDir]
+        if (!prevProject) return false
+        return prevProject.name === currentProject.manifest.name && (prevProject.version ?? '0.0.0') === (currentProject.manifest.version ?? '0.0.0')
+      })
+    ) {
       return {
         upToDate: false,
         issue: 'The workspace structure has changed since last install',
@@ -135,6 +160,17 @@ async function _checkDepsStatus (opts: CheckDepsStatusOptions): Promise<{ upToDa
       return { upToDate: true }
     }
 
+    const issue = await patchesAreModified({
+      rootManifestOptions,
+      rootDir: rootProjectManifestDir,
+      lastValidatedTimestamp: workspaceState.lastValidatedTimestamp,
+      pnpmfile: opts.pnpmfile,
+      hadPnpmfile: workspaceState.pnpmfileExists,
+    })
+    if (issue) {
+      return { upToDate: false, issue }
+    }
+
     logger.debug({ msg: 'Some manifest files were modified since the last validation. Continuing check.' })
 
     let readWantedLockfileAndDir: (projectDir: string) => Promise<{
@@ -204,23 +240,28 @@ async function _checkDepsStatus (opts: CheckDepsStatusOptions): Promise<{ upToDa
       rootManifestOptions,
     }
 
-    await Promise.all(modifiedProjects.map(async ({ project }) => {
-      const { wantedLockfile, wantedLockfileDir } = await readWantedLockfileAndDir(project.rootDir)
-      await assertWantedLockfileUpToDate(assertCtx, {
-        projectDir: project.rootDir,
-        projectId: getProjectId(project),
-        projectManifest: project.manifest,
-        wantedLockfile,
-        wantedLockfileDir,
-      })
-    }))
+    try {
+      await Promise.all(modifiedProjects.map(async ({ project }) => {
+        const { wantedLockfile, wantedLockfileDir } = await readWantedLockfileAndDir(project.rootDir)
+        await assertWantedLockfileUpToDate(assertCtx, {
+          projectDir: project.rootDir,
+          projectId: getProjectId(project),
+          projectManifest: project.manifest,
+          wantedLockfile,
+          wantedLockfileDir,
+        })
+      }))
+    } catch (err) {
+      return { upToDate: false, issue: (util.types.isNativeError(err) && 'message' in err) ? err.message : undefined }
+    }
 
     // update lastValidatedTimestamp to prevent pointless repeat
     await updateWorkspaceState({
       allProjects,
-      catalogs,
-      lastValidatedTimestamp: Date.now(),
       workspaceDir,
+      pnpmfileExists: workspaceState.pnpmfileExists,
+      settings: opts,
+      filteredInstall: workspaceState.filteredInstall,
     })
 
     return { upToDate: true }
@@ -260,6 +301,17 @@ async function _checkDepsStatus (opts: CheckDepsStatusOptions): Promise<{ upToDa
 
     if (!wantedLockfileStats) return throwLockfileNotFound(rootProjectManifestDir)
 
+    const issue = await patchesAreModified({
+      rootManifestOptions,
+      rootDir: rootProjectManifestDir,
+      lastValidatedTimestamp: wantedLockfileStats.mtime.valueOf(),
+      pnpmfile: opts.pnpmfile,
+      hadPnpmfile: workspaceState.pnpmfileExists,
+    })
+    if (issue) {
+      return { upToDate: false, issue }
+    }
+
     if (currentLockfileStats && wantedLockfileStats.mtime.valueOf() > currentLockfileStats.mtime.valueOf()) {
       const currentLockfile = await currentLockfilePromise
       const wantedLockfile = (await wantedLockfilePromise) ?? throwLockfileNotFound(rootProjectManifestDir)
@@ -273,23 +325,27 @@ async function _checkDepsStatus (opts: CheckDepsStatusOptions): Promise<{ upToDa
 
     if (manifestStats.mtime.valueOf() > wantedLockfileStats.mtime.valueOf()) {
       logger.debug({ msg: 'The manifest is newer than the lockfile. Continuing check.' })
-      await assertWantedLockfileUpToDate({
-        autoInstallPeers,
-        injectWorkspacePackages,
-        config: opts,
-        excludeLinksFromLockfile,
-        linkWorkspacePackages,
-        getManifestsByDir: () => ({}),
-        getWorkspacePackages: () => undefined,
-        rootDir: rootProjectManifestDir,
-        rootManifestOptions,
-      }, {
-        projectDir: rootProjectManifestDir,
-        projectId: '.' as ProjectId,
-        projectManifest: rootProjectManifest,
-        wantedLockfile: (await wantedLockfilePromise) ?? throwLockfileNotFound(rootProjectManifestDir),
-        wantedLockfileDir: rootProjectManifestDir,
-      })
+      try {
+        await assertWantedLockfileUpToDate({
+          autoInstallPeers,
+          injectWorkspacePackages,
+          config: opts,
+          excludeLinksFromLockfile,
+          linkWorkspacePackages,
+          getManifestsByDir: () => ({}),
+          getWorkspacePackages: () => undefined,
+          rootDir: rootProjectManifestDir,
+          rootManifestOptions,
+        }, {
+          projectDir: rootProjectManifestDir,
+          projectId: '.' as ProjectId,
+          projectManifest: rootProjectManifest,
+          wantedLockfile: (await wantedLockfilePromise) ?? throwLockfileNotFound(rootProjectManifestDir),
+          wantedLockfileDir: rootProjectManifestDir,
+        })
+      } catch (err) {
+        return { upToDate: false, issue: (util.types.isNativeError(err) && 'message' in err) ? err.message : undefined }
+      }
     } else if (currentLockfileStats) {
       logger.debug({ msg: 'The manifest file is not newer than the lockfile. Exiting check.' })
     } else {
@@ -308,7 +364,7 @@ async function _checkDepsStatus (opts: CheckDepsStatusOptions): Promise<{ upToDa
   // `rootProjectManifest` being `undefined` means that there's no root manifest.
   // Both means that `pnpm run` would fail, so checking lockfiles here is pointless.
   globalWarn('Skipping check.')
-  return { upToDate: true }
+  return { upToDate: undefined }
 }
 
 interface AssertWantedLockfileUpToDateContext {
@@ -428,3 +484,35 @@ function throwLockfileNotFound (wantedLockfileDir: string): never {
     hint: 'Run `pnpm install` to create the lockfile',
   })
 }
+
+async function patchesAreModified (opts: {
+  rootManifestOptions: OptionsFromRootManifest | undefined
+  rootDir: string
+  lastValidatedTimestamp: number
+  pnpmfile: string
+  hadPnpmfile: boolean
+}): Promise<string | undefined> {
+  if (opts.rootManifestOptions?.patchedDependencies) {
+    const allPatchStats = await Promise.all(Object.values(opts.rootManifestOptions.patchedDependencies).map((patchFile) => {
+      return safeStat(path.relative(opts.rootDir, patchFile))
+    }))
+    if (allPatchStats.some(
+      (patch) =>
+        patch && patch.mtime.valueOf() > opts.lastValidatedTimestamp
+    )) {
+      return 'Patches were modified'
+    }
+  }
+  const pnpmfilePath = getPnpmfilePath(opts.rootDir, opts.pnpmfile)
+  const pnpmfileStats = safeStatSync(pnpmfilePath)
+  if (pnpmfileStats != null && pnpmfileStats.mtime.valueOf() > opts.lastValidatedTimestamp) {
+    return `pnpmfile at "${pnpmfilePath}" was modified`
+  }
+  if (opts.hadPnpmfile && pnpmfileStats == null) {
+    return `pnpmfile at "${pnpmfilePath}" was removed`
+  }
+  if (!opts.hadPnpmfile && pnpmfileStats != null) {
+    return `pnpmfile at "${pnpmfilePath}" was added`
+  }
+  return undefined
+}
diff --git a/deps/status/src/statManifestFile.ts b/deps/status/src/statManifestFile.ts
@@ -4,18 +4,31 @@ import util from 'util'
 import { MANIFEST_BASE_NAMES } from '@pnpm/constants'
 
 export async function statManifestFile (projectRootDir: string): Promise<fs.Stats | undefined> {
-  const attempts = await Promise.all(MANIFEST_BASE_NAMES.map(async baseName => {
+  const attempts = await Promise.all(MANIFEST_BASE_NAMES.map((baseName) => {
     const manifestPath = path.join(projectRootDir, baseName)
-    let stats: fs.Stats
-    try {
-      stats = await fs.promises.stat(manifestPath)
-    } catch (error) {
-      if (util.types.isNativeError(error) && 'code' in error && error.code === 'ENOENT') {
-        return undefined
-      }
-      throw error
-    }
-    return stats
+    return safeStat(manifestPath)
   }))
   return attempts.find(stats => stats != null)
 }
+
+export async function safeStat (filePath: string): Promise<fs.Stats | undefined> {
+  try {
+    return await fs.promises.stat(filePath)
+  } catch (error) {
+    if (util.types.isNativeError(error) && 'code' in error && error.code === 'ENOENT') {
+      return undefined
+    }
+    throw error
+  }
+}
+
+export function safeStatSync (filePath: string): fs.Stats | undefined {
+  try {
+    return fs.statSync(filePath)
+  } catch (error) {
+    if (util.types.isNativeError(error) && 'code' in error && error.code === 'ENOENT') {
+      return undefined
+    }
+    throw error
+  }
+}
diff --git a/deps/status/tsconfig.json b/deps/status/tsconfig.json
@@ -21,6 +21,9 @@
     {
       "path": "../../crypto/object-hasher"
     },
+    {
+      "path": "../../hooks/pnpmfile"
+    },
     {
       "path": "../../lockfile/fs"
     },

diff --git a/exec/plugin-commands-script-runners/test/utils/index.ts b/exec/plugin-commands-script-runners/test/utils/index.ts
@@ -39,6 +39,7 @@ export const DEFAULT_OPTS = {
   pending: false,
   pnpmfile: './.pnpmfile.cjs',
   pnpmHomeDir: '',
+  preferWorkspacePackages: true,
   proxy: undefined,
   rawConfig: { registry: REGISTRY_URL },
   rawLocalConfig: {},
@@ -67,6 +68,7 @@ export const DLX_DEFAULT_OPTS = {
   bail: false,
   bin: 'node_modules/.bin',
   cacheDir: path.join(tmp, 'cache'),
+  excludeLinksFromLockfile: false,
   extraEnv: {},
   extraBinPaths: [],
   cliOptions: {},
@@ -80,6 +82,7 @@ export const DLX_DEFAULT_OPTS = {
   lock: true,
   pnpmfile: '.pnpmfile.cjs',
   pnpmHomeDir: '',
+  preferWorkspacePackages: true,
   rawConfig: { registry: REGISTRY_URL },
   rawLocalConfig: { registry: REGISTRY_URL },
   registries: {

diff --git a/exec/plugin-commands-script-runners/test/verifyDepsBeforeRun.ts b/exec/plugin-commands-script-runners/test/verifyDepsBeforeRun.ts
@@ -56,7 +56,7 @@ test('throw an error if verifyDepsBeforeRun is set to error', async () => {
   } catch (_err) {
     err = _err as Error
   }
-  expect(err.message).toContain('Cannot find a lockfile in')
+  expect(err.message).toContain('Cannot check whether dependencies are outdated')
 })
 
 test('install the dependencies if verifyDepsBeforeRun is set to install', async () => {

diff --git a/hooks/pnpmfile/src/index.ts b/hooks/pnpmfile/src/index.ts
@@ -1,5 +1,6 @@
 import type { CookedHooks } from './requireHooks'
 
+export { getPnpmfilePath } from './getPnpmfilePath'
 export { requireHooks } from './requireHooks'
 export { requirePnpmfile, BadReadPackageHookError } from './requirePnpmfile'
 export type { HookContext } from './Hooks'

diff --git a/lockfile/plugin-commands-audit/test/index.ts b/lockfile/plugin-commands-audit/test/index.ts
@@ -23,6 +23,7 @@ export const DEFAULT_OPTS = {
   ca: undefined,
   cacheDir: '../cache',
   cert: undefined,
+  excludeLinksFromLockfile: false,
   extraEnv: {},
   cliOptions: {},
   fetchRetries: 2,
@@ -46,6 +47,7 @@ export const DEFAULT_OPTS = {
   pending: false,
   pnpmfile: './.pnpmfile.cjs',
   pnpmHomeDir: '',
+  preferWorkspacePackages: true,
   proxy: undefined,
   rawConfig,
   rawLocalConfig: {},

diff --git a/package.json b/package.json
@@ -24,7 +24,7 @@
     "meta-updater": "pnpm --filter=@pnpm-private/updater compile && pnpm exec meta-updater",
     "lint:meta": "pnpm run meta-updater --test",
     "copy-artifacts": "ts-node __utils__/scripts/src/copy-artifacts.ts",
-    "make-release-description": "pnpm --filter=@pnpm/get-release-text run write-release-text",
+    "make-release-description": "pnpm --filter=@pnpm/get-release-text run write-release-ext",
     "release": "pnpm --filter=@pnpm/exe publish --tag=next-10 --access=public && pnpm publish --filter=!pnpm --filter=!@pnpm/exe --access=public && pnpm publish --filter=pnpm --tag=next-10 --access=public",
     "dev-setup": "pnpm -C=./pnpm/dev link -g"
   },
-Original file line number
+Diff line change
@@ Expand Up / @@ -21,6 +21,9 @@ @@
         {
           "path": "../../crypto/object-hasher"
         },
+        {
+          "path": "../../hooks/pnpmfile"
+        },
         {
           "path": "../../lockfile/fs"
         },
@@ Expand Down @@