[heft] Update file copy layer to support incremental disk cache (#4943)

* [heft] Update file copy layer to support incremental disk cache * rush change * (chore) Remove references to copying folders * Use object format, add unit tests * Always use array * Update unit test --------- Co-authored-by: David Michon <dmichon-msft@users.noreply.github.com>
microsoft · Sep 28, 2024 · 48e0497 · 48e0497
1 parent 30e232b
commit 48e0497
Show file tree

Hide file tree

Showing 7 changed files with 442 additions and 7 deletions.
diff --git a/apps/heft/src/operations/runners/TaskOperationRunner.ts b/apps/heft/src/operations/runners/TaskOperationRunner.ts
@@ -183,6 +183,7 @@ export class TaskOperationRunner implements IOperationRunner {
           ? copyFilesAsync(
               copyOperations,
               logger.terminal,
+              `${taskSession.tempFolderPath}/file-copy.json`,
               isWatchMode ? getWatchFileSystemAdapter() : undefined
             )
           : Promise.resolve(),

diff --git a/apps/heft/src/pluginFramework/IncrementalBuildInfo.ts b/apps/heft/src/pluginFramework/IncrementalBuildInfo.ts
@@ -0,0 +1,212 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+import * as path from 'path';
+
+import { FileSystem, Path } from '@rushstack/node-core-library';
+
+/**
+ * Information about an incremental build. This information is used to determine which files need to be rebuilt.
+ * @beta
+ */
+export interface IIncrementalBuildInfo {
+  /**
+   * A string that represents the configuration inputs for the build.
+   * If the configuration changes, the old build info object should be discarded.
+   */
+  configHash: string;
+
+  /**
+   * A map of absolute input file paths to their version strings.
+   * The version string should change if the file changes.
+   */
+  inputFileVersions: Map<string, string>;
+
+  /**
+   * A map of absolute output file paths to the input files they were computed from.
+   */
+  fileDependencies?: Map<string, string[]>;
+}
+
+/**
+ * Serialized version of {@link IIncrementalBuildInfo}.
+ * @beta
+ */
+export interface ISerializedIncrementalBuildInfo {
+  /**
+   * A string that represents the configuration inputs for the build.
+   * If the configuration changes, the old build info object should be discarded.
+   */
+  configHash: string;
+
+  /**
+   * A map of input files to their version strings.
+   * File paths are specified relative to the folder containing the build info file.
+   */
+  inputFileVersions: Record<string, string>;
+
+  /**
+   * Map of output file names to the corresponding index in `Object.entries(inputFileVersions)`.
+   * File paths are specified relative to the folder containing the build info file.
+   */
+  fileDependencies?: Record<string, number[]>;
+}
+
+/**
+ * Converts an absolute path to a path relative to a base path.
+ */
+const makePathRelative: (absolutePath: string, basePath: string) => string =
+  process.platform === 'win32'
+    ? (absolutePath: string, basePath: string) => {
+        // On Windows, need to normalize slashes
+        return Path.convertToSlashes(path.win32.relative(basePath, absolutePath));
+      }
+    : (absolutePath: string, basePath: string) => {
+        // On POSIX, can preserve existing slashes
+        return path.posix.relative(basePath, absolutePath);
+      };
+
+/**
+ * Serializes a build info object to a portable format that can be written to disk.
+ * @param state - The build info to serialize
+ * @param makePathPortable - A function that converts an absolute path to a portable path. This is a separate argument to support cross-platform tests.
+ * @returns The serialized build info
+ * @beta
+ */
+export function serializeBuildInfo(
+  state: IIncrementalBuildInfo,
+  makePathPortable: (absolutePath: string) => string
+): ISerializedIncrementalBuildInfo {
+  const fileIndices: Map<string, number> = new Map();
+  const inputFileVersions: Record<string, string> = {};
+
+  for (const [absolutePath, version] of state.inputFileVersions) {
+    const relativePath: string = makePathPortable(absolutePath);
+    fileIndices.set(absolutePath, fileIndices.size);
+    inputFileVersions[relativePath] = version;
+  }
+
+  const { fileDependencies: newFileDependencies } = state;
+  let fileDependencies: Record<string, number[]> | undefined;
+  if (newFileDependencies) {
+    fileDependencies = {};
+    for (const [absolutePath, dependencies] of newFileDependencies) {
+      const relativePath: string = makePathPortable(absolutePath);
+      const indices: number[] = [];
+      for (const dependency of dependencies) {
+        const index: number | undefined = fileIndices.get(dependency);
+        if (index === undefined) {
+          throw new Error(`Dependency not found: ${dependency}`);
+        }
+        indices.push(index);
+      }
+
+      fileDependencies[relativePath] = indices;
+    }
+  }
+
+  const serializedBuildInfo: ISerializedIncrementalBuildInfo = {
+    configHash: state.configHash,
+    inputFileVersions,
+    fileDependencies
+  };
+
+  return serializedBuildInfo;
+}
+
+/**
+ * Deserializes a build info object from its portable format.
+ * @param serializedBuildInfo - The build info to deserialize
+ * @param makePathAbsolute - A function that converts a portable path to an absolute path. This is a separate argument to support cross-platform tests.
+ * @returns The deserialized build info
+ */
+export function deserializeBuildInfo(
+  serializedBuildInfo: ISerializedIncrementalBuildInfo,
+  makePathAbsolute: (relativePath: string) => string
+): IIncrementalBuildInfo {
+  const inputFileVersions: Map<string, string> = new Map();
+  const absolutePathByIndex: string[] = [];
+  for (const [relativePath, version] of Object.entries(serializedBuildInfo.inputFileVersions)) {
+    const absolutePath: string = makePathAbsolute(relativePath);
+    absolutePathByIndex.push(absolutePath);
+    inputFileVersions.set(absolutePath, version);
+  }
+
+  let fileDependencies: Map<string, string[]> | undefined;
+  const { fileDependencies: serializedFileDependencies } = serializedBuildInfo;
+  if (serializedFileDependencies) {
+    fileDependencies = new Map();
+    for (const [relativeOutputFile, indices] of Object.entries(serializedFileDependencies)) {
+      const absoluteOutputFile: string = makePathAbsolute(relativeOutputFile);
+      const dependencies: string[] = [];
+      for (const index of Array.isArray(indices) ? indices : [indices]) {
+        const dependencyAbsolutePath: string | undefined = absolutePathByIndex[index];
+        if (dependencyAbsolutePath === undefined) {
+          throw new Error(`Dependency index not found: ${index}`);
+        }
+        dependencies.push(dependencyAbsolutePath);
+      }
+      fileDependencies.set(absoluteOutputFile, dependencies);
+    }
+  }
+
+  const buildInfo: IIncrementalBuildInfo = {
+    configHash: serializedBuildInfo.configHash,
+    inputFileVersions,
+    fileDependencies
+  };
+
+  return buildInfo;
+}
+
+/**
+ * Writes a build info object to disk.
+ * @param state - The build info to write
+ * @param filePath - The file path to write the build info to
+ * @beta
+ */
+export async function writeBuildInfoAsync(state: IIncrementalBuildInfo, filePath: string): Promise<void> {
+  const basePath: string = path.dirname(filePath);
+
+  const serializedBuildInfo: ISerializedIncrementalBuildInfo = serializeBuildInfo(
+    state,
+    (absolutePath: string) => {
+      return makePathRelative(basePath, absolutePath);
+    }
+  );
+
+  // This file is meant only for machine reading, so don't pretty-print it.
+  const stringified: string = JSON.stringify(serializedBuildInfo);
+
+  await FileSystem.writeFileAsync(filePath, stringified, { ensureFolderExists: true });
+}
+
+/**
+ * Reads a build info object from disk.
+ * @param filePath - The file path to read the build info from
+ * @returns The build info object, or undefined if the file does not exist or cannot be parsed
+ * @beta
+ */
+export async function tryReadBuildInfoAsync(filePath: string): Promise<IIncrementalBuildInfo | undefined> {
+  let serializedBuildInfo: ISerializedIncrementalBuildInfo | undefined;
+  try {
+    const fileContents: string = await FileSystem.readFileAsync(filePath);
+    serializedBuildInfo = JSON.parse(fileContents) as ISerializedIncrementalBuildInfo;
+  } catch (error) {
+    if (FileSystem.isNotExistError(error)) {
+      return;
+    }
+    throw error;
+  }
+
+  const basePath: string = path.dirname(filePath);
+
+  const buildInfo: IIncrementalBuildInfo = deserializeBuildInfo(
+    serializedBuildInfo,
+    (relativePath: string) => {
+      return path.resolve(basePath, relativePath);
+    }
+  );
+
+  return buildInfo;
+}
diff --git a/apps/heft/src/pluginFramework/tests/IncrementalBuildInfo.test.ts b/apps/heft/src/pluginFramework/tests/IncrementalBuildInfo.test.ts
@@ -0,0 +1,104 @@
+// Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
+// See LICENSE in the project root for license information.
+
+import path from 'path';
+
+import { Path } from '@rushstack/node-core-library';
+
+import {
+  serializeBuildInfo,
+  deserializeBuildInfo,
+  type IIncrementalBuildInfo,
+  type ISerializedIncrementalBuildInfo
+} from '../IncrementalBuildInfo';
+
+const posixBuildInfo: IIncrementalBuildInfo = {
+  configHash: 'foobar',
+  inputFileVersions: new Map([
+    ['/a/b/c/file1', '1'],
+    ['/a/b/c/file2', '2']
+  ]),
+  fileDependencies: new Map([
+    ['/a/b/c/output1', ['/a/b/c/file1']],
+    ['/a/b/c/output2', ['/a/b/c/file1', '/a/b/c/file2']]
+  ])
+};
+
+const win32BuildInfo: IIncrementalBuildInfo = {
+  configHash: 'foobar',
+  inputFileVersions: new Map([
+    ['A:\\b\\c\\file1', '1'],
+    ['A:\\b\\c\\file2', '2']
+  ]),
+  fileDependencies: new Map([
+    ['A:\\b\\c\\output1', ['A:\\b\\c\\file1']],
+    ['A:\\b\\c\\output2', ['A:\\b\\c\\file1', 'A:\\b\\c\\file2']]
+  ])
+};
+
+const posixBasePath: string = '/a/b/temp';
+const win32BasePath: string = 'A:\\b\\temp';
+
+function posixToPortable(absolutePath: string): string {
+  return path.posix.relative(posixBasePath, absolutePath);
+}
+function portableToPosix(portablePath: string): string {
+  return path.posix.resolve(posixBasePath, portablePath);
+}
+
+function win32ToPortable(absolutePath: string): string {
+  return Path.convertToSlashes(path.win32.relative(win32BasePath, absolutePath));
+}
+function portableToWin32(portablePath: string): string {
+  return path.win32.resolve(win32BasePath, portablePath);
+}
+
+describe(serializeBuildInfo.name, () => {
+  it('Round trips correctly (POSIX)', () => {
+    const serialized: ISerializedIncrementalBuildInfo = serializeBuildInfo(posixBuildInfo, posixToPortable);
+
+    const deserialized: IIncrementalBuildInfo = deserializeBuildInfo(serialized, portableToPosix);
+
+    expect(deserialized).toEqual(posixBuildInfo);
+  });
+
+  it('Round trips correctly (Win32)', () => {
+    const serialized: ISerializedIncrementalBuildInfo = serializeBuildInfo(win32BuildInfo, win32ToPortable);
+
+    const deserialized: IIncrementalBuildInfo = deserializeBuildInfo(serialized, portableToWin32);
+
+    expect(deserialized).toEqual(win32BuildInfo);
+  });
+
+  it('Converts (POSIX to Win32)', () => {
+    const serialized: ISerializedIncrementalBuildInfo = serializeBuildInfo(posixBuildInfo, posixToPortable);
+
+    const deserialized: IIncrementalBuildInfo = deserializeBuildInfo(serialized, portableToWin32);
+
+    expect(deserialized).toEqual(win32BuildInfo);
+  });
+
+  it('Converts (Win32 to POSIX)', () => {
+    const serialized: ISerializedIncrementalBuildInfo = serializeBuildInfo(win32BuildInfo, win32ToPortable);
+
+    const deserialized: IIncrementalBuildInfo = deserializeBuildInfo(serialized, portableToPosix);
+
+    expect(deserialized).toEqual(posixBuildInfo);
+  });
+
+  it('Has expected serialized format', () => {
+    const serializedPosix: ISerializedIncrementalBuildInfo = serializeBuildInfo(
+      posixBuildInfo,
+      posixToPortable
+    );
+    const serializedWin32: ISerializedIncrementalBuildInfo = serializeBuildInfo(
+      win32BuildInfo,
+      win32ToPortable
+    );
+
+    expect(serializedPosix).toMatchSnapshot('posix');
+    expect(serializedWin32).toMatchSnapshot('win32');
+
+    expect(serializedPosix).toEqual(serializedWin32);
+  });
+});
diff --git a/apps/heft/src/pluginFramework/tests/__snapshots__/IncrementalBuildInfo.test.ts.snap b/apps/heft/src/pluginFramework/tests/__snapshots__/IncrementalBuildInfo.test.ts.snap
@@ -0,0 +1,39 @@
+// Jest Snapshot v1, https://goo.gl/fbAQLP
+
+exports[`serializeBuildInfo Has expected serialized format: posix 1`] = `
+Object {
+  "configHash": "foobar",
+  "fileDependencies": Object {
+    "../c/output1": Array [
+      0,
+    ],
+    "../c/output2": Array [
+      0,
+      1,
+    ],
+  },
+  "inputFileVersions": Object {
+    "../c/file1": "1",
+    "../c/file2": "2",
+  },
+}
+`;
+
+exports[`serializeBuildInfo Has expected serialized format: win32 1`] = `
+Object {
+  "configHash": "foobar",
+  "fileDependencies": Object {
+    "../c/output1": Array [
+      0,
+    ],
+    "../c/output2": Array [
+      0,
+      1,
+    ],
+  },
+  "inputFileVersions": Object {
+    "../c/file1": "1",
+    "../c/file2": "2",
+  },
+}
+`;