Create a testing framework for healthcheck-based tools (#484)

If this API looks good I'll work on converting more of the tools to it.
trunk-io · Sep 26, 2023 · 8ded98f · 8ded98f
1 parent de0dae0
commit 8ded98f
Show file tree

Hide file tree

Showing 5 changed files with 111 additions and 26 deletions.
diff --git a/repo-tools/tool-test-helper/generate b/repo-tools/tool-test-helper/generate
@@ -31,7 +31,7 @@ tools:
       shims: [*{}*]
 """
 
-INITIAL_TEST_CONTENTS = """import { makeToolTestConfig, toolTest } from "tests";
+INITIAL_TEST_CONTENTS = """import { makeToolTestConfig, toolTest, toolInstallTest } from "tests";
 toolTest({
   toolName: "*{}*",
   toolVersion: "VERSION_HERE",
@@ -43,7 +43,14 @@ toolTest({
   ],
 });
 
+toolInstallTest({
+  toolName: "*{}*",
+  toolVersion: "VERSION_HERE",
+})
+
 // Guidelines for configuring tests:
+//  - Prefer using health check in config + toolInstallTest, if you must use toolTest leave a
+//    comment explaining why. Only one of the two options is sufficient.
 //  - Usually, just a version or help text command is sufficient
 //  - add a test for each command that is used in the plugin.yaml
 //  - exit code 0 is assumed, so set expectedExitCode if it is different

diff --git a/tests/driver/tool_driver.ts b/tests/driver/tool_driver.ts
@@ -67,12 +67,16 @@ lint:
 `;
   }
 
+  async setUpWithInstall() {
+    await this.setUp();
+    await this.installTool();
+  }
+
   /**
    * Setup a sandbox test directory by copying in test contents and conditionally:
    * 1. Creating a git repo
    * 2. Dumping a newly generated trunk.yaml
    * 3. Enabling the specified 'tool'
-   * 4. Sync to make sure it's available
    */
   async setUp() {
     await super.setUp();
@@ -101,7 +105,24 @@ lint:
         this.enabledVersion = foundIn.groups.version;
         this.debug("Enabled %s", this.enabledVersion);
       }
+    } catch (error) {
+      console.warn(`Failed to enable ${this.tool}`, error);
+      if ("stdout" in (error as any)) {
+        // trunk-ignore(eslint/@typescript-eslint/no-unsafe-member-access)
+        console.log("Error output:", ((error as any).stdout as Buffer).toString());
+      } else {
+        console.log("Error keys:  ", Object.keys(error as object));
+      }
+    }
+  }
 
+  async installTool() {
+    // Enable tested tool if specified
+    if (!this.tool || !this.sandboxPath) {
+      console.error("Tool or sandbox path not specified - we should not be here!");
+      return;
+    }
+    try {
       // Sync the tool to ensure it's available
       await this.runTrunk(["tools", "install", this.tool, "--ci"]);
       const tools_subdir = fs.existsSync(path.resolve(this.sandboxPath ?? "", ".trunk/dev-tools"))
@@ -121,6 +142,7 @@ lint:
           throw new Error(`Could not install or find installed ${shim}`);
         }
       }
+      this.debug("Installed %s", this.tool);
     } catch (error) {
       console.warn(`Failed to enable ${this.tool}`, error);
       if ("stdout" in (error as any)) {

diff --git a/tests/index.ts b/tests/index.ts
@@ -146,6 +146,35 @@ export const setupTrunkToolDriver = (
     version,
   );
 
+  beforeAll(async () => {
+    if (preCheck) {
+      // preCheck is not always async, but we must await in case it is.
+      await preCheck(driver);
+      driver.debug("Finished running custom preCheck hook");
+    }
+    await driver.setUpWithInstall();
+  });
+
+  afterAll(() => {
+    driver.tearDown();
+  });
+  return driver;
+};
+
+export const setUpTrunkToolDriverForHealthCheck = (
+  dirname: string,
+  { setupGit = true, setupTrunk = true, trunkVersion = undefined }: SetupSettings,
+  toolName?: string,
+  version?: string,
+  preCheck?: ToolTestCallback,
+): TrunkToolDriver => {
+  const driver = new TrunkToolDriver(
+    dirname,
+    { setupGit, setupTrunk, trunkVersion },
+    toolName,
+    version,
+  );
+
   beforeAll(async () => {
     if (preCheck) {
       // preCheck is not always async, but we must await in case it is.
@@ -161,6 +190,52 @@ export const setupTrunkToolDriver = (
   return driver;
 };
 
+const runInstall = async (
+  driver: TrunkToolDriver,
+  toolName: string,
+): Promise<{
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+}> => {
+  try {
+    const { stdout, stderr } = await driver.runTrunk(["tools", "install", toolName, "--ci"]);
+    return { exitCode: 0, stdout, stderr };
+  } catch (e: any) {
+    // trunk-ignore(eslint/@typescript-eslint/no-unsafe-member-access)
+    return { exitCode: e.code as number, stdout: e.stdout as string, stderr: e.stderr as string };
+  }
+};
+
+// NOTE(lauri): This is a variant of the testing framework that just validates a `trunk tools install`.
+// in case of tools with configured health checks, this should be a sufficient amount of testing. If not
+// the regular toolTest framework allows running arbitrary commands with the tool.
+// If this is deemed to provide sufficient test coverage it will become the sole tool testing framework
+// going forward.
+export const toolInstallTest = ({
+  toolName,
+  toolVersion,
+  dirName = path.dirname(caller()),
+  skipTestIf = (_version?: string) => false,
+  preCheck,
+}: {
+  toolName: string;
+  toolVersion: string;
+  dirName?: string;
+  skipTestIf?: (version?: string) => boolean;
+  preCheck?: ToolTestCallback;
+}) => {
+  const driver = setUpTrunkToolDriverForHealthCheck(dirName, {}, toolName, toolVersion, preCheck);
+  conditionalTest(skipTestIf(toolVersion), "tool ", async () => {
+    const { exitCode, stdout, stderr } = await runInstall(driver, toolName);
+    expect(exitCode).toEqual(0);
+    expect(stdout).toContain(toolName);
+    expect(stdout).toContain(toolVersion);
+    expect(stderr).toEqual("");
+    expect(stdout).not.toContain("Failures:");
+  });
+};
+
 interface ToolTestConfig {
   command: string[];
   expectedOut?: string;

diff --git a/tools/clangd-indexing-tools/clangd_indexing_tools.test.ts b/tools/clangd-indexing-tools/clangd_indexing_tools.test.ts
@@ -1,19 +1,5 @@
-import { makeToolTestConfig, toolTest } from "tests";
-toolTest({
+import { toolInstallTest } from "tests";
+toolInstallTest({
   toolName: "clangd-indexing-tools",
   toolVersion: "16.0.2",
-  testConfigs: [
-    makeToolTestConfig({
-      command: ["clangd-indexer", "--version"],
-      expectedOut: "LLVM version 16.0.2",
-    }),
-    makeToolTestConfig({
-      command: ["clangd-index-server", "--version"],
-      expectedOut: "LLVM version 16.0.2",
-    }),
-    makeToolTestConfig({
-      command: ["clangd-index-server-monitor", "--version"],
-      expectedOut: "LLVM version 16.0.2",
-    }),
-  ],
 });
diff --git a/tools/clangd/clangd.test.ts b/tools/clangd/clangd.test.ts
@@ -1,11 +1,6 @@
-import { makeToolTestConfig, toolTest } from "tests";
-toolTest({
+import { toolInstallTest } from "tests";
+
+toolInstallTest({
   toolName: "clangd",
   toolVersion: "16.0.2",
-  testConfigs: [
-    makeToolTestConfig({
-      command: ["clangd", "--version"],
-      expectedOut: "clangd version 16.0.2",
-    }),
-  ],
 });