feat: support afterAll hook (#387)

Author: ojeytonwilliams

packages/dom-evaluator/src/dom-test-evaluator.ts

@@ -7,6 +7,7 @@ import { assert as chaiAssert } from "chai";
 import type {
   TestEvaluator,
   Fail,
+  CodeEvent,
   TestEvent,
   InitEvent,
   Pass,
@@ -182,8 +183,18 @@ ${rawTest}`);
     return this.#runTest!(test);
   }
 
+  async runCode(code: string) {
+    try {
+      await eval(code);
+    } catch (err) {
+      // If the code throws an error, we want to log it to the console
+      // so that it can be debugged.
+      console.error("Error evaluating code:", code, err);
+    }
+  }
+
   async handleMessage(
-    e: TestEvent | InitEvent<InitTestFrameOptions>,
+    e: CodeEvent | TestEvent | InitEvent<InitTestFrameOptions>,
   ): Promise<void> {
     if (e.data.type === "test") {
       const result = await this.#runTest!(e.data.value);
@@ -192,6 +203,10 @@ ${rawTest}`);
     } else if (e.data.type === "init") {
       await this.init(e.data.value);
       self.parent.postMessage(READY_MESSAGE, "*");
+    } else if (e.data.type === "code") {
+      // This is used to run arbitrary non-test code, such as the afterAll hook.
+      await this.runCode(e.data.value);
+      e.ports[0].postMessage({ type: "code" });
     }
   }
 }

packages/javascript-evaluator/src/javascript-test-evaluator.ts

@@ -8,6 +8,7 @@ import type {
   Fail,
   InitEvent,
   TestEvent,
+  CodeEvent,
   InitWorkerOptions,
 } from "../../shared/src/interfaces/test-evaluator";
 import type { ReadyEvent } from "../../shared/src/interfaces/test-runner";
@@ -107,8 +108,18 @@ ${test};`);
     return this.#runTest!(test);
   }
 
+  async runCode(code: string) {
+    try {
+      await eval(code);
+    } catch (err) {
+      // If the code throws an error, we want to log it to the console
+      // so that it can be debugged.
+      console.error("Error evaluating code:", code, err);
+    }
+  }
+
   async handleMessage(
-    e: TestEvent | InitEvent<InitWorkerOptions>,
+    e: TestEvent | InitEvent<InitWorkerOptions> | CodeEvent,
   ): Promise<void> {
     if (e.data.type === "test") {
       const result = await this.#runTest!(e.data.value);
@@ -117,12 +128,16 @@ ${test};`);
     } else if (e.data.type === "init") {
       this.init(e.data.value);
       postMessage(READY_MESSAGE);
+    } else if (e.data.type === "code") {
+      // This is used to run arbitrary non-test code, such as the afterAll hook.
+      await this.runCode(e.data.value);
+      e.ports[0].postMessage({ type: "code" });
     }
   }
 }
 
 const worker = new JavascriptTestEvaluator();
 
-onmessage = function (e: TestEvent | InitEvent<InitWorkerOptions>) {
+onmessage = function (e: TestEvent | InitEvent<InitWorkerOptions> | CodeEvent) {
   void worker.handleMessage(e);
 };

packages/main/src/index.ts

@@ -50,6 +50,7 @@ export class FCCTestRunner {
       beforeAll?: string;
       beforeEach?: string;
       afterEach?: string;
+      afterAll?: string;
     };
     loadEnzyme?: boolean;
   }) {

packages/main/src/test-runner.ts

@@ -6,6 +6,7 @@ import type {
   InitTestFrameOptions,
   Pass,
   Fail,
+  CodeEvent,
 } from "../../shared/src/interfaces/test-evaluator";
 
 import {
@@ -70,6 +71,7 @@ const hideFrame = (iframe: HTMLIFrameElement) => {
 export class DOMTestRunner implements Runner {
   #testEvaluator: HTMLIFrameElement;
   #script: string;
+  #afterAll?: string;
 
   #createTestEvaluator({ assetPath, script }: RunnerConfig) {
     const iframe = document.createElement("iframe");
@@ -99,6 +101,8 @@ ${hooks.beforeAll}
 </script>`
       : "";
 
+    this.#afterAll = hooks?.afterAll;
+
     const isReady = new Promise((resolve) => {
       const listener = () => {
         this.#testEvaluator.removeEventListener("load", listener);
@@ -155,6 +159,18 @@ ${opts.source}`;
     return result;
   }
 
+  async #runCode(code: string) {
+    const msg: CodeEvent["data"] = {
+      type: "code",
+      value: code,
+    };
+
+    return post({
+      messenger: this.#testEvaluator.contentWindow!,
+      message: msg,
+    });
+  }
+
   async runAllTests(tests: string[]): Promise<(Pass | Fail)[]> {
     const results: (Pass | Fail)[] = [];
 
@@ -164,6 +180,10 @@ ${opts.source}`;
       results.push(result);
     }
 
+    if (this.#afterAll) {
+      await this.#runCode(this.#afterAll);
+    }
+
     return results;
   }
 
@@ -216,6 +236,18 @@ export class WorkerTestRunner implements Runner {
     }
   }
 
+  async #runCode(code: string) {
+    const msg: CodeEvent["data"] = {
+      type: "code",
+      value: code,
+    };
+
+    return post({
+      messenger: this.#testEvaluator,
+      message: msg,
+    });
+  }
+
   async runTest(test: string, timeout = 5000) {
     let terminateTimeoutId: ReturnType<typeof setTimeout> | undefined;
     const terminate = new Promise<Fail>((resolve) => {
@@ -253,6 +285,10 @@ export class WorkerTestRunner implements Runner {
       results.push(result);
     }
 
+    if (this.#opts?.hooks?.afterAll) {
+      await this.#runCode(this.#opts.hooks.afterAll);
+    }
+
     return results;
   }
 

packages/python-evaluator/src/python-test-evaluator.ts

@@ -13,6 +13,7 @@ import {
   Pass,
   TestEvaluator,
   TestEvent,
+  CodeEvent,
 } from "../../shared/src/interfaces/test-evaluator";
 import { ReadyEvent } from "../../shared/src/interfaces/test-runner";
 import { postCloneableMessage } from "../../shared/src/messages";
@@ -216,8 +217,18 @@ class PythonTestEvaluator implements TestEvaluator {
     return this.#runTest!(test);
   }
 
+  async runCode(code: string) {
+    try {
+      await eval(code);
+    } catch (err) {
+      // If the code throws an error, we want to log it to the console
+      // so that it can be debugged.
+      console.error("Error evaluating code:", code, err);
+    }
+  }
+
   async handleMessage(
-    e: TestEvent | InitEvent<InitWorkerOptions>,
+    e: TestEvent | InitEvent<InitWorkerOptions> | CodeEvent,
   ): Promise<void> {
     if (e.data.type === "test") {
       const result = await this.#runTest!(e.data.value);
@@ -226,12 +237,18 @@ class PythonTestEvaluator implements TestEvaluator {
     } else if (e.data.type === "init") {
       await this.init(e.data.value);
       postMessage(READY_MESSAGE);
+    } else if (e.data.type === "code") {
+      // This is used to run arbitrary non-test code, such as the afterAll hook.
+      await this.runCode(e.data.value);
+      e.ports[0].postMessage({ type: "code" });
     }
   }
 }
 
 const worker = new PythonTestEvaluator();
 
-globalThis.onmessage = function (e: TestEvent | InitEvent<InitWorkerOptions>) {
+globalThis.onmessage = function (
+  e: TestEvent | InitEvent<InitWorkerOptions> | CodeEvent,
+) {
   void worker.handleMessage(e);
 };

packages/shared/src/interfaces/test-evaluator.ts

@@ -20,6 +20,7 @@ export interface Fail extends Logged {
 }
 
 export type TestEvent = MessageEvent<{ type: "test"; value: string }>;
+export type CodeEvent = MessageEvent<{ type: "code"; value: string }>;
 export type InitEvent<Data> = MessageEvent<{
   type: "init";
   value: Data;
@@ -36,6 +37,7 @@ export interface InitTestFrameOptions {
     beforeAll?: string;
     beforeEach?: string;
     afterEach?: string;
+    afterAll?: string;
   };
 }
 
@@ -49,6 +51,7 @@ export interface InitWorkerOptions {
     beforeEach?: string;
     beforeAll?: string;
     afterEach?: string;
+    afterAll?: string;
   };
 }
 

packages/tests/integration-tests/index.test.ts

@@ -416,7 +416,7 @@ for(let i = 0; i < 3; i++) {
       ]);
     });
 
-    it("should handle empty array in runAllTests", async () => {
+    it("should be able to handle empty arrays passed to runAllTests", async () => {
       const result = await page.evaluate(async (type) => {
         const runner = await window.FCCTestRunner.createTestRunner({
           type,
@@ -428,6 +428,72 @@ for(let i = 0; i < 3; i++) {
       expect(result).toEqual([]);
     });
 
+    it("should run the afterAll hook after running all tests", async () => {
+      const result = await page.evaluate(async (type) => {
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type,
+          hooks: {
+            beforeAll: "globalThis.x = 0;",
+            afterAll: "globalThis.x = 1;",
+          },
+        });
+
+        // Individual tests do not trigger the afterAll hook, but runAllTests
+        // does.
+        const initial = await runner.runAllTests(["assert.equal(x, 0);"]);
+
+        // Check that afterAll ran
+        const after = await runner.runTest("assert.equal(x, 1);");
+
+        return [initial, after];
+      }, type);
+
+      expect(result).toEqual([[{ pass: true }], { pass: true }]);
+    });
+
+    it("should run the afterAll hook even if a test fails", async () => {
+      const result = await page.evaluate(async (type) => {
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type,
+          hooks: {
+            beforeAll: "globalThis.x = 0;",
+            afterAll: "globalThis.x = 1;",
+          },
+        });
+
+        // Run a failing test
+        await runner.runAllTests([
+          "assert.equal(x, 0);",
+          "assert.equal(1, 2);", // This should fail
+        ]);
+
+        // Check that afterAll ran
+        return runner.runTest("assert.equal(x, 1);");
+      }, type);
+
+      expect(result).toEqual({ pass: true });
+    });
+
+    it("should console.error the error if the afterAll hook fails", async () => {
+      expect.assertions(2);
+      page.once("console", (msg) => {
+        expect(msg.type()).toBe("error");
+      });
+
+      const result = await page.evaluate(async (type) => {
+        const runner = await window.FCCTestRunner.createTestRunner({
+          type,
+          hooks: {
+            afterAll: "throw new Error('afterAll error');",
+          },
+        });
+
+        return runner.runAllTests(["assert.equal(1, 1);"]);
+      }, type);
+
+      expect(result).toEqual([{ pass: true }]);
+    });
+
     it("should support top-level await in tests", async () => {
       const result = await page.evaluate(async (type) => {
         const runner = await window.FCCTestRunner.createTestRunner({