diff --git a/packages/cli/src/index.test.ts b/packages/cli/src/index.test.ts
index fe4b0bc..af4991c 100644
--- a/packages/cli/src/index.test.ts
+++ b/packages/cli/src/index.test.ts
@@ -154,7 +154,7 @@ describe("Mocha Remote CLI", () => {
     const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "mocha-remote-cli-test-"));
     const outFile = path.join(tempDir, "out.txt");
     expect(fs.existsSync(outFile)).equals(false, "Expected the outfile to be missing");
-    const cliProcess = spawnCli("--port", "0", "--context", `endlessLoop,outFile=${outFile}`,  "--", "tsx", "src/test/hanging-client.ts");
+    const cliProcess = spawnCli("--port", "0", "--exit-on-error", "--context", `endlessLoop,outFile=${outFile}`,  "--", "tsx", "src/test/hanging-client.ts");
 
     const outFileContent = await waitForFile(outFile);
     const success = cliProcess.kill("SIGINT"); // Interrupt Mocha Remote CLI
@@ -175,7 +175,7 @@ describe("Mocha Remote CLI", () => {
     const outFile = path.join(tempDir, "out.txt");
     expect(fs.existsSync(outFile)).equals(false, "Expected the outfile to be missing");
     // Using spawnCli because it ignores stdout, which makes the spawnSync hang for some reason
-    const cliProcess = spawnCli("--port", "0", "--context", `outFile=${outFile}`,  "--", "tsx", "src/test/hanging-client.ts");
+    const cliProcess = spawnCli("--port", "0", "--exit-on-error", "--context", `outFile=${outFile}`,  "--", "tsx", "src/test/hanging-client.ts");
     await new Promise<void>(resolve => cliProcess.once("close", resolve));
 
     expect(cliProcess.exitCode).equals(0, "Expected signal of the mocha-remote cli to remain a success");
diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts
index 32943c9..d22408c 100644
--- a/packages/cli/src/index.ts
+++ b/packages/cli/src/index.ts
@@ -23,8 +23,8 @@ function cleanup() {
   cleanupTasks.clear();
   // Execute a chain of promises
   tasks.reduce((previous, task) => previous.then(task), Promise.resolve()).catch(err => {
-    // eslint-disable-next-line no-console
-    console.error(`Failed while cleaning up: ${err}`);
+    /* eslint-disable-next-line no-console */
+    console.error(chalk.red("ERROR"), `Failed to clean up: ${err.message}`);
   });
 }
 
@@ -80,11 +80,9 @@ type ServerOptions = {
 
 // Start the server
 export async function startServer({ log, server, command, exitOnError }: ServerOptions): Promise<void> {
-  cleanupTasks.add(async () => {
-    if (server.listening) {
-      await server.stop();
-      log("🧹 Stopped the server");
-    }
+  cleanupTasks.add(() => {
+    // Ensure that errors during cleanup doesn't result in a failure
+    exitOnError = false;
   });
 
   server.on('started', server => {
@@ -139,15 +137,15 @@ export async function startServer({ log, server, command, exitOnError }: ServerO
     }
   });
 
-  await server.start();
-  // User stopping the process in a terminal
-  process.on("SIGINT", () => {
-    server.stop().then(undefined, err => {
-      /* eslint-disable-next-line no-console */
-      console.error(chalk.red("ERROR"), `Failed to stop server: ${err.message}`);
-    });
+  cleanupTasks.add(async () => {
+    if (server.listening) {
+      await server.stop();
+      log("🧹 Stopped the server");
+    }
   });
 
+  await server.start();
+
   if (command && command.length > 0) {
     const [commandName, ...args] = command;
     const commandProcess = cp.spawn(commandName, args, {