test: improve flaky test (#514)

package.json

@@ -64,6 +64,7 @@
 		"@types/node": "^20.9.4",
 		"@types/react": "^18.2.38",
 		"@types/semver": "^7.5.6",
+		"@types/split2": "^4.2.3",
 		"cachedir": "^2.4.0",
 		"chokidar": "^3.5.3",
 		"clean-pkg-json": "^1.2.0",
@@ -85,6 +86,7 @@
 		"pkgroll": "^2.0.1",
 		"semver": "^7.5.4",
 		"simple-git-hooks": "^2.9.0",
+		"split2": "^4.2.0",
 		"strip-ansi": "^7.1.0",
 		"type-fest": "^4.8.2",
 		"type-flag": "^3.0.0",

src/cli.ts

@@ -15,6 +15,12 @@ import {
 import { isFeatureSupported, testRunnerGlob } from './utils/node-features.js';
 import { createIpcServer } from './utils/ipc/server.js';
 
+// const debug = (...messages: any[]) => {
+// 	if (process.env.DEBUG) {
+// 		console.log(...messages);
+// 	}
+// };
+
 const relaySignals = (
 	childProcess: ChildProcess,
 	ipcSocket: Server,
@@ -31,6 +37,12 @@ const relaySignals = (
 		}
 	});
 
+	/**
+	 * Wait for signal from preflight bindHiddenSignalsHandler
+	 * Ideally the timeout should be as low as possible
+	 * since the child lets the parent know that it received
+	 * the signal
+	 */
 	const waitForSignalFromChild = () => {
 		const p = new Promise<NodeJS.Signals | undefined>((resolve) => {
 			// Aribrary timeout based on flaky tests
@@ -62,12 +74,19 @@ const relaySignals = (
 		 */
 		const signalFromChild = await waitForSignalFromChild();
 
+		// debug({
+		// 	signalFromChild,
+		// });
+
 		/**
 		 * If child didn't receive a signal, it's either because it was
 		 * sent to the parent directly via kill PID or the child is
 		 * unresponsive (e.g. infinite loop). Relay signal to child.
 		 */
 		if (signalFromChild !== signal) {
+			// debug('killing child', {
+			// 	signal,
+			// });
 			childProcess.kill(signal);
 
 			/**

tests/specs/cli.ts

@@ -322,6 +322,7 @@ export default testSuite(({ describe }, node: NodeApis) => {
 							stdout => stdout.includes('READY') && CtrlC,
 							`echo EXIT_CODE: ${isWindows ? '$LastExitCode' : '$?'}\r`,
 						],
+						9000,
 					);
 
 					expectMatchInOrder(output, [
@@ -338,10 +339,12 @@ export default testSuite(({ describe }, node: NodeApis) => {
 						[
 							// Windows doesn't support shebangs
 							`${node.path} ${tsxPath} ${path.join(fixture.path, 'infinite-loop.js')}\r`,
-							stdout => /\d+\r\n/.test(stdout) && CtrlC,
+							stdout => /^\r?\d+$/.test(stdout) && CtrlC,
 							`echo EXIT_CODE: ${isWindows ? '$LastExitCode' : '$?'}\r`,
 						],
+						9000,
 					);
+
 					expect(output).toMatch(/EXIT_CODE:\s+130/);
 				}, 10_000);
 			});

tests/utils/pty-shell/index.ts

@@ -1,6 +1,8 @@
+import { setTimeout } from 'timers/promises';
 import { fileURLToPath } from 'url';
-import { execaNode } from 'execa';
+import { execaNode, type NodeOptions } from 'execa';
 import stripAnsi from 'strip-ansi';
+import split from 'split2';
 
 export const isWindows = process.platform === 'win32';
 const shell = isWindows ? 'powershell.exe' : 'bash';
@@ -9,54 +11,112 @@ const commandCaret = `${isWindows ? '>' : '$'} `;
 type ConditionalStdin = (outChunk: string) => string | false;
 type StdInArray = (string | ConditionalStdin)[];
 
-const getStdin = (
-	stdins: StdInArray,
-): ConditionalStdin | undefined => {
-	const stdin = stdins.shift();
-	return (
-		typeof stdin === 'string'
-			? outChunk => outChunk.includes(commandCaret) && stdin
-			: stdin
-	);
-};
+const throwTimeout = (
+	timeout: number,
+	abortController: AbortController,
+) => (
+	setTimeout(timeout, true, abortController).then(
+		() => {
+			throw new Error(`Timeout: ${timeout}ms`);
+		},
+		() => {},
+	)
+);
 
-export const ptyShell = (
+export const ptyShell = async (
 	stdins: StdInArray,
-) => new Promise<string>((resolve, reject) => {
+	timeout?: number,
+	options?: NodeOptions<'utf8'> & { debug?: string },
+) => {
 	const childProcess = execaNode(
 		fileURLToPath(new URL('node-pty.mjs', import.meta.url)),
 		[shell],
 		{
+			...options,
 			stdio: 'pipe',
 		},
 	);
 
-	childProcess.on('error', reject);
-
-	let currentStdin = getStdin(stdins);
+	let currentStdin = stdins.shift();
 
 	let buffer = Buffer.alloc(0);
 	childProcess.stdout!.on('data', (data) => {
 		buffer = Buffer.concat([buffer, data]);
-		const outString = stripAnsi(data.toString());
+		if (buffer.toString().endsWith(commandCaret)) {
+			if (!currentStdin) {
+				childProcess.kill();
+			} else if (typeof currentStdin === 'string') {
+				if (options?.debug) {
+					console.log({
+						name: options.debug,
+						send: currentStdin,
+					});
+				}
 
-		if (currentStdin) {
-			const stdin = currentStdin(outString);
-			if (stdin) {
-				childProcess.send(stdin);
-				currentStdin = getStdin(stdins);
+				childProcess.send(currentStdin);
+				currentStdin = stdins.shift();
 			}
-		} else if (outString.includes(commandCaret)) {
-			childProcess.kill();
 		}
 	});
 
-	childProcess.stderr!.on('data', (data) => {
-		reject(new Error(stripAnsi(data.toString())));
-	});
+	childProcess.stdout!.pipe(split()).on('data', (line) => {
+		line = stripAnsi(line);
+
+		if (options?.debug) {
+			console.log({ line });
+		}
 
-	childProcess.on('exit', () => {
-		const outString = stripAnsi(buffer.toString());
-		resolve(outString);
+		if (typeof currentStdin === 'function') {
+			const send = currentStdin(line);
+			if (send) {
+				if (options?.debug) {
+					console.log({
+						name: options.debug,
+						send,
+					});
+				}
+				childProcess.send(send);
+				currentStdin = stdins.shift();
+			}
+		}
 	});
-});
+
+	const abortController = new AbortController();
+
+	const promises = [
+		new Promise<void>((resolve, reject) => {
+			childProcess.on('error', reject);
+			childProcess.stderr!.on('data', (data) => {
+				reject(new Error(stripAnsi(data.toString())));
+			});
+			childProcess.on('exit', resolve);
+		}),
+	];
+
+	if (typeof timeout === 'number') {
+		promises.push(
+			throwTimeout(timeout, abortController).catch((error) => {
+				childProcess.kill();
+
+				if (options?.debug) {
+					const outString = stripAnsi(buffer.toString());
+					console.log('Incomplete output', {
+						name: options.debug,
+						outString,
+						stdins,
+					});
+				}
+
+				throw error;
+			}),
+		);
+	}
+
+	try {
+		await Promise.race(promises);
+	} finally {
+		abortController.abort();
+	}
+
+	return stripAnsi(buffer.toString());
+};