From 3e4c26b8f6399b855444145395868a211646d223 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Fri, 22 Jul 2022 13:14:03 +0100 Subject: [PATCH 01/49] feat: process recycling --- docs/Configuration.md | 18 ++ e2e/snapshot/__tests__/snapshot.test_copy.js | 41 ++++ ...s-obsolete-external-snapshots.test.js.snap | 3 + ...emoves-obsolete-external-snapshots.test.js | 3 + .../handle-property-matchers.test.js.snap | 7 + .../handle-property-matchers.test.js | 3 + packages/jest-runner/src/index.ts | 1 + packages/jest-types/src/Config.ts | 4 + .../src/__tests__/process-integration.test.js | 39 +++- .../jest-worker/src/base/BaseWorkerPool.ts | 1 + packages/jest-worker/src/index.ts | 1 + packages/jest-worker/src/types.ts | 20 +- .../src/workers/ChildProcessWorker.ts | 61 +++++- .../__tests__/ChildProcessWorker.test.js | 191 ++++++++++++++++++ .../workers/__tests__/processChild.test.js | 15 ++ .../jest-worker/src/workers/processChild.ts | 20 ++ 16 files changed, 416 insertions(+), 12 deletions(-) create mode 100644 e2e/snapshot/__tests__/snapshot.test_copy.js create mode 100644 e2e/to-match-inline-snapshot/__tests__/__snapshots__/removes-obsolete-external-snapshots.test.js.snap create mode 100644 e2e/to-match-inline-snapshot/__tests__/removes-obsolete-external-snapshots.test.js create mode 100644 e2e/to-match-snapshot/__tests__/__snapshots__/handle-property-matchers.test.js.snap create mode 100644 e2e/to-match-snapshot/__tests__/handle-property-matchers.test.js diff --git a/docs/Configuration.md b/docs/Configuration.md index 75028672b31c..619ec7864857 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -2267,3 +2267,21 @@ This option allows comments in `package.json`. Include the comment text as the v } } ``` + +### `workerIdleMemoryLimit` \[number] + +Specifies the memory limit for workers before they are recycled and is primarily a work-around for [this issue](https://github.com/facebook/jest/issues/11956); + +After the worker has executed a test the memory usage of it is checked. If it exceeds the value specified the worker is killed and restarted. The limit can be specified in 2 ways + +* < 1 - The value is assumed to be a percentage of system memory. So 0.5 sets the memory limit of the worker to half of the total system memory +* > 1 - Assumed to be a fixed byte value + +```js tab +/** @type {import('jest').Config} */ +const config = { + workerIdleMemoryLimit: 0.2, +}; + +module.exports = config; +``` diff --git a/e2e/snapshot/__tests__/snapshot.test_copy.js b/e2e/snapshot/__tests__/snapshot.test_copy.js new file mode 100644 index 000000000000..e05d063e652b --- /dev/null +++ b/e2e/snapshot/__tests__/snapshot.test_copy.js @@ -0,0 +1,41 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + * + */ +'use strict'; + +describe('snapshot', () => { + it('works with plain objects and the title has `escape` characters', () => { + const test = { + a: 1, + b: '2', + c: 'three`', + }; + expect(test).toMatchSnapshot(); + test.d = '4'; + expect(test).toMatchSnapshot(); + }); + + it('is not influenced by previous counter', () => { + const test = { + a: 43, + b: '43', + c: 'fortythree', + }; + expect(test).toMatchSnapshot(); + }); + + it('cannot be used with .not', () => { + expect(() => expect('').not.toMatchSnapshot()).toThrow( + 'Snapshot matchers cannot be used with not', + ); + }); + + // Issue reported here: https://github.com/facebook/jest/issues/2969 + it('works with \\r\\n', () => { + expect('
\r\n
').toMatchSnapshot(); + }); +}); diff --git a/e2e/to-match-inline-snapshot/__tests__/__snapshots__/removes-obsolete-external-snapshots.test.js.snap b/e2e/to-match-inline-snapshot/__tests__/__snapshots__/removes-obsolete-external-snapshots.test.js.snap new file mode 100644 index 000000000000..ac6dd2b9f9fa --- /dev/null +++ b/e2e/to-match-inline-snapshot/__tests__/__snapshots__/removes-obsolete-external-snapshots.test.js.snap @@ -0,0 +1,3 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`removes obsolete external snapshots 1`] = `"1"`; diff --git a/e2e/to-match-inline-snapshot/__tests__/removes-obsolete-external-snapshots.test.js b/e2e/to-match-inline-snapshot/__tests__/removes-obsolete-external-snapshots.test.js new file mode 100644 index 000000000000..87029c5b4bb7 --- /dev/null +++ b/e2e/to-match-inline-snapshot/__tests__/removes-obsolete-external-snapshots.test.js @@ -0,0 +1,3 @@ +test('removes obsolete external snapshots', () => { + expect('1').toMatchInlineSnapshot(); +}); \ No newline at end of file diff --git a/e2e/to-match-snapshot/__tests__/__snapshots__/handle-property-matchers.test.js.snap b/e2e/to-match-snapshot/__tests__/__snapshots__/handle-property-matchers.test.js.snap new file mode 100644 index 000000000000..b13b35707eae --- /dev/null +++ b/e2e/to-match-snapshot/__tests__/__snapshots__/handle-property-matchers.test.js.snap @@ -0,0 +1,7 @@ +// Jest Snapshot v1, https://goo.gl/fbAQLP + +exports[`handles property matchers 1`] = ` +Object { + "createdAt": Any, +} +`; diff --git a/e2e/to-match-snapshot/__tests__/handle-property-matchers.test.js b/e2e/to-match-snapshot/__tests__/handle-property-matchers.test.js new file mode 100644 index 000000000000..fa8f626b773d --- /dev/null +++ b/e2e/to-match-snapshot/__tests__/handle-property-matchers.test.js @@ -0,0 +1,3 @@ +test('handles property matchers', () => { + expect({createdAt: new Date()}).toMatchSnapshot({createdAt: expect.any(Date)}); +}); \ No newline at end of file diff --git a/packages/jest-runner/src/index.ts b/packages/jest-runner/src/index.ts index d43f899f796d..3debd22e1b2e 100644 --- a/packages/jest-runner/src/index.ts +++ b/packages/jest-runner/src/index.ts @@ -107,6 +107,7 @@ export default class TestRunner extends EmittingTestRunner { const worker = new Worker(require.resolve('./testWorker'), { exposedMethods: ['worker'], forkOptions: {serialization: 'json', stdio: 'pipe'}, + idleMemoryLimit: this._globalConfig.workerIdleMemoryLimit, maxRetries: 3, numWorkers: this._globalConfig.maxWorkers, setupArgs: [{serializableResolvers: Array.from(resolvers.values())}], diff --git a/packages/jest-types/src/Config.ts b/packages/jest-types/src/Config.ts index e7854413bfd6..87a4eb29bdc2 100644 --- a/packages/jest-types/src/Config.ts +++ b/packages/jest-types/src/Config.ts @@ -327,6 +327,7 @@ export type InitialOptions = Partial<{ watchAll: boolean; watchman: boolean; watchPlugins: Array]>; + workerIdleMemoryLimit: number; }>; export type SnapshotUpdateState = 'all' | 'new' | 'none'; @@ -420,6 +421,7 @@ export type GlobalConfig = { path: string; config: Record; }> | null; + workerIdleMemoryLimit?: number; }; export type ProjectConfig = { @@ -478,6 +480,7 @@ export type ProjectConfig = { transformIgnorePatterns: Array; watchPathIgnorePatterns: Array; unmockedModulePathPatterns?: Array; + workerIdleMemoryLimit?: number; }; export type Argv = Arguments< @@ -571,5 +574,6 @@ export type Argv = Arguments< watchAll: boolean; watchman: boolean; watchPathIgnorePatterns: Array; + workerIdleMemoryLimit: number; }> >; diff --git a/packages/jest-worker/src/__tests__/process-integration.test.js b/packages/jest-worker/src/__tests__/process-integration.test.js index d5e3f0950504..6546fab8cb03 100644 --- a/packages/jest-worker/src/__tests__/process-integration.test.js +++ b/packages/jest-worker/src/__tests__/process-integration.test.js @@ -6,7 +6,12 @@ */ import EventEmitter from 'events'; -import {CHILD_MESSAGE_CALL, PARENT_MESSAGE_OK} from '../types'; +import { + CHILD_MESSAGE_CALL, + CHILD_MESSAGE_MEM_USAGE, + PARENT_MESSAGE_OK, + WorkerFarmOptions, +} from '../types'; let Farm; let mockForkedProcesses; @@ -175,4 +180,36 @@ describe('Jest Worker Integration', () => { expect(await promise1).toBe('worker-1'); expect(await promise2).toBe('worker-2'); }); + + it('should check for memory limits', async () => { + /** @type WorkerFarmOptions */ + const options = { + computeWorkerKey: () => '1234567890abcdef', + exposedMethods: ['foo', 'bar'], + idleMemoryLimit: 0.4, + numWorkers: 2, + }; + + const farm = new Farm('/tmp/baz.js', options); + + // Send a call to the farm + const promise0 = farm.foo('param-0'); + + // Send different responses for each call (from the same child). + replySuccess(0, 'worker-0'); + + // Check that all the calls have been received by the same child). + // We're not using the assertCallsToChild helper because we need to check + // for other send types. + expect(mockForkedProcesses[0].send).toHaveBeenCalledTimes(3); + expect(mockForkedProcesses[0].send.mock.calls[1][0]).toEqual([ + CHILD_MESSAGE_CALL, + true, + 'foo', + ['param-0'], + ]); + expect(mockForkedProcesses[0].send.mock.calls[2][0]).toEqual([ + CHILD_MESSAGE_MEM_USAGE, + ]); + }); }); diff --git a/packages/jest-worker/src/base/BaseWorkerPool.ts b/packages/jest-worker/src/base/BaseWorkerPool.ts index da6e57d1b707..801e30eba957 100644 --- a/packages/jest-worker/src/base/BaseWorkerPool.ts +++ b/packages/jest-worker/src/base/BaseWorkerPool.ts @@ -40,6 +40,7 @@ export default class BaseWorkerPool { for (let i = 0; i < options.numWorkers; i++) { const workerOptions: WorkerOptions = { forkOptions, + idleMemoryLimit: this._options.idleMemoryLimit, maxRetries, resourceLimits, setupArgs, diff --git a/packages/jest-worker/src/index.ts b/packages/jest-worker/src/index.ts index c540c6a5fabe..e8a531f6458b 100644 --- a/packages/jest-worker/src/index.ts +++ b/packages/jest-worker/src/index.ts @@ -96,6 +96,7 @@ export class Worker { const workerPoolOptions: WorkerPoolOptions = { enableWorkerThreads: this._options.enableWorkerThreads ?? false, forkOptions: this._options.forkOptions ?? {}, + idleMemoryLimit: this._options.idleMemoryLimit, maxRetries: this._options.maxRetries ?? 3, numWorkers: this._options.numWorkers ?? Math.max(cpus().length - 1, 1), resourceLimits: this._options.resourceLimits ?? {}, diff --git a/packages/jest-worker/src/types.ts b/packages/jest-worker/src/types.ts index 7bf383a4d7cc..fb51ecc6d8fb 100644 --- a/packages/jest-worker/src/types.ts +++ b/packages/jest-worker/src/types.ts @@ -36,11 +36,13 @@ export type WorkerModule = { export const CHILD_MESSAGE_INITIALIZE = 0; export const CHILD_MESSAGE_CALL = 1; export const CHILD_MESSAGE_END = 2; +export const CHILD_MESSAGE_MEM_USAGE = 3; export const PARENT_MESSAGE_OK = 0; export const PARENT_MESSAGE_CLIENT_ERROR = 1; export const PARENT_MESSAGE_SETUP_ERROR = 2; export const PARENT_MESSAGE_CUSTOM = 3; +export const PARENT_MESSAGE_MEM_USAGE = 4; export type PARENT_MESSAGE_ERROR = | typeof PARENT_MESSAGE_CLIENT_ERROR @@ -122,6 +124,7 @@ export type WorkerFarmOptions = { options?: WorkerPoolOptions, ) => WorkerPoolInterface; workerSchedulingPolicy?: WorkerSchedulingPolicy; + idleMemoryLimit?: number; }; export type WorkerPoolOptions = { @@ -131,6 +134,7 @@ export type WorkerPoolOptions = { maxRetries: number; numWorkers: number; enableWorkerThreads: boolean; + idleMemoryLimit?: number; }; export type WorkerOptions = { @@ -141,6 +145,7 @@ export type WorkerOptions = { workerId: number; workerData?: unknown; workerPath: string; + idleMemoryLimit?: number; }; // Messages passed from the parent to the children. @@ -174,10 +179,15 @@ export type ChildMessageEnd = [ boolean, // processed ]; +export type ChildMessageMemUsage = [ + typeof CHILD_MESSAGE_MEM_USAGE, // type +]; + export type ChildMessage = | ChildMessageInitialize | ChildMessageCall - | ChildMessageEnd; + | ChildMessageEnd + | ChildMessageMemUsage; // Messages passed from the children to the parent. @@ -191,6 +201,11 @@ export type ParentMessageOk = [ unknown, // result ]; +export type ParentMessageMemUsage = [ + typeof PARENT_MESSAGE_MEM_USAGE, // type + number, // used memory in bytes +]; + export type ParentMessageError = [ PARENT_MESSAGE_ERROR, // type string, // constructor @@ -202,7 +217,8 @@ export type ParentMessageError = [ export type ParentMessage = | ParentMessageOk | ParentMessageError - | ParentMessageCustom; + | ParentMessageCustom + | ParentMessageMemUsage; // Queue types. diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index 9381310a30dc..242147aa21da 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -6,17 +6,20 @@ */ import {ChildProcess, fork} from 'child_process'; +import {totalmem} from 'os'; import {PassThrough} from 'stream'; import mergeStream = require('merge-stream'); import {stdout as stdoutSupportsColor} from 'supports-color'; import { CHILD_MESSAGE_INITIALIZE, + CHILD_MESSAGE_MEM_USAGE, ChildMessage, OnCustomMessage, OnEnd, OnStart, PARENT_MESSAGE_CLIENT_ERROR, PARENT_MESSAGE_CUSTOM, + PARENT_MESSAGE_MEM_USAGE, PARENT_MESSAGE_OK, PARENT_MESSAGE_SETUP_ERROR, ParentMessage, @@ -64,6 +67,9 @@ export default class ChildProcessWorker implements WorkerInterface { private _exitPromise: Promise; private _resolveExitPromise!: () => void; + private _childIdleMemoryUsage: number | null; + private _childIdleMemoryUsageLimit: number | null; + private _restarting = false; constructor(options: WorkerOptions) { this._options = options; @@ -73,6 +79,8 @@ export default class ChildProcessWorker implements WorkerInterface { this._fakeStream = null; this._stdout = null; this._stderr = null; + this._childIdleMemoryUsage = null; + this._childIdleMemoryUsageLimit = options.idleMemoryLimit || null; this._exitPromise = new Promise(resolve => { this._resolveExitPromise = resolve; @@ -82,6 +90,8 @@ export default class ChildProcessWorker implements WorkerInterface { } initialize(): void { + this._restarting = false; + const forceColor = stdoutSupportsColor ? {FORCE_COLOR: '1'} : {}; const child = fork(require.resolve('./processChild'), [], { cwd: process.cwd(), @@ -201,17 +211,44 @@ export default class ChildProcessWorker implements WorkerInterface { case PARENT_MESSAGE_CUSTOM: this._onCustomMessage(response[1]); break; + + case PARENT_MESSAGE_MEM_USAGE: + this._childIdleMemoryUsage = response[1]; + + this._performRestartIfRequired(); + + break; + default: throw new TypeError(`Unexpected response from worker: ${response[0]}`); } } + private _performRestartIfRequired(): void { + let limit = this._childIdleMemoryUsageLimit; + + if (limit && limit > 0 && limit <= 1) { + limit = Math.floor(totalmem() * limit); + } + + if ( + limit && + this._childIdleMemoryUsage && + this._childIdleMemoryUsage > limit + ) { + this._restarting = true; + + this.killChild(); + } + } + private _onExit(exitCode: number | null, signal: NodeJS.Signals | null) { if ( - exitCode !== 0 && - exitCode !== null && - exitCode !== SIGTERM_EXIT_CODE && - exitCode !== SIGKILL_EXIT_CODE + (exitCode !== 0 && + exitCode !== null && + exitCode !== SIGTERM_EXIT_CODE && + exitCode !== SIGKILL_EXIT_CODE) || + this._restarting ) { this.initialize(); @@ -241,7 +278,12 @@ export default class ChildProcessWorker implements WorkerInterface { onCustomMessage: OnCustomMessage, ): void { onProcessStart(this); + this._onProcessEnd = (...args) => { + if (this._childIdleMemoryUsageLimit) { + this._child.send([CHILD_MESSAGE_MEM_USAGE]); + } + // Clean the request to avoid sending past requests to workers that fail // while waiting for a new request (timers, unhandled rejections...) this._request = null; @@ -260,12 +302,13 @@ export default class ChildProcessWorker implements WorkerInterface { return this._exitPromise; } - forceExit(): void { + killChild(): NodeJS.Timeout { this._child.kill('SIGTERM'); - const sigkillTimeout = setTimeout( - () => this._child.kill('SIGKILL'), - SIGKILL_DELAY, - ); + return setTimeout(() => this._child.kill('SIGKILL'), SIGKILL_DELAY); + } + + forceExit(): void { + const sigkillTimeout = this.killChild(); this._exitPromise.then(() => clearTimeout(sigkillTimeout)); } diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js index 6bcfcc0ff654..4956cbf96f6b 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js @@ -12,9 +12,12 @@ import supportsColor from 'supports-color'; import { CHILD_MESSAGE_CALL, CHILD_MESSAGE_INITIALIZE, + CHILD_MESSAGE_MEM_USAGE, PARENT_MESSAGE_CLIENT_ERROR, PARENT_MESSAGE_CUSTOM, + PARENT_MESSAGE_MEM_USAGE, PARENT_MESSAGE_OK, + WorkerOptions, } from '../../types'; jest.useFakeTimers(); @@ -23,9 +26,12 @@ let Worker; let forkInterface; let childProcess; let originalExecArgv; +let totalmem; beforeEach(() => { jest.mock('child_process'); + jest.mock('os'); + originalExecArgv = process.execArgv; childProcess = require('child_process'); @@ -40,6 +46,8 @@ beforeEach(() => { return forkInterface; }); + totalmem = require('os').totalmem; + Worker = require('../ChildProcessWorker').default; }); @@ -449,3 +457,186 @@ it('does not send SIGKILL if SIGTERM exited the process', async () => { jest.runAllTimers(); expect(forkInterface.kill.mock.calls).toEqual([['SIGTERM']]); }); + +it('should check for memory limits and not restart if under percentage limit', async () => { + const memoryConfig = { + limit: 0.2, + processHeap: 2500, + totalMem: 16000, + }; + + /** @type WorkerOptions */ + const options = { + forkOptions: {}, + idleMemoryLimit: memoryConfig.limit, + maxRetries: 3, + workerPath: '/tmp/foo', + }; + const worker = new Worker(options); + + const onProcessStart = jest.fn(); + const onProcessEnd = jest.fn(); + const onCustomMessage = jest.fn(); + + worker.send( + [CHILD_MESSAGE_CALL, false, 'foo', []], + onProcessStart, + onProcessEnd, + onCustomMessage, + ); + + // Only onProcessStart has been called + expect(onProcessStart).toHaveBeenCalledTimes(1); + expect(onProcessEnd).not.toHaveBeenCalled(); + expect(onCustomMessage).not.toHaveBeenCalled(); + + // then first call replies... + forkInterface.emit('message', [PARENT_MESSAGE_OK]); + + expect(onProcessEnd).toHaveBeenCalledTimes(1); + + // This is the initalization call. + expect(forkInterface.send.mock.calls[0][0]).toEqual([ + CHILD_MESSAGE_INITIALIZE, + false, + '/tmp/foo', + undefined, + ]); + + // This is the child message + expect(forkInterface.send.mock.calls[1][0]).toEqual([ + CHILD_MESSAGE_CALL, + false, + 'foo', + [], + ]); + + // This is the subsequent call to get memory usage + expect(forkInterface.send.mock.calls[2][0]).toEqual([ + CHILD_MESSAGE_MEM_USAGE, + ]); + + totalmem.mockReturnValue(memoryConfig.totalMem); + + forkInterface.emit('message', [ + PARENT_MESSAGE_MEM_USAGE, + memoryConfig.processHeap, + ]); + + expect(totalmem).toHaveBeenCalledTimes(1); + expect(forkInterface.kill).not.toHaveBeenCalled(); +}); + +it('should check for memory limits and not restart if under absolute limit', async () => { + const memoryConfig = { + limit: 2600, + processHeap: 2500, + totalMem: 16000, + }; + + /** @type WorkerOptions */ + const options = { + forkOptions: {}, + idleMemoryLimit: memoryConfig.limit, + maxRetries: 3, + workerPath: '/tmp/foo', + }; + const worker = new Worker(options); + + const onProcessStart = jest.fn(); + const onProcessEnd = jest.fn(); + const onCustomMessage = jest.fn(); + + worker.send( + [CHILD_MESSAGE_CALL, false, 'foo', []], + onProcessStart, + onProcessEnd, + onCustomMessage, + ); + + totalmem.mockReturnValue(memoryConfig.totalMem); + + forkInterface.emit('message', [ + PARENT_MESSAGE_MEM_USAGE, + memoryConfig.processHeap, + ]); + + expect(totalmem).not.toHaveBeenCalled(); + expect(forkInterface.kill).not.toHaveBeenCalled(); +}); + +it('should check for memory limits and restart if above percentage limit', async () => { + const memoryConfig = { + limit: 0.01, + processHeap: 2500, + totalMem: 16000, + }; + + /** @type WorkerOptions */ + const options = { + forkOptions: {}, + idleMemoryLimit: memoryConfig.limit, + maxRetries: 3, + workerPath: '/tmp/foo', + }; + const worker = new Worker(options); + + const onProcessStart = jest.fn(); + const onProcessEnd = jest.fn(); + const onCustomMessage = jest.fn(); + + worker.send( + [CHILD_MESSAGE_CALL, false, 'foo', []], + onProcessStart, + onProcessEnd, + onCustomMessage, + ); + + totalmem.mockReturnValue(memoryConfig.totalMem); + + forkInterface.emit('message', [ + PARENT_MESSAGE_MEM_USAGE, + memoryConfig.processHeap, + ]); + + expect(totalmem).toHaveBeenCalledTimes(1); + expect(forkInterface.kill).toHaveBeenCalledTimes(1); +}); + +it('should check for memory limits and restart if above absolute limit', async () => { + const memoryConfig = { + limit: 2000, + processHeap: 2500, + totalMem: 16000, + }; + + /** @type WorkerOptions */ + const options = { + forkOptions: {}, + idleMemoryLimit: memoryConfig.limit, + maxRetries: 3, + workerPath: '/tmp/foo', + }; + const worker = new Worker(options); + + const onProcessStart = jest.fn(); + const onProcessEnd = jest.fn(); + const onCustomMessage = jest.fn(); + + worker.send( + [CHILD_MESSAGE_CALL, false, 'foo', []], + onProcessStart, + onProcessEnd, + onCustomMessage, + ); + + totalmem.mockReturnValue(memoryConfig.totalMem); + + forkInterface.emit('message', [ + PARENT_MESSAGE_MEM_USAGE, + memoryConfig.processHeap, + ]); + + expect(totalmem).not.toHaveBeenCalled(); + expect(forkInterface.kill).toHaveBeenCalledTimes(1); +}); diff --git a/packages/jest-worker/src/workers/__tests__/processChild.test.js b/packages/jest-worker/src/workers/__tests__/processChild.test.js index 55b29107bdf7..b1520765f1d5 100644 --- a/packages/jest-worker/src/workers/__tests__/processChild.test.js +++ b/packages/jest-worker/src/workers/__tests__/processChild.test.js @@ -18,7 +18,9 @@ import { CHILD_MESSAGE_CALL, CHILD_MESSAGE_END, CHILD_MESSAGE_INITIALIZE, + CHILD_MESSAGE_MEM_USAGE, PARENT_MESSAGE_CLIENT_ERROR, + PARENT_MESSAGE_MEM_USAGE, PARENT_MESSAGE_OK, } from '../../types'; @@ -141,6 +143,19 @@ it('lazily requires the file', () => { expect(initializeParm).toBe(undefined); }); +it('should return memory usage', () => { + process.send = jest.fn(); + + expect(mockCount).toBe(0); + + process.emit('message', [CHILD_MESSAGE_MEM_USAGE]); + + expect(process.send.mock.calls[0][0]).toEqual([ + PARENT_MESSAGE_MEM_USAGE, + expect.any(Number), + ]); +}); + it('calls initialize with the correct arguments', () => { expect(mockCount).toBe(0); diff --git a/packages/jest-worker/src/workers/processChild.ts b/packages/jest-worker/src/workers/processChild.ts index dda7938aa415..3302843cf2f4 100644 --- a/packages/jest-worker/src/workers/processChild.ts +++ b/packages/jest-worker/src/workers/processChild.ts @@ -9,12 +9,15 @@ import { CHILD_MESSAGE_CALL, CHILD_MESSAGE_END, CHILD_MESSAGE_INITIALIZE, + CHILD_MESSAGE_MEM_USAGE, ChildMessageCall, ChildMessageInitialize, PARENT_MESSAGE_CLIENT_ERROR, PARENT_MESSAGE_ERROR, + PARENT_MESSAGE_MEM_USAGE, PARENT_MESSAGE_OK, PARENT_MESSAGE_SETUP_ERROR, + ParentMessageMemUsage, } from '../types'; type UnknownFunction = (...args: Array) => unknown | Promise; @@ -53,6 +56,10 @@ const messageListener: NodeJS.MessageListener = (request: any) => { end(); break; + case CHILD_MESSAGE_MEM_USAGE: + reportMemoryUsage(); + break; + default: throw new TypeError( `Unexpected request from parent process: ${request[0]}`, @@ -77,6 +84,19 @@ function reportInitializeError(error: Error) { return reportError(error, PARENT_MESSAGE_SETUP_ERROR); } +function reportMemoryUsage() { + if (!process || !process.send) { + throw new Error('Child can only be used on a forked process'); + } + + const msg: ParentMessageMemUsage = [ + PARENT_MESSAGE_MEM_USAGE, + process.memoryUsage().heapUsed, + ]; + + process.send(msg); +} + function reportError(error: Error, type: PARENT_MESSAGE_ERROR) { if (!process || !process.send) { throw new Error('Child can only be used on a forked process'); From 833e429daa4d76637d602a72b00dfc987ae5e3e6 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Fri, 22 Jul 2022 13:20:14 +0100 Subject: [PATCH 02/49] chore: changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5c0c99c16f5..5a64da16c611 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - `[jest-config]` [**BREAKING**] Make `snapshotFormat` default to `escapeString: false` and `printBasicPrototype: false` ([#13036](https://github.com/facebook/jest/pull/13036)) - `[jest-environment-jsdom]` [**BREAKING**] Upgrade to `jsdom@20` ([#13037](https://github.com/facebook/jest/pull/13037)) - `[pretty-format]` [**BREAKING**] Remove `ConvertAnsi` plugin in favour of `jest-serializer-ansi-escapes` ([#13040](https://github.com/facebook/jest/pull/13040)) +- `[jest-worker]` Adds `workerIdleMemoryLimit` option which is used as a check for worker memory leaks >= Node 16.11.0 and recycles child workers as required. ([#13056](https://github.com/facebook/jest/pull/13056)) ### Fixes From 7da8127a505b6aa7c008e138a71075921fe5df02 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Fri, 22 Jul 2022 13:28:46 +0100 Subject: [PATCH 03/49] chore: remove temp files --- e2e/snapshot/__tests__/snapshot.test_copy.js | 41 ------------------- ...s-obsolete-external-snapshots.test.js.snap | 3 -- ...emoves-obsolete-external-snapshots.test.js | 3 -- .../handle-property-matchers.test.js.snap | 7 ---- .../handle-property-matchers.test.js | 3 -- 5 files changed, 57 deletions(-) delete mode 100644 e2e/snapshot/__tests__/snapshot.test_copy.js delete mode 100644 e2e/to-match-inline-snapshot/__tests__/__snapshots__/removes-obsolete-external-snapshots.test.js.snap delete mode 100644 e2e/to-match-inline-snapshot/__tests__/removes-obsolete-external-snapshots.test.js delete mode 100644 e2e/to-match-snapshot/__tests__/__snapshots__/handle-property-matchers.test.js.snap delete mode 100644 e2e/to-match-snapshot/__tests__/handle-property-matchers.test.js diff --git a/e2e/snapshot/__tests__/snapshot.test_copy.js b/e2e/snapshot/__tests__/snapshot.test_copy.js deleted file mode 100644 index e05d063e652b..000000000000 --- a/e2e/snapshot/__tests__/snapshot.test_copy.js +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - * - */ -'use strict'; - -describe('snapshot', () => { - it('works with plain objects and the title has `escape` characters', () => { - const test = { - a: 1, - b: '2', - c: 'three`', - }; - expect(test).toMatchSnapshot(); - test.d = '4'; - expect(test).toMatchSnapshot(); - }); - - it('is not influenced by previous counter', () => { - const test = { - a: 43, - b: '43', - c: 'fortythree', - }; - expect(test).toMatchSnapshot(); - }); - - it('cannot be used with .not', () => { - expect(() => expect('').not.toMatchSnapshot()).toThrow( - 'Snapshot matchers cannot be used with not', - ); - }); - - // Issue reported here: https://github.com/facebook/jest/issues/2969 - it('works with \\r\\n', () => { - expect('
\r\n
').toMatchSnapshot(); - }); -}); diff --git a/e2e/to-match-inline-snapshot/__tests__/__snapshots__/removes-obsolete-external-snapshots.test.js.snap b/e2e/to-match-inline-snapshot/__tests__/__snapshots__/removes-obsolete-external-snapshots.test.js.snap deleted file mode 100644 index ac6dd2b9f9fa..000000000000 --- a/e2e/to-match-inline-snapshot/__tests__/__snapshots__/removes-obsolete-external-snapshots.test.js.snap +++ /dev/null @@ -1,3 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`removes obsolete external snapshots 1`] = `"1"`; diff --git a/e2e/to-match-inline-snapshot/__tests__/removes-obsolete-external-snapshots.test.js b/e2e/to-match-inline-snapshot/__tests__/removes-obsolete-external-snapshots.test.js deleted file mode 100644 index 87029c5b4bb7..000000000000 --- a/e2e/to-match-inline-snapshot/__tests__/removes-obsolete-external-snapshots.test.js +++ /dev/null @@ -1,3 +0,0 @@ -test('removes obsolete external snapshots', () => { - expect('1').toMatchInlineSnapshot(); -}); \ No newline at end of file diff --git a/e2e/to-match-snapshot/__tests__/__snapshots__/handle-property-matchers.test.js.snap b/e2e/to-match-snapshot/__tests__/__snapshots__/handle-property-matchers.test.js.snap deleted file mode 100644 index b13b35707eae..000000000000 --- a/e2e/to-match-snapshot/__tests__/__snapshots__/handle-property-matchers.test.js.snap +++ /dev/null @@ -1,7 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`handles property matchers 1`] = ` -Object { - "createdAt": Any, -} -`; diff --git a/e2e/to-match-snapshot/__tests__/handle-property-matchers.test.js b/e2e/to-match-snapshot/__tests__/handle-property-matchers.test.js deleted file mode 100644 index fa8f626b773d..000000000000 --- a/e2e/to-match-snapshot/__tests__/handle-property-matchers.test.js +++ /dev/null @@ -1,3 +0,0 @@ -test('handles property matchers', () => { - expect({createdAt: new Date()}).toMatchSnapshot({createdAt: expect.any(Date)}); -}); \ No newline at end of file From 40a23cf135e345ea70008752d65dc14e5bb163f8 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 25 Jul 2022 14:46:09 +0100 Subject: [PATCH 04/49] test: more complete functional testing --- .gitignore | 4 + jest.config.mjs | 7 +- .../src/__tests__/process-integration.test.js | 1 + .../src/workers/ChildProcessWorker.ts | 94 ++++++++-- .../__tests__/ChildProcessWorker.test.js | 34 +--- .../ChildProcessWorkerEdgeCases.test.js | 169 ++++++++++++++++++ .../ChildProcessWorkerEdgeCasesWorker.js | 19 ++ 7 files changed, 281 insertions(+), 47 deletions(-) create mode 100644 packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js create mode 100644 packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js diff --git a/.gitignore b/.gitignore index 7dfb5101b363..7d691794efeb 100644 --- a/.gitignore +++ b/.gitignore @@ -53,3 +53,7 @@ api-extractor.json **/.pnp.* crowdin-cli.jar + +# We don't want this temp file +packages/jest-worker/src/workers/processChild.js +packages/jest-worker/src/types.js diff --git a/jest.config.mjs b/jest.config.mjs index 39ff9114a445..646d09cc0baf 100644 --- a/jest.config.mjs +++ b/jest.config.mjs @@ -66,6 +66,7 @@ export default { '/packages/jest-snapshot/src/__tests__/plugins', '/packages/jest-snapshot/src/__tests__/fixtures/', '/packages/jest-validate/src/__tests__/fixtures/', + '/packages/jest-worker/src/workers/__tests__/__fixtures__/', '/e2e/__tests__/iterator-to-null-test.ts', '/e2e/__tests__/tsIntegration.test.ts', // this test needs types to be build, it runs in a separate CI job through `jest.config.ts.mjs` ], @@ -73,7 +74,11 @@ export default { transform: { '\\.[jt]sx?$': require.resolve('babel-jest'), }, - watchPathIgnorePatterns: ['coverage'], + watchPathIgnorePatterns: [ + 'coverage', + '/packages/jest-worker/src/workers/processChild.js', + '/packages/jest-worker/src/types.js', + ], watchPlugins: [ require.resolve('jest-watch-typeahead/filename'), require.resolve('jest-watch-typeahead/testname'), diff --git a/packages/jest-worker/src/__tests__/process-integration.test.js b/packages/jest-worker/src/__tests__/process-integration.test.js index 6546fab8cb03..2674ab21c27f 100644 --- a/packages/jest-worker/src/__tests__/process-integration.test.js +++ b/packages/jest-worker/src/__tests__/process-integration.test.js @@ -20,6 +20,7 @@ function mockBuildForkedProcess() { const mockChild = new EventEmitter(); mockChild.send = jest.fn(); + mockChild.connected = true; return mockChild; } diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index 242147aa21da..4e27519ea473 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -5,7 +5,7 @@ * LICENSE file in the root directory of this source tree. */ -import {ChildProcess, fork} from 'child_process'; +import {ChildProcess, ForkOptions, fork} from 'child_process'; import {totalmem} from 'os'; import {PassThrough} from 'stream'; import mergeStream = require('merge-stream'); @@ -67,9 +67,14 @@ export default class ChildProcessWorker implements WorkerInterface { private _exitPromise: Promise; private _resolveExitPromise!: () => void; + + private _memoryUsagePromise: Promise | undefined; + private _resolveMemoryUsage: ((arg0: number) => void) | undefined; + private _childIdleMemoryUsage: number | null; private _childIdleMemoryUsageLimit: number | null; private _restarting = false; + private _memoryUsageCheck = false; constructor(options: WorkerOptions) { this._options = options; @@ -93,7 +98,7 @@ export default class ChildProcessWorker implements WorkerInterface { this._restarting = false; const forceColor = stdoutSupportsColor ? {FORCE_COLOR: '1'} : {}; - const child = fork(require.resolve('./processChild'), [], { + const options: ForkOptions = { cwd: process.cwd(), env: { ...process.env, @@ -106,7 +111,9 @@ export default class ChildProcessWorker implements WorkerInterface { serialization: 'advanced', silent: true, ...this._options.forkOptions, - }); + }; + + const child = fork(require.resolve('./processChild'), [], options); if (child.stdout) { if (!this._stdout) { @@ -215,6 +222,13 @@ export default class ChildProcessWorker implements WorkerInterface { case PARENT_MESSAGE_MEM_USAGE: this._childIdleMemoryUsage = response[1]; + if (this._resolveMemoryUsage) { + this._resolveMemoryUsage(response[1]); + + this._resolveMemoryUsage = undefined; + this._memoryUsagePromise = undefined; + } + this._performRestartIfRequired(); break; @@ -225,20 +239,24 @@ export default class ChildProcessWorker implements WorkerInterface { } private _performRestartIfRequired(): void { - let limit = this._childIdleMemoryUsageLimit; + if (this._memoryUsageCheck) { + this._memoryUsageCheck = false; - if (limit && limit > 0 && limit <= 1) { - limit = Math.floor(totalmem() * limit); - } + let limit = this._childIdleMemoryUsageLimit; - if ( - limit && - this._childIdleMemoryUsage && - this._childIdleMemoryUsage > limit - ) { - this._restarting = true; + if (limit && limit > 0 && limit <= 1) { + limit = Math.floor(totalmem() * limit); + } + + if ( + limit && + this._childIdleMemoryUsage && + this._childIdleMemoryUsage > limit + ) { + this._restarting = true; - this.killChild(); + this.killChild(); + } } } @@ -280,8 +298,8 @@ export default class ChildProcessWorker implements WorkerInterface { onProcessStart(this); this._onProcessEnd = (...args) => { - if (this._childIdleMemoryUsageLimit) { - this._child.send([CHILD_MESSAGE_MEM_USAGE]); + if (this._childIdleMemoryUsageLimit && this._child.connected) { + this.checkMemoryUsage(); } // Clean the request to avoid sending past requests to workers that fail @@ -316,6 +334,15 @@ export default class ChildProcessWorker implements WorkerInterface { return this._options.workerId; } + /** + * Gets the process id of the worker. + * + * @returns Process id. + */ + getWorkerPid(): number { + return this._child.pid; + } + getStdout(): NodeJS.ReadableStream | null { return this._stdout; } @@ -324,6 +351,41 @@ export default class ChildProcessWorker implements WorkerInterface { return this._stderr; } + /** + * Gets the last reported memory usage. + * + * @returns Memory usage in bytes. + */ + getMemoryUsage(): Promise { + if (!this._memoryUsagePromise) { + this._memoryUsagePromise = new Promise((resolve, reject) => { + this._resolveMemoryUsage = resolve; + + if (!this._child.connected) { + reject(new Error('Child process is not running.')); + } + }); + + this._child.send([CHILD_MESSAGE_MEM_USAGE]); + } + + return this._memoryUsagePromise; + } + + /** + * Gets updated memory usage and restarts if required + */ + checkMemoryUsage(): void { + if (this._childIdleMemoryUsageLimit) { + this._memoryUsageCheck = true; + this._child.send([CHILD_MESSAGE_MEM_USAGE]); + } else { + console.warn( + 'Memory usage of workers can only be checked if a limit is set', + ); + } + } + private _getFakeStream() { if (!this._fakeStream) { this._fakeStream = new PassThrough(); diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js index 4956cbf96f6b..38f3a2ced4d1 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js @@ -37,6 +37,7 @@ beforeEach(() => { childProcess = require('child_process'); childProcess.fork.mockImplementation(() => { forkInterface = Object.assign(new EventEmitter(), { + connected: true, kill: jest.fn(), send: jest.fn(), stderr: new PassThrough(), @@ -543,16 +544,7 @@ it('should check for memory limits and not restart if under absolute limit', asy }; const worker = new Worker(options); - const onProcessStart = jest.fn(); - const onProcessEnd = jest.fn(); - const onCustomMessage = jest.fn(); - - worker.send( - [CHILD_MESSAGE_CALL, false, 'foo', []], - onProcessStart, - onProcessEnd, - onCustomMessage, - ); + worker.checkMemoryUsage(); totalmem.mockReturnValue(memoryConfig.totalMem); @@ -581,16 +573,7 @@ it('should check for memory limits and restart if above percentage limit', async }; const worker = new Worker(options); - const onProcessStart = jest.fn(); - const onProcessEnd = jest.fn(); - const onCustomMessage = jest.fn(); - - worker.send( - [CHILD_MESSAGE_CALL, false, 'foo', []], - onProcessStart, - onProcessEnd, - onCustomMessage, - ); + worker.checkMemoryUsage(); totalmem.mockReturnValue(memoryConfig.totalMem); @@ -619,16 +602,7 @@ it('should check for memory limits and restart if above absolute limit', async ( }; const worker = new Worker(options); - const onProcessStart = jest.fn(); - const onProcessEnd = jest.fn(); - const onCustomMessage = jest.fn(); - - worker.send( - [CHILD_MESSAGE_CALL, false, 'foo', []], - onProcessStart, - onProcessEnd, - onCustomMessage, - ); + worker.checkMemoryUsage(); totalmem.mockReturnValue(memoryConfig.totalMem); diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js new file mode 100644 index 000000000000..cd924fe30162 --- /dev/null +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -0,0 +1,169 @@ +import {rm, writeFile} from 'fs/promises'; +import {join} from 'path'; +import {transformFileAsync} from '@babel/core'; +import {CHILD_MESSAGE_CALL, CHILD_MESSAGE_MEM_USAGE} from '../../types'; +import ChildProcessWorker from '../ChildProcessWorker'; + +jest.retryTimes(1); + +/** @type ChildProcessWorker */ +let worker; +let int; + +const filesToBuild = ['../processChild', '../../types']; + +beforeAll(async () => { + for (const file of filesToBuild) { + const result = await transformFileAsync(join(__dirname, `${file}.ts`)); + + await writeFile(join(__dirname, `${file}.js`), result.code, { + encoding: 'utf-8', + }); + } +}); + +afterAll(async () => { + for (const file of filesToBuild) { + await rm(join(__dirname, `${file}.js`)); + } +}); + +beforeEach(() => {}); + +afterEach(async () => { + if (worker) { + worker.forceExit(); + await worker.waitForExit(); + } + + clearInterval(int); +}); + +function waitForChange(fn, limit = 100) { + const inital = fn(); + + return new Promise((resolve, reject) => { + let count = 0; + const int = setInterval(() => { + const updated = fn(); + + if (inital !== updated) { + resolve(updated); + clearInterval(int); + } + + if (count > limit) { + reject(new Error('Timeout waiting for change')); + } + + count++; + }, 50); + }); +} + +test('should get memory usage', async () => { + worker = new ChildProcessWorker({ + maxRetries: 0, + workerPath: join( + __dirname, + '__fixtures__', + 'ChildProcessWorkerEdgeCasesWorker', + ), + }); + + expect(await worker.getMemoryUsage()).toBeGreaterThan(0); +}, 10000); + +test('should recycle on idle limit breach', async () => { + worker = new ChildProcessWorker({ + // There is no way this is fitting into 1000 bytes, so it should restart + // after requesting a memory usage update + idleMemoryLimit: 1000, + maxRetries: 0, + workerPath: join( + __dirname, + '__fixtures__', + 'ChildProcessWorkerEdgeCasesWorker', + ), + }); + + const startPid = worker.getWorkerPid(); + expect(startPid).toBeGreaterThanOrEqual(0); + + worker.checkMemoryUsage(); + + await waitForChange(() => worker.getWorkerPid()); + + const endPid = worker.getWorkerPid(); + expect(endPid).toBeGreaterThanOrEqual(0); + expect(endPid).not.toEqual(startPid); +}, 10000); + +test('should automatically recycle on idle limit breach', async () => { + worker = new ChildProcessWorker({ + // There is no way this is fitting into 1000 bytes, so it should restart + // after requesting a memory usage update + idleMemoryLimit: 1000, + maxRetries: 0, + workerPath: join( + __dirname, + '__fixtures__', + 'ChildProcessWorkerEdgeCasesWorker', + ), + }); + + const startPid = worker.getWorkerPid(); + expect(startPid).toBeGreaterThanOrEqual(0); + + const onStart = jest.fn(); + const onEnd = jest.fn(); + const onCustom = jest.fn(); + + worker.send( + [CHILD_MESSAGE_CALL, true, 'safeFunction', []], + onStart, + onEnd, + onCustom, + ); + + await waitForChange(() => worker.getWorkerPid()); + + const endPid = worker.getWorkerPid(); + expect(endPid).toBeGreaterThanOrEqual(0); + expect(endPid).not.toEqual(startPid); +}, 10000); + +test('should cleanly exit on crash', async () => { + worker = new ChildProcessWorker({ + forkOptions: { + // Forcibly set the heap limit so we can crash the process easily. + execArgv: ['--max-old-space-size=50'], + }, + // There is no way this is fitting into 1000 bytes, so it should restart + // after requesting a memory usage update + idleMemoryLimit: 1000, + + maxRetries: 0, + workerPath: join( + __dirname, + '__fixtures__', + 'ChildProcessWorkerEdgeCasesWorker', + ), + }); + + const startPid = worker.getWorkerPid(); + expect(startPid).toBeGreaterThanOrEqual(0); + + const onStart = jest.fn(); + const onEnd = jest.fn(); + const onCustom = jest.fn(); + + worker.send( + [CHILD_MESSAGE_CALL, true, 'leakMemory', []], + onStart, + onEnd, + onCustom, + ); + + await worker.waitForExit(); +}, 10000); diff --git a/packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js b/packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js new file mode 100644 index 000000000000..1c7f7a28a08e --- /dev/null +++ b/packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js @@ -0,0 +1,19 @@ +let leakStore = ''; + +/** + * This exists to force a memory leak in the worker tests. + */ +function leakMemory() { + while (true) { + leakStore += '#'.repeat(1000); + } +} + +function safeFunction() { + // Doesn't do anything. +} + +module.exports = { + leakMemory, + safeFunction, +}; From 1961d658be03de531d914631b025850405880cd8 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 25 Jul 2022 15:25:38 +0100 Subject: [PATCH 05/49] docs: update docs for PR review --- docs/Configuration.md | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index 619ec7864857..dcb193c99c85 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -2254,28 +2254,16 @@ Default: `true` Whether to use [`watchman`](https://facebook.github.io/watchman/) for file crawling. -### `//` \[string] - -This option allows comments in `package.json`. Include the comment text as the value of this key: - -```json title="package.json" -{ - "name": "my-project", - "jest": { - "//": "Comment goes here", - "verbose": true - } -} -``` - ### `workerIdleMemoryLimit` \[number] +Default: `undefined` + Specifies the memory limit for workers before they are recycled and is primarily a work-around for [this issue](https://github.com/facebook/jest/issues/11956); After the worker has executed a test the memory usage of it is checked. If it exceeds the value specified the worker is killed and restarted. The limit can be specified in 2 ways * < 1 - The value is assumed to be a percentage of system memory. So 0.5 sets the memory limit of the worker to half of the total system memory -* > 1 - Assumed to be a fixed byte value +* \> 1 - Assumed to be a fixed byte value ```js tab /** @type {import('jest').Config} */ @@ -2285,3 +2273,26 @@ const config = { module.exports = config; ``` +```ts tab +import type {Config} from 'jest'; + +const config: Config = { + workerIdleMemoryLimit: 0.2, +}; + +export default config; +``` + +### `//` \[string] + +This option allows comments in `package.json`. Include the comment text as the value of this key: + +```json title="package.json" +{ + "name": "my-project", + "jest": { + "//": "Comment goes here", + "verbose": true + } +} +``` From d7f292f89b0fb25f8aad2b498849311e92722acb Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 25 Jul 2022 15:41:58 +0100 Subject: [PATCH 06/49] chore: try this test config --- .../workers/__tests__/ChildProcessWorkerEdgeCases.test.js | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index cd924fe30162..ddede112e7b4 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -139,10 +139,6 @@ test('should cleanly exit on crash', async () => { // Forcibly set the heap limit so we can crash the process easily. execArgv: ['--max-old-space-size=50'], }, - // There is no way this is fitting into 1000 bytes, so it should restart - // after requesting a memory usage update - idleMemoryLimit: 1000, - maxRetries: 0, workerPath: join( __dirname, @@ -166,4 +162,4 @@ test('should cleanly exit on crash', async () => { ); await worker.waitForExit(); -}, 10000); +}, 30000); From 8bfcd74fc28d00d5af387758315b5c3bb5ecf409 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 25 Jul 2022 17:02:51 +0100 Subject: [PATCH 07/49] chore: linting --- docs/Configuration.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index dcb193c99c85..7ad06d09cfdc 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -2262,8 +2262,8 @@ Specifies the memory limit for workers before they are recycled and is primarily After the worker has executed a test the memory usage of it is checked. If it exceeds the value specified the worker is killed and restarted. The limit can be specified in 2 ways -* < 1 - The value is assumed to be a percentage of system memory. So 0.5 sets the memory limit of the worker to half of the total system memory -* \> 1 - Assumed to be a fixed byte value +- < 1 - The value is assumed to be a percentage of system memory. So 0.5 sets the memory limit of the worker to half of the total system memory +- \> 1 - Assumed to be a fixed byte value ```js tab /** @type {import('jest').Config} */ @@ -2273,6 +2273,7 @@ const config = { module.exports = config; ``` + ```ts tab import type {Config} from 'jest'; From fd1be6f6c95a171e9db858f5b9a65a414e730518 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 25 Jul 2022 17:28:33 +0100 Subject: [PATCH 08/49] feat: add improved error checking and promise handling --- .../src/workers/ChildProcessWorker.ts | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index 4e27519ea473..1953294e6ef0 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -358,15 +358,33 @@ export default class ChildProcessWorker implements WorkerInterface { */ getMemoryUsage(): Promise { if (!this._memoryUsagePromise) { - this._memoryUsagePromise = new Promise((resolve, reject) => { + let rejectCallback!: (err: Error) => void; + + const promise = new Promise((resolve, reject) => { this._resolveMemoryUsage = resolve; + rejectCallback = reject; + }); + this._memoryUsagePromise = promise; + + if (!this._child.connected && rejectCallback) { + rejectCallback(new Error('Child process is not running.')); + + this._memoryUsagePromise = undefined; + this._resolveMemoryUsage = undefined; + + return promise; + } - if (!this._child.connected) { - reject(new Error('Child process is not running.')); + this._child.send([CHILD_MESSAGE_MEM_USAGE], err => { + if (err && rejectCallback) { + this._memoryUsagePromise = undefined; + this._resolveMemoryUsage = undefined; + + rejectCallback(err); } }); - this._child.send([CHILD_MESSAGE_MEM_USAGE]); + return promise; } return this._memoryUsagePromise; @@ -378,7 +396,11 @@ export default class ChildProcessWorker implements WorkerInterface { checkMemoryUsage(): void { if (this._childIdleMemoryUsageLimit) { this._memoryUsageCheck = true; - this._child.send([CHILD_MESSAGE_MEM_USAGE]); + this._child.send([CHILD_MESSAGE_MEM_USAGE], err => { + if (err) { + console.error('Unable to check memory usage', err); + } + }); } else { console.warn( 'Memory usage of workers can only be checked if a limit is set', From 233477ef8ccadd6d3f4db70a4fab068c5b183b06 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 25 Jul 2022 17:42:49 +0100 Subject: [PATCH 09/49] chore: add facebook header --- .../workers/__tests__/ChildProcessWorkerEdgeCases.test.js | 7 +++++++ .../__fixtures__/ChildProcessWorkerEdgeCasesWorker.js | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index ddede112e7b4..fb1c111ff364 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -1,3 +1,10 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + import {rm, writeFile} from 'fs/promises'; import {join} from 'path'; import {transformFileAsync} from '@babel/core'; diff --git a/packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js b/packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js index 1c7f7a28a08e..460af8068bcd 100644 --- a/packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js +++ b/packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js @@ -1,3 +1,10 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + let leakStore = ''; /** From 19fb71c844eeb6b6570616b3114e0909f29da079 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 25 Jul 2022 20:39:28 +0100 Subject: [PATCH 10/49] fix: use os spy rather than mock to retain platform functions --- .../src/workers/__tests__/ChildProcessWorker.test.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js index 38f3a2ced4d1..d01ca0fab4dd 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js @@ -6,6 +6,7 @@ */ import EventEmitter from 'events'; +import * as Os from 'os'; import {PassThrough} from 'stream'; import getStream from 'get-stream'; import supportsColor from 'supports-color'; @@ -30,7 +31,6 @@ let totalmem; beforeEach(() => { jest.mock('child_process'); - jest.mock('os'); originalExecArgv = process.execArgv; @@ -47,7 +47,7 @@ beforeEach(() => { return forkInterface; }); - totalmem = require('os').totalmem; + totalmem = jest.spyOn(Os, 'totalmem'); Worker = require('../ChildProcessWorker').default; }); From e2a853d7ea2b25f16139a72118780ae3795a58d7 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 25 Jul 2022 20:59:03 +0100 Subject: [PATCH 11/49] fix: spying of totalmem --- .../src/workers/__tests__/ChildProcessWorker.test.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js index d01ca0fab4dd..3249661604f3 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js @@ -6,7 +6,6 @@ */ import EventEmitter from 'events'; -import * as Os from 'os'; import {PassThrough} from 'stream'; import getStream from 'get-stream'; import supportsColor from 'supports-color'; @@ -29,6 +28,11 @@ let childProcess; let originalExecArgv; let totalmem; +beforeAll(() => { + const os = require('os'); + totalmem = jest.spyOn(os, 'totalmem'); +}); + beforeEach(() => { jest.mock('child_process'); @@ -47,7 +51,7 @@ beforeEach(() => { return forkInterface; }); - totalmem = jest.spyOn(Os, 'totalmem'); + totalmem.mockReset(); Worker = require('../ChildProcessWorker').default; }); From ab6554db31aad420ceecbef362c4af12cad9152d Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 25 Jul 2022 21:02:03 +0100 Subject: [PATCH 12/49] chore: add some logging to help debugging --- .../workers/__tests__/ChildProcessWorkerEdgeCases.test.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index fb1c111ff364..8806d6fac4b7 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -78,7 +78,12 @@ test('should get memory usage', async () => { ), }); - expect(await worker.getMemoryUsage()).toBeGreaterThan(0); + const memoryUsagePromise = worker.getMemoryUsage(); + expect(memoryUsagePromise).toBeInstanceOf(Promise); + + console.log(1, memoryUsagePromise); + expect(await memoryUsagePromise).toBeGreaterThan(0); + console.log(2, memoryUsagePromise); }, 10000); test('should recycle on idle limit breach', async () => { From 207ae05205a184f4e1c6ae0dc3b0e044f947860a Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 25 Jul 2022 21:41:13 +0100 Subject: [PATCH 13/49] chore: more debugging --- .../workers/__tests__/ChildProcessWorkerEdgeCases.test.js | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index 8806d6fac4b7..1ffc092ce08c 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -69,6 +69,8 @@ function waitForChange(fn, limit = 100) { } test('should get memory usage', async () => { + console.log(1); + worker = new ChildProcessWorker({ maxRetries: 0, workerPath: join( @@ -78,12 +80,14 @@ test('should get memory usage', async () => { ), }); + console.log(2, worker); + const memoryUsagePromise = worker.getMemoryUsage(); expect(memoryUsagePromise).toBeInstanceOf(Promise); - console.log(1, memoryUsagePromise); + console.log(3, memoryUsagePromise); expect(await memoryUsagePromise).toBeGreaterThan(0); - console.log(2, memoryUsagePromise); + console.log(4, memoryUsagePromise); }, 10000); test('should recycle on idle limit breach', async () => { From c4d2389e9320863665cb630c51e26478f4d051e2 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 25 Jul 2022 22:01:56 +0100 Subject: [PATCH 14/49] chore: more debugging --- packages/jest-worker/src/workers/ChildProcessWorker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index 1953294e6ef0..89bf58be922b 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -109,7 +109,7 @@ export default class ChildProcessWorker implements WorkerInterface { execArgv: process.execArgv.filter(v => !/^--(debug|inspect)/.test(v)), // default to advanced serialization in order to match worker threads serialization: 'advanced', - silent: true, + silent: false, ...this._options.forkOptions, }; From 0f5c7bb9224062c6a6649c82e3aa9f766f442ac1 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 25 Jul 2022 22:15:01 +0100 Subject: [PATCH 15/49] chore: check files exist --- .../__tests__/ChildProcessWorkerEdgeCases.test.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index 1ffc092ce08c..51e43d926198 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -5,7 +5,7 @@ * LICENSE file in the root directory of this source tree. */ -import {rm, writeFile} from 'fs/promises'; +import {access, rm, writeFile} from 'fs/promises'; import {join} from 'path'; import {transformFileAsync} from '@babel/core'; import {CHILD_MESSAGE_CALL, CHILD_MESSAGE_MEM_USAGE} from '../../types'; @@ -35,8 +35,6 @@ afterAll(async () => { } }); -beforeEach(() => {}); - afterEach(async () => { if (worker) { worker.forceExit(); @@ -68,6 +66,12 @@ function waitForChange(fn, limit = 100) { }); } +test.each(filesToBuild)('%s should exist', async file => { + const path = join(__dirname, `${file}.js`); + + expect(() => access(path)).not.toThrow(); +}); + test('should get memory usage', async () => { console.log(1); From d35e4b9b07f437a74da3519171e884f0d53bc2e1 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 25 Jul 2022 22:28:25 +0100 Subject: [PATCH 16/49] chore: try as a specific js import --- packages/jest-worker/src/workers/ChildProcessWorker.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index 89bf58be922b..acb3f7a61d47 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -113,7 +113,7 @@ export default class ChildProcessWorker implements WorkerInterface { ...this._options.forkOptions, }; - const child = fork(require.resolve('./processChild'), [], options); + const child = fork(require.resolve('./processChild.js'), [], options); if (child.stdout) { if (!this._stdout) { From daee499613a0c88bf6ee1efc969d09f00ad1196b Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Tue, 26 Jul 2022 11:57:50 +0100 Subject: [PATCH 17/49] chore: try this --- jest.config.mjs | 3 +- packages/jest-worker/src/types.ts | 8 + .../src/workers/ChildProcessWorker.ts | 10 +- .../ChildProcessWorkerEdgeCases.test.js | 34 ++-- .../src/workers/__tests__/processChild.js | 151 ++++++++++++++++++ 5 files changed, 191 insertions(+), 15 deletions(-) create mode 100644 packages/jest-worker/src/workers/__tests__/processChild.js diff --git a/jest.config.mjs b/jest.config.mjs index 646d09cc0baf..192ccd07dcfa 100644 --- a/jest.config.mjs +++ b/jest.config.mjs @@ -76,8 +76,7 @@ export default { }, watchPathIgnorePatterns: [ 'coverage', - '/packages/jest-worker/src/workers/processChild.js', - '/packages/jest-worker/src/types.js', + '/packages/jest-worker/src/workers/__tests__/__temp__', ], watchPlugins: [ require.resolve('jest-watch-typeahead/filename'), diff --git a/packages/jest-worker/src/types.ts b/packages/jest-worker/src/types.ts index fb51ecc6d8fb..89428fbc279a 100644 --- a/packages/jest-worker/src/types.ts +++ b/packages/jest-worker/src/types.ts @@ -148,6 +148,14 @@ export type WorkerOptions = { idleMemoryLimit?: number; }; +export type ChildProcessWorkerOptions = WorkerOptions & { + /** + * This mainly exists so the path can be changed during testing. + * https://github.com/facebook/jest/issues/9543 + */ + childWorkerPath?: string; +}; + // Messages passed from the parent to the children. export type MessagePort = typeof EventEmitter & { diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index acb3f7a61d47..8ff9128c57bc 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -14,6 +14,7 @@ import { CHILD_MESSAGE_INITIALIZE, CHILD_MESSAGE_MEM_USAGE, ChildMessage, + ChildProcessWorkerOptions, OnCustomMessage, OnEnd, OnStart, @@ -76,7 +77,9 @@ export default class ChildProcessWorker implements WorkerInterface { private _restarting = false; private _memoryUsageCheck = false; - constructor(options: WorkerOptions) { + private _childWorkerPath: string; + + constructor(options: ChildProcessWorkerOptions) { this._options = options; this._request = null; @@ -91,6 +94,9 @@ export default class ChildProcessWorker implements WorkerInterface { this._resolveExitPromise = resolve; }); + this._childWorkerPath = + options.childWorkerPath || require.resolve('./processChild'); + this.initialize(); } @@ -113,7 +119,7 @@ export default class ChildProcessWorker implements WorkerInterface { ...this._options.forkOptions, }; - const child = fork(require.resolve('./processChild.js'), [], options); + const child = fork(this._childWorkerPath, [], options); if (child.stdout) { if (!this._stdout) { diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index 51e43d926198..f62bb03cd03a 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -5,8 +5,8 @@ * LICENSE file in the root directory of this source tree. */ -import {access, rm, writeFile} from 'fs/promises'; -import {join} from 'path'; +import {access, mkdir, rm, writeFile} from 'fs/promises'; +import {dirname, join} from 'path'; import {transformFileAsync} from '@babel/core'; import {CHILD_MESSAGE_CALL, CHILD_MESSAGE_MEM_USAGE} from '../../types'; import ChildProcessWorker from '../ChildProcessWorker'; @@ -17,22 +17,30 @@ jest.retryTimes(1); let worker; let int; -const filesToBuild = ['../processChild', '../../types']; +const root = join('../../'); +const filesToBuild = ['workers/processChild', 'types']; +const writeDestination = join(__dirname, '__temp__'); +const childWorkerPath = join(writeDestination, 'workers/processChild.js'); beforeAll(async () => { + await mkdir(writeDestination, {recursive: true}); + for (const file of filesToBuild) { - const result = await transformFileAsync(join(__dirname, `${file}.ts`)); + const sourcePath = join(__dirname, root, `${file}.ts`); + const writePath = join(writeDestination, `${file}.js`); + + await mkdir(dirname(writePath), {recursive: true}); + + const result = await transformFileAsync(sourcePath); - await writeFile(join(__dirname, `${file}.js`), result.code, { + await writeFile(writePath, result.code, { encoding: 'utf-8', }); } }); afterAll(async () => { - for (const file of filesToBuild) { - await rm(join(__dirname, `${file}.js`)); - } + await rm(writeDestination, {force: true, recursive: true}); }); afterEach(async () => { @@ -66,16 +74,17 @@ function waitForChange(fn, limit = 100) { }); } -test.each(filesToBuild)('%s should exist', async file => { - const path = join(__dirname, `${file}.js`); +test.each(filesToBuild)('%s.js should exist', async file => { + const path = join(writeDestination, `${file}.js`); - expect(() => access(path)).not.toThrow(); + await expect(async () => await access(path)).not.toThrowError(); }); test('should get memory usage', async () => { console.log(1); worker = new ChildProcessWorker({ + childWorkerPath, maxRetries: 0, workerPath: join( __dirname, @@ -96,6 +105,7 @@ test('should get memory usage', async () => { test('should recycle on idle limit breach', async () => { worker = new ChildProcessWorker({ + childWorkerPath, // There is no way this is fitting into 1000 bytes, so it should restart // after requesting a memory usage update idleMemoryLimit: 1000, @@ -121,6 +131,7 @@ test('should recycle on idle limit breach', async () => { test('should automatically recycle on idle limit breach', async () => { worker = new ChildProcessWorker({ + childWorkerPath, // There is no way this is fitting into 1000 bytes, so it should restart // after requesting a memory usage update idleMemoryLimit: 1000, @@ -155,6 +166,7 @@ test('should automatically recycle on idle limit breach', async () => { test('should cleanly exit on crash', async () => { worker = new ChildProcessWorker({ + childWorkerPath, forkOptions: { // Forcibly set the heap limit so we can crash the process easily. execArgv: ['--max-old-space-size=50'], diff --git a/packages/jest-worker/src/workers/__tests__/processChild.js b/packages/jest-worker/src/workers/__tests__/processChild.js new file mode 100644 index 000000000000..e99065023330 --- /dev/null +++ b/packages/jest-worker/src/workers/__tests__/processChild.js @@ -0,0 +1,151 @@ +"use strict"; + +var _types = require("../types"); + +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ +let file = null; +let setupArgs = []; +let initialized = false; +/** + * This file is a small bootstrapper for workers. It sets up the communication + * between the worker and the parent process, interpreting parent messages and + * sending results back. + * + * The file loaded will be lazily initialized the first time any of the workers + * is called. This is done for optimal performance: if the farm is initialized, + * but no call is made to it, child Node processes will be consuming the least + * possible amount of memory. + * + * If an invalid message is detected, the child will exit (by throwing) with a + * non-zero exit code. + */ + +const messageListener = request => { + switch (request[0]) { + case _types.CHILD_MESSAGE_INITIALIZE: + const init = request; + file = init[2]; + setupArgs = request[3]; + break; + + case _types.CHILD_MESSAGE_CALL: + const call = request; + execMethod(call[2], call[3]); + break; + + case _types.CHILD_MESSAGE_END: + end(); + break; + + case _types.CHILD_MESSAGE_MEM_USAGE: + reportMemoryUsage(); + break; + + default: + throw new TypeError(`Unexpected request from parent process: ${request[0]}`); + } +}; + +process.on('message', messageListener); + +function reportSuccess(result) { + if (!process || !process.send) { + throw new Error('Child can only be used on a forked process'); + } + + process.send([_types.PARENT_MESSAGE_OK, result]); +} + +function reportClientError(error) { + return reportError(error, _types.PARENT_MESSAGE_CLIENT_ERROR); +} + +function reportInitializeError(error) { + return reportError(error, _types.PARENT_MESSAGE_SETUP_ERROR); +} + +function reportMemoryUsage() { + if (!process || !process.send) { + throw new Error('Child can only be used on a forked process'); + } + + const msg = [_types.PARENT_MESSAGE_MEM_USAGE, process.memoryUsage().heapUsed]; + process.send(msg); +} + +function reportError(error, type) { + if (!process || !process.send) { + throw new Error('Child can only be used on a forked process'); + } + + if (error == null) { + error = new Error('"null" or "undefined" thrown'); + } + + process.send([type, error.constructor && error.constructor.name, error.message, error.stack, typeof error === 'object' ? { ...error + } : error]); +} + +function end() { + const main = require(file); + + if (!main.teardown) { + exitProcess(); + return; + } + + execFunction(main.teardown, main, [], exitProcess, exitProcess); +} + +function exitProcess() { + // Clean up open handles so the process ideally exits gracefully + process.removeListener('message', messageListener); +} + +function execMethod(method, args) { + const main = require(file); + + let fn; + + if (method === 'default') { + fn = main.__esModule ? main['default'] : main; + } else { + fn = main[method]; + } + + function execHelper() { + execFunction(fn, main, args, reportSuccess, reportClientError); + } + + if (initialized || !main.setup) { + execHelper(); + return; + } + + initialized = true; + execFunction(main.setup, main, setupArgs, execHelper, reportInitializeError); +} + +const isPromise = obj => !!obj && (typeof obj === 'object' || typeof obj === 'function') && typeof obj.then === 'function'; + +function execFunction(fn, ctx, args, onResult, onError) { + let result; + + try { + result = fn.apply(ctx, args); + } catch (err) { + onError(err); + return; + } + + if (isPromise(result)) { + result.then(onResult, onError); + } else { + onResult(result); + } +} \ No newline at end of file From fd96e788b0dc758a3fa34134ee65e25722703ff5 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Tue, 26 Jul 2022 13:33:00 +0100 Subject: [PATCH 18/49] chore: remove debugging and cleanup --- .../src/workers/ChildProcessWorker.ts | 2 +- .../ChildProcessWorkerEdgeCases.test.js | 7 - .../src/workers/__tests__/processChild.js | 151 ------------------ 3 files changed, 1 insertion(+), 159 deletions(-) delete mode 100644 packages/jest-worker/src/workers/__tests__/processChild.js diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index 8ff9128c57bc..e88288cffd7c 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -115,7 +115,7 @@ export default class ChildProcessWorker implements WorkerInterface { execArgv: process.execArgv.filter(v => !/^--(debug|inspect)/.test(v)), // default to advanced serialization in order to match worker threads serialization: 'advanced', - silent: false, + silent: true, ...this._options.forkOptions, }; diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index f62bb03cd03a..7d97b58d7556 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -81,8 +81,6 @@ test.each(filesToBuild)('%s.js should exist', async file => { }); test('should get memory usage', async () => { - console.log(1); - worker = new ChildProcessWorker({ childWorkerPath, maxRetries: 0, @@ -93,14 +91,9 @@ test('should get memory usage', async () => { ), }); - console.log(2, worker); - const memoryUsagePromise = worker.getMemoryUsage(); expect(memoryUsagePromise).toBeInstanceOf(Promise); - - console.log(3, memoryUsagePromise); expect(await memoryUsagePromise).toBeGreaterThan(0); - console.log(4, memoryUsagePromise); }, 10000); test('should recycle on idle limit breach', async () => { diff --git a/packages/jest-worker/src/workers/__tests__/processChild.js b/packages/jest-worker/src/workers/__tests__/processChild.js deleted file mode 100644 index e99065023330..000000000000 --- a/packages/jest-worker/src/workers/__tests__/processChild.js +++ /dev/null @@ -1,151 +0,0 @@ -"use strict"; - -var _types = require("../types"); - -/** - * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ -let file = null; -let setupArgs = []; -let initialized = false; -/** - * This file is a small bootstrapper for workers. It sets up the communication - * between the worker and the parent process, interpreting parent messages and - * sending results back. - * - * The file loaded will be lazily initialized the first time any of the workers - * is called. This is done for optimal performance: if the farm is initialized, - * but no call is made to it, child Node processes will be consuming the least - * possible amount of memory. - * - * If an invalid message is detected, the child will exit (by throwing) with a - * non-zero exit code. - */ - -const messageListener = request => { - switch (request[0]) { - case _types.CHILD_MESSAGE_INITIALIZE: - const init = request; - file = init[2]; - setupArgs = request[3]; - break; - - case _types.CHILD_MESSAGE_CALL: - const call = request; - execMethod(call[2], call[3]); - break; - - case _types.CHILD_MESSAGE_END: - end(); - break; - - case _types.CHILD_MESSAGE_MEM_USAGE: - reportMemoryUsage(); - break; - - default: - throw new TypeError(`Unexpected request from parent process: ${request[0]}`); - } -}; - -process.on('message', messageListener); - -function reportSuccess(result) { - if (!process || !process.send) { - throw new Error('Child can only be used on a forked process'); - } - - process.send([_types.PARENT_MESSAGE_OK, result]); -} - -function reportClientError(error) { - return reportError(error, _types.PARENT_MESSAGE_CLIENT_ERROR); -} - -function reportInitializeError(error) { - return reportError(error, _types.PARENT_MESSAGE_SETUP_ERROR); -} - -function reportMemoryUsage() { - if (!process || !process.send) { - throw new Error('Child can only be used on a forked process'); - } - - const msg = [_types.PARENT_MESSAGE_MEM_USAGE, process.memoryUsage().heapUsed]; - process.send(msg); -} - -function reportError(error, type) { - if (!process || !process.send) { - throw new Error('Child can only be used on a forked process'); - } - - if (error == null) { - error = new Error('"null" or "undefined" thrown'); - } - - process.send([type, error.constructor && error.constructor.name, error.message, error.stack, typeof error === 'object' ? { ...error - } : error]); -} - -function end() { - const main = require(file); - - if (!main.teardown) { - exitProcess(); - return; - } - - execFunction(main.teardown, main, [], exitProcess, exitProcess); -} - -function exitProcess() { - // Clean up open handles so the process ideally exits gracefully - process.removeListener('message', messageListener); -} - -function execMethod(method, args) { - const main = require(file); - - let fn; - - if (method === 'default') { - fn = main.__esModule ? main['default'] : main; - } else { - fn = main[method]; - } - - function execHelper() { - execFunction(fn, main, args, reportSuccess, reportClientError); - } - - if (initialized || !main.setup) { - execHelper(); - return; - } - - initialized = true; - execFunction(main.setup, main, setupArgs, execHelper, reportInitializeError); -} - -const isPromise = obj => !!obj && (typeof obj === 'object' || typeof obj === 'function') && typeof obj.then === 'function'; - -function execFunction(fn, ctx, args, onResult, onError) { - let result; - - try { - result = fn.apply(ctx, args); - } catch (err) { - onError(err); - return; - } - - if (isPromise(result)) { - result.then(onResult, onError); - } else { - onResult(result); - } -} \ No newline at end of file From 576ca6356e9802f1a94bee98e070de65f26b963d Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Tue, 26 Jul 2022 13:57:20 +0100 Subject: [PATCH 19/49] chore: debug failing test --- packages/jest-worker/src/types.ts | 5 +++++ .../jest-worker/src/workers/ChildProcessWorker.ts | 4 +++- .../__tests__/ChildProcessWorkerEdgeCases.test.js | 11 +++++++++-- 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/jest-worker/src/types.ts b/packages/jest-worker/src/types.ts index 89428fbc279a..52b75a757c02 100644 --- a/packages/jest-worker/src/types.ts +++ b/packages/jest-worker/src/types.ts @@ -154,6 +154,11 @@ export type ChildProcessWorkerOptions = WorkerOptions & { * https://github.com/facebook/jest/issues/9543 */ childWorkerPath?: string; + /** + * This is useful for debugging individual tests allowing you to see + * the raw output of the worker. + */ + silent?: boolean; }; // Messages passed from the parent to the children. diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index e88288cffd7c..858937c287e7 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -78,6 +78,7 @@ export default class ChildProcessWorker implements WorkerInterface { private _memoryUsageCheck = false; private _childWorkerPath: string; + private _silent = true; constructor(options: ChildProcessWorkerOptions) { this._options = options; @@ -96,6 +97,7 @@ export default class ChildProcessWorker implements WorkerInterface { this._childWorkerPath = options.childWorkerPath || require.resolve('./processChild'); + this._silent = options.silent ?? true; this.initialize(); } @@ -115,7 +117,7 @@ export default class ChildProcessWorker implements WorkerInterface { execArgv: process.execArgv.filter(v => !/^--(debug|inspect)/.test(v)), // default to advanced serialization in order to match worker threads serialization: 'advanced', - silent: true, + silent: this._silent, ...this._options.forkOptions, }; diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index 7d97b58d7556..01a7723173d4 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -165,6 +165,7 @@ test('should cleanly exit on crash', async () => { execArgv: ['--max-old-space-size=50'], }, maxRetries: 0, + silent: false, workerPath: join( __dirname, '__fixtures__', @@ -172,8 +173,10 @@ test('should cleanly exit on crash', async () => { ), }); - const startPid = worker.getWorkerPid(); - expect(startPid).toBeGreaterThanOrEqual(0); + const pid = worker.getWorkerPid(); + expect(pid).toBeGreaterThanOrEqual(0); + + console.log(1, pid); const onStart = jest.fn(); const onEnd = jest.fn(); @@ -186,5 +189,9 @@ test('should cleanly exit on crash', async () => { onCustom, ); + console.log(2, pid, worker.waitForExit()); + await worker.waitForExit(); + + console.log(3, pid, worker.waitForExit()); }, 30000); From 491b16c2fa3c51a97a7cbba532374dcb1a2fe2e8 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Tue, 26 Jul 2022 15:32:01 +0100 Subject: [PATCH 20/49] fix: windows tests --- .gitignore | 5 ++--- packages/jest-worker/src/workers/ChildProcessWorker.ts | 10 ++++++++-- .../__tests__/ChildProcessWorkerEdgeCases.test.js | 10 ++-------- 3 files changed, 12 insertions(+), 13 deletions(-) diff --git a/.gitignore b/.gitignore index 7d691794efeb..1a8975f59cf9 100644 --- a/.gitignore +++ b/.gitignore @@ -54,6 +54,5 @@ api-extractor.json crowdin-cli.jar -# We don't want this temp file -packages/jest-worker/src/workers/processChild.js -packages/jest-worker/src/types.js +# We don't want these temp files +packages/jest-worker/src/workers/__tests__/__temp__ diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index 858937c287e7..8bd9f053a183 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -31,6 +31,7 @@ import { const SIGNAL_BASE_EXIT_CODE = 128; const SIGKILL_EXIT_CODE = SIGNAL_BASE_EXIT_CODE + 9; const SIGTERM_EXIT_CODE = SIGNAL_BASE_EXIT_CODE + 15; +const SIGABRT_EXIT_CODE = SIGNAL_BASE_EXIT_CODE + 6; // How long to wait after SIGTERM before sending SIGKILL const SIGKILL_DELAY = 500; @@ -269,11 +270,16 @@ export default class ChildProcessWorker implements WorkerInterface { } private _onExit(exitCode: number | null, signal: NodeJS.Signals | null) { + // When a out of memory crash occurs + // - Mac/Unix signal=SIGABRT + // - Windows exitCode=134 + if ( (exitCode !== 0 && exitCode !== null && exitCode !== SIGTERM_EXIT_CODE && - exitCode !== SIGKILL_EXIT_CODE) || + exitCode !== SIGKILL_EXIT_CODE && + exitCode !== SIGABRT_EXIT_CODE) || this._restarting ) { this.initialize(); @@ -282,7 +288,7 @@ export default class ChildProcessWorker implements WorkerInterface { this._child.send(this._request); } } else { - if (signal === 'SIGABRT') { + if (signal === 'SIGABRT' || exitCode === SIGABRT_EXIT_CODE) { // When a child process worker crashes due to lack of memory this prevents // jest from spinning and failing to exit. It could be argued it should restart // the process, but if you're running out of memory then restarting processes diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index 01a7723173d4..f32c35ecf899 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -165,7 +165,7 @@ test('should cleanly exit on crash', async () => { execArgv: ['--max-old-space-size=50'], }, maxRetries: 0, - silent: false, + silent: true, workerPath: join( __dirname, '__fixtures__', @@ -176,8 +176,6 @@ test('should cleanly exit on crash', async () => { const pid = worker.getWorkerPid(); expect(pid).toBeGreaterThanOrEqual(0); - console.log(1, pid); - const onStart = jest.fn(); const onEnd = jest.fn(); const onCustom = jest.fn(); @@ -189,9 +187,5 @@ test('should cleanly exit on crash', async () => { onCustom, ); - console.log(2, pid, worker.waitForExit()); - await worker.waitForExit(); - - console.log(3, pid, worker.waitForExit()); -}, 30000); +}, 5000); From 479c123e5082567fb2e0303b0e4af63cf87ff18d Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Tue, 26 Jul 2022 20:49:10 +0100 Subject: [PATCH 21/49] chore: use verbose output to help --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bcd52d1d0ae1..3f7a7015f80a 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "lint:prettier:ci": "yarn lint:prettier-script --check", "remove-examples": "node ./scripts/remove-examples.mjs", "test-ci-partial": "yarn test-ci-partial:parallel -i", - "test-ci-partial:parallel": "yarn jest --color --config jest.config.ci.mjs", + "test-ci-partial:parallel": "yarn jest --verbose --color --config jest.config.ci.mjs", "test-leak": "yarn jest -i --detectLeaks --color jest-mock jest-diff jest-repl pretty-format", "test-ts": "yarn jest --config jest.config.ts.mjs", "test-types": "yarn test-ts --selectProjects type-tests", From e3e63e48078079b4a116b4292703557cb9487b79 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Tue, 26 Jul 2022 21:33:27 +0100 Subject: [PATCH 22/49] chore: disable silent reporter --- jest.config.ci.mjs | 9 +++++---- package.json | 2 +- packages/jest-worker/src/workers/ChildProcessWorker.ts | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/jest.config.ci.mjs b/jest.config.ci.mjs index 2b2f36983f5e..fbf7ab6f1590 100644 --- a/jest.config.ci.mjs +++ b/jest.config.ci.mjs @@ -12,15 +12,16 @@ export default { ...baseConfig, coverageReporters: ['json'], reporters: [ + 'default', 'github-actions', [ 'jest-junit', {outputDirectory: 'reports/junit', outputName: 'js-test-results.xml'}, ], - [ - 'jest-silent-reporter', - {showPaths: true, showWarnings: true, useDots: true}, - ], + // [ + // 'jest-silent-reporter', + // {showPaths: true, showWarnings: true, useDots: true}, + // ], 'summary', ], }; diff --git a/package.json b/package.json index 3f7a7015f80a..bcd52d1d0ae1 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "lint:prettier:ci": "yarn lint:prettier-script --check", "remove-examples": "node ./scripts/remove-examples.mjs", "test-ci-partial": "yarn test-ci-partial:parallel -i", - "test-ci-partial:parallel": "yarn jest --verbose --color --config jest.config.ci.mjs", + "test-ci-partial:parallel": "yarn jest --color --config jest.config.ci.mjs", "test-leak": "yarn jest -i --detectLeaks --color jest-mock jest-diff jest-repl pretty-format", "test-ts": "yarn jest --config jest.config.ts.mjs", "test-types": "yarn test-ts --selectProjects type-tests", diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index 8bd9f053a183..3708f71d4ff2 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -224,6 +224,7 @@ export default class ChildProcessWorker implements WorkerInterface { this._onProcessEnd(error, null); break; + case PARENT_MESSAGE_CUSTOM: this._onCustomMessage(response[1]); break; @@ -239,7 +240,6 @@ export default class ChildProcessWorker implements WorkerInterface { } this._performRestartIfRequired(); - break; default: From 74ce892e49f5a221fe9cd3d851407ef627c6ab22 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 10:20:45 +0100 Subject: [PATCH 23/49] chore: temporary change to allow me to see where the tests are stalling --- packages/jest-test-sequencer/src/index.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/jest-test-sequencer/src/index.ts b/packages/jest-test-sequencer/src/index.ts index 2a643c4f777d..7fe5d8a61b06 100644 --- a/packages/jest-test-sequencer/src/index.ts +++ b/packages/jest-test-sequencer/src/index.ts @@ -163,7 +163,7 @@ export default class TestSequencer { cache[test.path] && cache[test.path][1]; tests.forEach(test => (test.duration = time(this._getCache(test), test))); - return tests.sort((testA, testB) => { + const sorted = tests.sort((testA, testB) => { const cacheA = this._getCache(testA); const cacheB = this._getCache(testB); const failedA = hasFailed(cacheA, testA); @@ -180,6 +180,14 @@ export default class TestSequencer { return fileSize(testA) < fileSize(testB) ? 1 : -1; } }); + + process.stdout.write('Expected test execution order\n'); + tests.forEach(x => { + process.stdout.write(` ${x.path}\n`); + }); + process.stdout.write('\n'); + + return sorted; } allFailedTests(tests: Array): Array | Promise> { From 8467b8b60e1d808fd3f3c33cd409ef2ee9350c94 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 10:47:00 +0100 Subject: [PATCH 24/49] chore: set sensible timeout --- .github/workflows/test.yml | 2 ++ packages/jest-config/src/normalize.ts | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d7060b18db3a..14be1e525767 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,6 +32,7 @@ jobs: id: cpu-cores uses: SimenB/github-actions-cpu-cores@v1 - name: run tests + timeout-minutes: 10 run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} test-jasmine: @@ -57,4 +58,5 @@ jobs: id: cpu-cores uses: SimenB/github-actions-cpu-cores@v1 - name: run tests using jest-jasmine + timeout-minutes: 10 run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} diff --git a/packages/jest-config/src/normalize.ts b/packages/jest-config/src/normalize.ts index 52e67fc288a2..421fdec11e27 100644 --- a/packages/jest-config/src/normalize.ts +++ b/packages/jest-config/src/normalize.ts @@ -967,6 +967,7 @@ export default async function normalize( case 'watch': case 'watchAll': case 'watchman': + case 'workerIdleMemoryLimit': value = oldOptions[key]; break; case 'watchPlugins': From 6338419394bdd9e346640c5c3884f4d1c3bb109b Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 11:04:38 +0100 Subject: [PATCH 25/49] chore: there's an argument for what i want to do --- .github/workflows/test.yml | 4 ++-- packages/jest-test-sequencer/src/index.ts | 10 +--------- 2 files changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 14be1e525767..461ff3e29595 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: uses: SimenB/github-actions-cpu-cores@v1 - name: run tests timeout-minutes: 10 - run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} + run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} --listTests test-jasmine: strategy: @@ -59,4 +59,4 @@ jobs: uses: SimenB/github-actions-cpu-cores@v1 - name: run tests using jest-jasmine timeout-minutes: 10 - run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} + run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} --listTests diff --git a/packages/jest-test-sequencer/src/index.ts b/packages/jest-test-sequencer/src/index.ts index 7fe5d8a61b06..2a643c4f777d 100644 --- a/packages/jest-test-sequencer/src/index.ts +++ b/packages/jest-test-sequencer/src/index.ts @@ -163,7 +163,7 @@ export default class TestSequencer { cache[test.path] && cache[test.path][1]; tests.forEach(test => (test.duration = time(this._getCache(test), test))); - const sorted = tests.sort((testA, testB) => { + return tests.sort((testA, testB) => { const cacheA = this._getCache(testA); const cacheB = this._getCache(testB); const failedA = hasFailed(cacheA, testA); @@ -180,14 +180,6 @@ export default class TestSequencer { return fileSize(testA) < fileSize(testB) ? 1 : -1; } }); - - process.stdout.write('Expected test execution order\n'); - tests.forEach(x => { - process.stdout.write(` ${x.path}\n`); - }); - process.stdout.write('\n'); - - return sorted; } allFailedTests(tests: Array): Array | Promise> { From d2acdc399dc06df120ff9d99adcd1a84c3a40007 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 11:16:29 +0100 Subject: [PATCH 26/49] chore: try this --- .circleci/config.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 34b8da8592e5..392611e5ec43 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,7 +28,7 @@ jobs: node-version: << parameters.node-version >> - node/install-packages: *install - run: - command: yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL + command: yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL --listTests - store_test_results: path: reports/junit @@ -42,7 +42,11 @@ jobs: node-version: lts/* - node/install-packages: *install - run: - command: JEST_JASMINE=1 yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL && JEST_JASMINE=1 yarn test-leak + name: Test + command: JEST_JASMINE=1 yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL --listTests --runInBand + - run: + name: Leak test + command: JEST_JASMINE=1 yarn test-leak --listTests --runInBand - store_test_results: path: reports/junit From dea2ef22c45e337ad75ebc20efc738e8d7e55b51 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 11:41:01 +0100 Subject: [PATCH 27/49] chore: remove now I have the test order --- .circleci/config.yml | 6 +++--- .github/workflows/test.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 392611e5ec43..6b145580d00b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,7 +28,7 @@ jobs: node-version: << parameters.node-version >> - node/install-packages: *install - run: - command: yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL --listTests + command: yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL - store_test_results: path: reports/junit @@ -43,10 +43,10 @@ jobs: - node/install-packages: *install - run: name: Test - command: JEST_JASMINE=1 yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL --listTests --runInBand + command: JEST_JASMINE=1 yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL --runInBand - run: name: Leak test - command: JEST_JASMINE=1 yarn test-leak --listTests --runInBand + command: JEST_JASMINE=1 yarn test-leak --runInBand - store_test_results: path: reports/junit diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 461ff3e29595..14be1e525767 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: uses: SimenB/github-actions-cpu-cores@v1 - name: run tests timeout-minutes: 10 - run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} --listTests + run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} test-jasmine: strategy: @@ -59,4 +59,4 @@ jobs: uses: SimenB/github-actions-cpu-cores@v1 - name: run tests using jest-jasmine timeout-minutes: 10 - run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} --listTests + run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} From 44ff11d093c9776b9ccef48a6f7c230a0ca6d09c Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 12:05:43 +0100 Subject: [PATCH 28/49] chore: single thread to track down the failing test suite --- .circleci/config.yml | 6 +++--- .github/workflows/test.yml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b145580d00b..ada2ec6dd9ec 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,7 +28,7 @@ jobs: node-version: << parameters.node-version >> - node/install-packages: *install - run: - command: yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL + command: yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL --runInBand - store_test_results: path: reports/junit @@ -43,10 +43,10 @@ jobs: - node/install-packages: *install - run: name: Test - command: JEST_JASMINE=1 yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL --runInBand + command: JEST_JASMINE=1 yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL - run: name: Leak test - command: JEST_JASMINE=1 yarn test-leak --runInBand + command: JEST_JASMINE=1 yarn test-leak - store_test_results: path: reports/junit diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 14be1e525767..73137c3e92a1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: uses: SimenB/github-actions-cpu-cores@v1 - name: run tests timeout-minutes: 10 - run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} + run: yarn test-ci-partial:parallel --runInBand --shard=${{ matrix.shard }} test-jasmine: strategy: @@ -59,4 +59,4 @@ jobs: uses: SimenB/github-actions-cpu-cores@v1 - name: run tests using jest-jasmine timeout-minutes: 10 - run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} + run: yarn jest-jasmine-ci --runInBand --shard=${{ matrix.shard }} From 6b3ba00b16b817df8f0f0748e9358d713f4580e1 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 12:55:46 +0100 Subject: [PATCH 29/49] chore: does skipping this test make the timeouts go away? --- e2e/__tests__/fatalWorkerError.test.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/e2e/__tests__/fatalWorkerError.test.ts b/e2e/__tests__/fatalWorkerError.test.ts index 477932738da0..ab7a246e2484 100644 --- a/e2e/__tests__/fatalWorkerError.test.ts +++ b/e2e/__tests__/fatalWorkerError.test.ts @@ -19,7 +19,7 @@ const DIR = path.resolve(tmpdir(), 'fatal-worker-error'); beforeEach(() => cleanup(DIR)); afterAll(() => cleanup(DIR)); -test('fails a test that terminates the worker with a fatal error', () => { +test.skip('fails a test that terminates the worker with a fatal error', () => { const testFiles = { ...generateTestFilesToForceUsingWorkers(), '__tests__/fatalWorkerError.test.js': ` From f208aa947f2552c62623f430a623db8e714ed971 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 14:56:55 +0100 Subject: [PATCH 30/49] fix: fatal but not out of memory errors should retry --- .github/workflows/test.yml | 4 +- e2e/__tests__/fatalWorkerError.test.ts | 2 +- .../src/workers/ChildProcessWorker.ts | 83 ++++++++++++++----- .../ChildProcessWorkerEdgeCases.test.js | 73 +++++++++++++++- .../ChildProcessWorkerEdgeCasesWorker.js | 5 ++ 5 files changed, 138 insertions(+), 29 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 73137c3e92a1..bf6e7f2bd0fc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,7 +32,7 @@ jobs: id: cpu-cores uses: SimenB/github-actions-cpu-cores@v1 - name: run tests - timeout-minutes: 10 + timeout-minutes: 20 run: yarn test-ci-partial:parallel --runInBand --shard=${{ matrix.shard }} test-jasmine: @@ -58,5 +58,5 @@ jobs: id: cpu-cores uses: SimenB/github-actions-cpu-cores@v1 - name: run tests using jest-jasmine - timeout-minutes: 10 + timeout-minutes: 20 run: yarn jest-jasmine-ci --runInBand --shard=${{ matrix.shard }} diff --git a/e2e/__tests__/fatalWorkerError.test.ts b/e2e/__tests__/fatalWorkerError.test.ts index ab7a246e2484..477932738da0 100644 --- a/e2e/__tests__/fatalWorkerError.test.ts +++ b/e2e/__tests__/fatalWorkerError.test.ts @@ -19,7 +19,7 @@ const DIR = path.resolve(tmpdir(), 'fatal-worker-error'); beforeEach(() => cleanup(DIR)); afterAll(() => cleanup(DIR)); -test.skip('fails a test that terminates the worker with a fatal error', () => { +test('fails a test that terminates the worker with a fatal error', () => { const testFiles = { ...generateTestFilesToForceUsingWorkers(), '__tests__/fatalWorkerError.test.js': ` diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index 3708f71d4ff2..891d56eeb6ec 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -31,11 +31,18 @@ import { const SIGNAL_BASE_EXIT_CODE = 128; const SIGKILL_EXIT_CODE = SIGNAL_BASE_EXIT_CODE + 9; const SIGTERM_EXIT_CODE = SIGNAL_BASE_EXIT_CODE + 15; -const SIGABRT_EXIT_CODE = SIGNAL_BASE_EXIT_CODE + 6; // How long to wait after SIGTERM before sending SIGKILL const SIGKILL_DELAY = 500; +enum WorkerStates { + STARTING = 'starting', + OK = 'ok', + OUT_OF_MEMORY = 'oom', + RESTARTING = 'restarting', + SHUTTING_DOWN = 'shutdown', +} + /** * This class wraps the child process and provides a nice interface to * communicate with. It takes care of: @@ -75,8 +82,8 @@ export default class ChildProcessWorker implements WorkerInterface { private _childIdleMemoryUsage: number | null; private _childIdleMemoryUsageLimit: number | null; - private _restarting = false; private _memoryUsageCheck = false; + private _state: WorkerStates; private _childWorkerPath: string; private _silent = true; @@ -100,11 +107,23 @@ export default class ChildProcessWorker implements WorkerInterface { options.childWorkerPath || require.resolve('./processChild'); this._silent = options.silent ?? true; + this._state = WorkerStates.STARTING; this.initialize(); } initialize(): void { - this._restarting = false; + if ( + this._state === WorkerStates.OUT_OF_MEMORY || + this._state === WorkerStates.SHUTTING_DOWN + ) { + return; + } + + if (this._child && this._child.connected) { + this._child.kill('SIGKILL'); + } + + this._state = WorkerStates.STARTING; const forceColor = stdoutSupportsColor ? {FORCE_COLOR: '1'} : {}; const options: ForkOptions = { @@ -142,6 +161,8 @@ export default class ChildProcessWorker implements WorkerInterface { } this._stderr.add(child.stderr); + + child.stderr.on('data', this._detectOutOfMemoryCrash.bind(this)); } child.on('message', this._onMessage.bind(this)); @@ -155,6 +176,7 @@ export default class ChildProcessWorker implements WorkerInterface { ]); this._child = child; + this._state = WorkerStates.OK; this._retries++; @@ -173,10 +195,33 @@ export default class ChildProcessWorker implements WorkerInterface { error.stack!, {type: 'WorkerError'}, ]); + + // Clear the request so we don't keep executing it. + this._request = null; + } + } + + private _detectOutOfMemoryCrash(chunk: any): void { + let str: string | undefined = undefined; + + if (chunk instanceof Buffer) { + str = chunk.toString('utf8'); + } else if (typeof chunk === 'string') { + str = chunk; + } + + if ( + str && + this._state !== WorkerStates.OUT_OF_MEMORY && + str.includes('JavaScript heap out of memory') + ) { + this._state = WorkerStates.OUT_OF_MEMORY; } } private _shutdown() { + this._state = WorkerStates.SHUTTING_DOWN; + // End the temporary streams so the merged streams end too if (this._fakeStream) { this._fakeStream.end(); @@ -262,25 +307,28 @@ export default class ChildProcessWorker implements WorkerInterface { this._childIdleMemoryUsage && this._childIdleMemoryUsage > limit ) { - this._restarting = true; + this._state = WorkerStates.RESTARTING; this.killChild(); } } } - private _onExit(exitCode: number | null, signal: NodeJS.Signals | null) { - // When a out of memory crash occurs - // - Mac/Unix signal=SIGABRT - // - Windows exitCode=134 + private _onExit(exitCode: number | null) { + if (exitCode !== 0 && this._state === WorkerStates.OUT_OF_MEMORY) { + this._onProcessEnd( + new Error('Jest worker ran out of memory and crashed'), + null, + ); - if ( + this._shutdown(); + } else if ( (exitCode !== 0 && exitCode !== null && exitCode !== SIGTERM_EXIT_CODE && exitCode !== SIGKILL_EXIT_CODE && - exitCode !== SIGABRT_EXIT_CODE) || - this._restarting + this._state !== WorkerStates.SHUTTING_DOWN) || + this._state === WorkerStates.RESTARTING ) { this.initialize(); @@ -288,17 +336,6 @@ export default class ChildProcessWorker implements WorkerInterface { this._child.send(this._request); } } else { - if (signal === 'SIGABRT' || exitCode === SIGABRT_EXIT_CODE) { - // When a child process worker crashes due to lack of memory this prevents - // jest from spinning and failing to exit. It could be argued it should restart - // the process, but if you're running out of memory then restarting processes - // is only going to make matters worse. - this._onProcessEnd( - new Error(`Process exited unexpectedly: ${signal}`), - null, - ); - } - this._shutdown(); } } @@ -340,6 +377,8 @@ export default class ChildProcessWorker implements WorkerInterface { } forceExit(): void { + this._state = WorkerStates.SHUTTING_DOWN; + const sigkillTimeout = this.killChild(); this._exitPromise.then(() => clearTimeout(sigkillTimeout)); } diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index f32c35ecf899..5a7d59dd09ce 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -57,7 +57,8 @@ function waitForChange(fn, limit = 100) { return new Promise((resolve, reject) => { let count = 0; - const int = setInterval(() => { + + int = setInterval(() => { const updated = fn(); if (inital !== updated) { @@ -94,7 +95,7 @@ test('should get memory usage', async () => { const memoryUsagePromise = worker.getMemoryUsage(); expect(memoryUsagePromise).toBeInstanceOf(Promise); expect(await memoryUsagePromise).toBeGreaterThan(0); -}, 10000); +}); test('should recycle on idle limit breach', async () => { worker = new ChildProcessWorker({ @@ -120,7 +121,7 @@ test('should recycle on idle limit breach', async () => { const endPid = worker.getWorkerPid(); expect(endPid).toBeGreaterThanOrEqual(0); expect(endPid).not.toEqual(startPid); -}, 10000); +}); test('should automatically recycle on idle limit breach', async () => { worker = new ChildProcessWorker({ @@ -155,7 +156,7 @@ test('should automatically recycle on idle limit breach', async () => { const endPid = worker.getWorkerPid(); expect(endPid).toBeGreaterThanOrEqual(0); expect(endPid).not.toEqual(startPid); -}, 10000); +}); test('should cleanly exit on crash', async () => { worker = new ChildProcessWorker({ @@ -189,3 +190,67 @@ test('should cleanly exit on crash', async () => { await worker.waitForExit(); }, 5000); + +test('should handle regular fatal crashes', async () => { + worker = new ChildProcessWorker({ + childWorkerPath, + maxRetries: 4, + workerPath: join( + __dirname, + '__fixtures__', + 'ChildProcessWorkerEdgeCasesWorker', + ), + }); + + const startPid = worker.getWorkerPid(); + expect(startPid).toBeGreaterThanOrEqual(0); + + const onStart = jest.fn(); + const onEnd = jest.fn(); + const onCustom = jest.fn(); + + worker.send( + [CHILD_MESSAGE_CALL, true, 'fatalExitCode', []], + onStart, + onEnd, + onCustom, + ); + + const pids = new Set(); + + while (true) { + // Ideally this would use Promise.any but it's not supported in Node 14 + // so doing this instead. + + const newPid = await new Promise(resolve => { + const resolved = false; + + const to = setTimeout(() => { + if (!resolved) { + this.resolved = true; + resolve(undefined); + } + }, 250); + + waitForChange(() => worker.getWorkerPid()).then(() => { + clearTimeout(to); + + if (!resolved) { + resolve(worker.getWorkerPid()); + } + }); + }); + + if (typeof newPid === 'number') { + pids.add(newPid); + } else { + break; + } + } + + // Expect the pids to be retries + 1 because it is restarted + // one last time at the end ready for the next request. + expect(pids.size).toEqual(5); + + worker.forceExit(); +}); diff --git a/packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js b/packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js index 460af8068bcd..9b87a49db61f 100644 --- a/packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js +++ b/packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js @@ -16,11 +16,16 @@ function leakMemory() { } } +function fatalExitCode() { + process.exit(134); +} + function safeFunction() { // Doesn't do anything. } module.exports = { + fatalExitCode, leakMemory, safeFunction, }; From 1ca06bc6b6d14cf291d5be600ef26de07dcf67c3 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 15:08:10 +0100 Subject: [PATCH 31/49] fix: out of memory test --- .../__tests__/ChildProcessWorker.test.js | 27 ++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js index 3249661604f3..ab7e91660bd7 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js @@ -411,13 +411,38 @@ it('when out of memory occurs the worker is killed and exits', async () => { expect(onProcessEnd).not.toHaveBeenCalled(); expect(onCustomMessage).not.toHaveBeenCalled(); + forkInterface.stderr.emit( + 'data', + `<--- Last few GCs ---> + + [20048:0x7fa356200000] 349 ms: Mark-sweep (reduce) 49.2 (80.6) -> 49.0 (51.6) MB, 6.8 / 0.0 ms (+ 59.5 ms in 35 steps since start of marking, biggest step 2.3 ms, walltime since start of marking 68 ms) (average mu = 0.679, current mu = 0.679) finali[20048:0x7fa356200000] 418 ms: Mark-sweep 50.0 (51.6) -> 49.9 (55.6) MB, 67.8 / 0.0 ms (average mu = 0.512, current mu = 0.004) allocation failure scavenge might not succeed + + + <--- JS stacktrace ---> + + FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory + 1: 0x10da153a5 node::Abort() (.cold.1) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] + 2: 0x10c6f09b9 node::Abort() [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] + 3: 0x10c6f0b2f node::OnFatalError(char const*, char const*) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] + 4: 0x10c86ff37 v8::Utils::ReportOOMFailure(v8::internal::Isolate*, char const*, bool) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] + 5: 0x10c86fed3 v8::internal::V8::FatalProcessOutOfMemory(v8::internal::Isolate*, char const*, bool) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] + 6: 0x10ca2eed5 v8::internal::Heap::FatalProcessOutOfMemory(char const*) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] + 7: 0x10ca2d8f6 v8::internal::Heap::CollectGarbage(v8::internal::AllocationSpace, v8::internal::GarbageCollectionReason, v8::GCCallbackFlags) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] + 8: 0x10ca39e30 v8::internal::Heap::AllocateRawWithLightRetrySlowPath(int, v8::internal::AllocationType, v8::internal::AllocationOrigin, v8::internal::AllocationAlignment) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] + 9: 0x10ca39eb1 v8::internal::Heap::AllocateRawWithRetryOrFailSlowPath(int, v8::internal::AllocationType, v8::internal::AllocationOrigin, v8::internal::AllocationAlignment) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] + 10: 0x10ca070d7 v8::internal::Factory::NewFillerObject(int, bool, v8::internal::AllocationType, v8::internal::AllocationOrigin) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] + 11: 0x10cdb896e v8::internal::Runtime_AllocateInYoungGeneration(int, unsigned long*, v8::internal::Isolate*) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] + 12: 0x10d157ed9 Builtins_CEntry_Return1_DontSaveFPRegs_ArgvOnStack_NoBuiltinExit [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] + 13: 0x10d158e0f Builtins_StringAdd_CheckNone [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] + 14: 0x10d1b5740 Builtins_StringRepeat [/Users/paul/.nvm/versions/node/v16.10.0/bin/node]`, + ); forkInterface.emit('exit', null, 'SIGABRT'); // We don't want it to try and restart. expect(childProcess.fork).toHaveBeenCalledTimes(1); expect(onProcessEnd).toHaveBeenCalledTimes(1); expect(onProcessEnd).toHaveBeenCalledWith( - new Error('Process exited unexpectedly: SIGABRT'), + new Error('Jest worker ran out of memory and crashed'), null, ); From c98dd320cad46b9d182e6e6d6b03749875a5270c Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 15:28:44 +0100 Subject: [PATCH 32/49] chore: increase timeout --- .../src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index 5a7d59dd09ce..b994529e6e5f 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -189,7 +189,7 @@ test('should cleanly exit on crash', async () => { ); await worker.waitForExit(); -}, 5000); +}, 10000); test('should handle regular fatal crashes', async () => { worker = new ChildProcessWorker({ From c21b48eff009179f5954d8008434dbd445d63e55 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 15:43:28 +0100 Subject: [PATCH 33/49] chore: debugging output --- .../src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index 5a7d59dd09ce..3700d76c5e54 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -166,7 +166,7 @@ test('should cleanly exit on crash', async () => { execArgv: ['--max-old-space-size=50'], }, maxRetries: 0, - silent: true, + silent: false, workerPath: join( __dirname, '__fixtures__', From 98f5e82d23c9af0b933b51430672b5569dcce4cb Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 16:31:24 +0100 Subject: [PATCH 34/49] chore: limit to just failing test and add logging --- .github/workflows/test.yml | 2 +- packages/jest-worker/src/workers/ChildProcessWorker.ts | 4 +++- .../src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf6e7f2bd0fc..ef631dbe79ed 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: uses: SimenB/github-actions-cpu-cores@v1 - name: run tests timeout-minutes: 20 - run: yarn test-ci-partial:parallel --runInBand --shard=${{ matrix.shard }} + run: yarn test-ci-partial:parallel --runInBand --shard=${{ matrix.shard }} ChildProcessWorkerEdgeCases test-jasmine: strategy: diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index 891d56eeb6ec..ce7b7108ae4a 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -314,7 +314,9 @@ export default class ChildProcessWorker implements WorkerInterface { } } - private _onExit(exitCode: number | null) { + private _onExit(exitCode: number | null, signal) { + console.log({exitCode, signal, state: this._state}); + if (exitCode !== 0 && this._state === WorkerStates.OUT_OF_MEMORY) { this._onProcessEnd( new Error('Jest worker ran out of memory and crashed'), diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index 4cffe968a103..864bdfa1a574 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -158,7 +158,7 @@ test('should automatically recycle on idle limit breach', async () => { expect(endPid).not.toEqual(startPid); }); -test('should cleanly exit on crash', async () => { +test.only('should cleanly exit on crash', async () => { worker = new ChildProcessWorker({ childWorkerPath, forkOptions: { From a1b647f57fe06c9b581f1de053e43fff158f1a76 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 17:01:48 +0100 Subject: [PATCH 35/49] chore: more debugging --- .circleci/config.yml | 2 +- packages/jest-worker/src/workers/ChildProcessWorker.ts | 4 ++++ .../src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js | 2 +- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ada2ec6dd9ec..55bd90364349 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -59,4 +59,4 @@ workflows: matrix: parameters: node-version: ['14', '16', '18'] - - test-jest-jasmine + # - test-jest-jasmine diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index ce7b7108ae4a..1d6c7454aa06 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -210,6 +210,10 @@ export default class ChildProcessWorker implements WorkerInterface { str = chunk; } + console.log({ + str, + }); + if ( str && this._state !== WorkerStates.OUT_OF_MEMORY && diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index 864bdfa1a574..bfa4f57dee30 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -166,7 +166,7 @@ test.only('should cleanly exit on crash', async () => { execArgv: ['--max-old-space-size=50'], }, maxRetries: 0, - silent: false, + silent: true, workerPath: join( __dirname, '__fixtures__', From 54dccb34413274984473b3da6ce9314aa2ab07e7 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 17:17:47 +0100 Subject: [PATCH 36/49] feat: add stderr concat --- .../src/workers/ChildProcessWorker.ts | 43 +++++++++++-------- .../__tests__/ChildProcessWorker.test.js | 23 ++++------ 2 files changed, 33 insertions(+), 33 deletions(-) diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index 1d6c7454aa06..dd07a6ab0280 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -161,10 +161,9 @@ export default class ChildProcessWorker implements WorkerInterface { } this._stderr.add(child.stderr); - - child.stderr.on('data', this._detectOutOfMemoryCrash.bind(this)); } + this._detectOutOfMemoryCrash(child); child.on('message', this._onMessage.bind(this)); child.on('exit', this._onExit.bind(this)); @@ -201,26 +200,32 @@ export default class ChildProcessWorker implements WorkerInterface { } } - private _detectOutOfMemoryCrash(chunk: any): void { - let str: string | undefined = undefined; + private _detectOutOfMemoryCrash(child: ChildProcess): void { + let stderrStr = ''; - if (chunk instanceof Buffer) { - str = chunk.toString('utf8'); - } else if (typeof chunk === 'string') { - str = chunk; - } + const handler = (chunk: any) => { + if (this._state !== WorkerStates.OUT_OF_MEMORY) { + let str: string | undefined = undefined; - console.log({ - str, - }); + if (chunk instanceof Buffer) { + str = chunk.toString('utf8'); + } else if (typeof chunk === 'string') { + str = chunk; + } - if ( - str && - this._state !== WorkerStates.OUT_OF_MEMORY && - str.includes('JavaScript heap out of memory') - ) { - this._state = WorkerStates.OUT_OF_MEMORY; - } + if (str) { + stderrStr += str; + } + + if (stderrStr.includes('JavaScript heap out of memory')) { + this._state = WorkerStates.OUT_OF_MEMORY; + } + } + + console.log({state: this._state, stderrStr}); + }; + + child.stderr?.on('data', handler); } private _shutdown() { diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js index ab7e91660bd7..0948ea82be57 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorker.test.js @@ -411,6 +411,7 @@ it('when out of memory occurs the worker is killed and exits', async () => { expect(onProcessEnd).not.toHaveBeenCalled(); expect(onCustomMessage).not.toHaveBeenCalled(); + // Splitting the emit into 2 to check concat is happening. forkInterface.stderr.emit( 'data', `<--- Last few GCs ---> @@ -420,22 +421,16 @@ it('when out of memory occurs the worker is killed and exits', async () => { <--- JS stacktrace ---> - FATAL ERROR: Reached heap limit Allocation failed - JavaScript heap out of memory + FATAL ERROR: Reached heap limit Allocation failed - JavaScript he`, + ); + + forkInterface.stderr.emit( + 'data', + `ap out of memory 1: 0x10da153a5 node::Abort() (.cold.1) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] - 2: 0x10c6f09b9 node::Abort() [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] - 3: 0x10c6f0b2f node::OnFatalError(char const*, char const*) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] - 4: 0x10c86ff37 v8::Utils::ReportOOMFailure(v8::internal::Isolate*, char const*, bool) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] - 5: 0x10c86fed3 v8::internal::V8::FatalProcessOutOfMemory(v8::internal::Isolate*, char const*, bool) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] - 6: 0x10ca2eed5 v8::internal::Heap::FatalProcessOutOfMemory(char const*) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] - 7: 0x10ca2d8f6 v8::internal::Heap::CollectGarbage(v8::internal::AllocationSpace, v8::internal::GarbageCollectionReason, v8::GCCallbackFlags) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] - 8: 0x10ca39e30 v8::internal::Heap::AllocateRawWithLightRetrySlowPath(int, v8::internal::AllocationType, v8::internal::AllocationOrigin, v8::internal::AllocationAlignment) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] - 9: 0x10ca39eb1 v8::internal::Heap::AllocateRawWithRetryOrFailSlowPath(int, v8::internal::AllocationType, v8::internal::AllocationOrigin, v8::internal::AllocationAlignment) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] - 10: 0x10ca070d7 v8::internal::Factory::NewFillerObject(int, bool, v8::internal::AllocationType, v8::internal::AllocationOrigin) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] - 11: 0x10cdb896e v8::internal::Runtime_AllocateInYoungGeneration(int, unsigned long*, v8::internal::Isolate*) [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] - 12: 0x10d157ed9 Builtins_CEntry_Return1_DontSaveFPRegs_ArgvOnStack_NoBuiltinExit [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] - 13: 0x10d158e0f Builtins_StringAdd_CheckNone [/Users/paul/.nvm/versions/node/v16.10.0/bin/node] - 14: 0x10d1b5740 Builtins_StringRepeat [/Users/paul/.nvm/versions/node/v16.10.0/bin/node]`, + 2: 0x10c6f09b9 node::Abort() [/Users/paul/.nvm/versions/node/v16.10.0/bin/node]`, ); + forkInterface.emit('exit', null, 'SIGABRT'); // We don't want it to try and restart. From 0f5da7d09b7695146cea1bb02462f3073d11072b Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 17:29:50 +0100 Subject: [PATCH 37/49] chore: back to full tests --- .circleci/config.yml | 4 ++-- .github/workflows/test.yml | 2 +- packages/jest-worker/src/workers/ChildProcessWorker.ts | 6 +----- .../workers/__tests__/ChildProcessWorkerEdgeCases.test.js | 2 +- 4 files changed, 5 insertions(+), 9 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 55bd90364349..61be28b5a409 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -28,7 +28,7 @@ jobs: node-version: << parameters.node-version >> - node/install-packages: *install - run: - command: yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL --runInBand + command: yarn test-ci-partial --shard=$(expr $CIRCLE_NODE_INDEX + 1)/$CIRCLE_NODE_TOTAL - store_test_results: path: reports/junit @@ -59,4 +59,4 @@ workflows: matrix: parameters: node-version: ['14', '16', '18'] - # - test-jest-jasmine + - test-jest-jasmine diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ef631dbe79ed..bf6e7f2bd0fc 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: uses: SimenB/github-actions-cpu-cores@v1 - name: run tests timeout-minutes: 20 - run: yarn test-ci-partial:parallel --runInBand --shard=${{ matrix.shard }} ChildProcessWorkerEdgeCases + run: yarn test-ci-partial:parallel --runInBand --shard=${{ matrix.shard }} test-jasmine: strategy: diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index dd07a6ab0280..f297f1c0df0a 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -221,8 +221,6 @@ export default class ChildProcessWorker implements WorkerInterface { this._state = WorkerStates.OUT_OF_MEMORY; } } - - console.log({state: this._state, stderrStr}); }; child.stderr?.on('data', handler); @@ -323,9 +321,7 @@ export default class ChildProcessWorker implements WorkerInterface { } } - private _onExit(exitCode: number | null, signal) { - console.log({exitCode, signal, state: this._state}); - + private _onExit(exitCode: number | null) { if (exitCode !== 0 && this._state === WorkerStates.OUT_OF_MEMORY) { this._onProcessEnd( new Error('Jest worker ran out of memory and crashed'), diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index bfa4f57dee30..b994529e6e5f 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -158,7 +158,7 @@ test('should automatically recycle on idle limit breach', async () => { expect(endPid).not.toEqual(startPid); }); -test.only('should cleanly exit on crash', async () => { +test('should cleanly exit on crash', async () => { worker = new ChildProcessWorker({ childWorkerPath, forkOptions: { From debf03441b69973402796973e0c86bf20f64f05b Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 17:47:58 +0100 Subject: [PATCH 38/49] test: fix use simple count in case same pid comes up twice --- .github/workflows/test.yml | 4 ++-- .../__tests__/ChildProcessWorkerEdgeCases.test.js | 12 +++++++----- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index bf6e7f2bd0fc..3952d84931f6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -33,7 +33,7 @@ jobs: uses: SimenB/github-actions-cpu-cores@v1 - name: run tests timeout-minutes: 20 - run: yarn test-ci-partial:parallel --runInBand --shard=${{ matrix.shard }} + run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} test-jasmine: strategy: @@ -59,4 +59,4 @@ jobs: uses: SimenB/github-actions-cpu-cores@v1 - name: run tests using jest-jasmine timeout-minutes: 20 - run: yarn jest-jasmine-ci --runInBand --shard=${{ matrix.shard }} + run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index b994529e6e5f..d5ef40abf27e 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -216,12 +216,14 @@ test('should handle regular fatal crashes', async () => { onCustom, ); - const pids = new Set(); + let pidChanges = 0; while (true) { // Ideally this would use Promise.any but it's not supported in Node 14 - // so doing this instead. - + // so doing this instead. Essentially what we're doing is looping and + // capturing the pid every time it changes. When it stops changing the + // timeout will be hit and we should be left with a collection of all + // the pids used by the worker. const newPid = await new Promise(resolve => { const resolved = false; @@ -242,7 +244,7 @@ test('should handle regular fatal crashes', async () => { }); if (typeof newPid === 'number') { - pids.add(newPid); + pidChanges++; } else { break; } @@ -250,7 +252,7 @@ test('should handle regular fatal crashes', async () => { // Expect the pids to be retries + 1 because it is restarted // one last time at the end ready for the next request. - expect(pids.size).toEqual(5); + expect(pidChanges).toEqual(5); worker.forceExit(); }); From 461905521beb5e82883e7a15efda8ad591d9a819 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 18:18:25 +0100 Subject: [PATCH 39/49] test: add retry and increase timeout --- e2e/__tests__/workerForceExit.test.ts | 5 ++++- .../workers/__tests__/ChildProcessWorkerEdgeCases.test.js | 5 +++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/e2e/__tests__/workerForceExit.test.ts b/e2e/__tests__/workerForceExit.test.ts index 6ce9032faf99..ce83fa8cc724 100644 --- a/e2e/__tests__/workerForceExit.test.ts +++ b/e2e/__tests__/workerForceExit.test.ts @@ -15,6 +15,9 @@ import { } from '../Utils'; import runJest from '../runJest'; +// These tests appear to be slow/flakey on Windows +jest.retryTimes(5); + const DIR = resolve(tmpdir(), 'worker-force-exit'); beforeEach(() => cleanup(DIR)); @@ -74,4 +77,4 @@ test('force exits a worker that fails to exit gracefully', async () => { expect(pidNumber).not.toBeNaN(); expect(await findProcess('pid', pidNumber)).toHaveLength(0); -}); +}, 15000); diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index d5ef40abf27e..2d2118869750 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -11,7 +11,8 @@ import {transformFileAsync} from '@babel/core'; import {CHILD_MESSAGE_CALL, CHILD_MESSAGE_MEM_USAGE} from '../../types'; import ChildProcessWorker from '../ChildProcessWorker'; -jest.retryTimes(1); +// These tests appear to be slow/flakey on Windows +jest.retryTimes(5); /** @type ChildProcessWorker */ let worker; @@ -189,7 +190,7 @@ test('should cleanly exit on crash', async () => { ); await worker.waitForExit(); -}, 10000); +}, 15000); test('should handle regular fatal crashes', async () => { worker = new ChildProcessWorker({ From da47b50176d1d88c6d1d2e2a6a289f719428e34c Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 18:42:51 +0100 Subject: [PATCH 40/49] test: more retries --- e2e/__tests__/workerForceExit.test.ts | 2 +- .../src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/e2e/__tests__/workerForceExit.test.ts b/e2e/__tests__/workerForceExit.test.ts index ce83fa8cc724..3ade72d2b3db 100644 --- a/e2e/__tests__/workerForceExit.test.ts +++ b/e2e/__tests__/workerForceExit.test.ts @@ -16,7 +16,7 @@ import { import runJest from '../runJest'; // These tests appear to be slow/flakey on Windows -jest.retryTimes(5); +jest.retryTimes(10); const DIR = resolve(tmpdir(), 'worker-force-exit'); diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js index 2d2118869750..58cbf1be515c 100644 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js @@ -12,7 +12,7 @@ import {CHILD_MESSAGE_CALL, CHILD_MESSAGE_MEM_USAGE} from '../../types'; import ChildProcessWorker from '../ChildProcessWorker'; // These tests appear to be slow/flakey on Windows -jest.retryTimes(5); +jest.retryTimes(10); /** @type ChildProcessWorker */ let worker; From ba00b26f1cf35d3cfd9e5cb2c79d3d97ae1cf6b8 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Wed, 27 Jul 2022 20:29:08 +0100 Subject: [PATCH 41/49] chore: revert changes to jest config --- jest.config.ci.mjs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/jest.config.ci.mjs b/jest.config.ci.mjs index fbf7ab6f1590..2b2f36983f5e 100644 --- a/jest.config.ci.mjs +++ b/jest.config.ci.mjs @@ -12,16 +12,15 @@ export default { ...baseConfig, coverageReporters: ['json'], reporters: [ - 'default', 'github-actions', [ 'jest-junit', {outputDirectory: 'reports/junit', outputName: 'js-test-results.xml'}, ], - // [ - // 'jest-silent-reporter', - // {showPaths: true, showWarnings: true, useDots: true}, - // ], + [ + 'jest-silent-reporter', + {showPaths: true, showWarnings: true, useDots: true}, + ], 'summary', ], }; From 019c5d41fe891d87529a3822bd7590c5715b74fd Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Thu, 28 Jul 2022 15:11:29 +0100 Subject: [PATCH 42/49] feat: port idle usage to thread workers --- packages/jest-worker/src/types.ts | 26 +- .../src/workers/ChildProcessWorker.ts | 30 +- .../src/workers/NodeThreadsWorker.ts | 174 ++++++++++- .../ChildProcessWorkerEdgeCases.test.js | 259 ---------------- .../workers/__tests__/WorkerEdgeCases.test.js | 278 ++++++++++++++++++ ...rEdgeCasesWorker.js => EdgeCasesWorker.js} | 17 +- .../jest-worker/src/workers/threadChild.ts | 20 ++ 7 files changed, 515 insertions(+), 289 deletions(-) delete mode 100644 packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js create mode 100644 packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js rename packages/jest-worker/src/workers/__tests__/__fixtures__/{ChildProcessWorkerEdgeCasesWorker.js => EdgeCasesWorker.js} (65%) diff --git a/packages/jest-worker/src/types.ts b/packages/jest-worker/src/types.ts index 52b75a757c02..302771e10b0a 100644 --- a/packages/jest-worker/src/types.ts +++ b/packages/jest-worker/src/types.ts @@ -78,6 +78,11 @@ export interface WorkerInterface { getWorkerId(): number; getStderr(): NodeJS.ReadableStream | null; getStdout(): NodeJS.ReadableStream | null; + /** + * Some system level identifier for the worker. IE, process id, thread id, etc. + */ + getWorkerSystemId(): number; + getMemoryUsage(): Promise; } export type PoolExitResult = { @@ -145,10 +150,16 @@ export type WorkerOptions = { workerId: number; workerData?: unknown; workerPath: string; + /** + * After a job has executed the memory usage it should return to. + * + * @remarks + * Note this is different from ResourceLimits in that it checks at idle, after + * a job is complete. So you could have a resource limit of 500MB but an idle + * limit of 50MB. The latter will only trigger if after a job has completed the + * memory usage hasn't returned back down under 50MB. + */ idleMemoryLimit?: number; -}; - -export type ChildProcessWorkerOptions = WorkerOptions & { /** * This mainly exists so the path can be changed during testing. * https://github.com/facebook/jest/issues/9543 @@ -245,3 +256,12 @@ export type QueueChildMessage = { onEnd: OnEnd; onCustomMessage: OnCustomMessage; }; + +export enum WorkerStates { + STARTING = 'starting', + OK = 'ok', + OUT_OF_MEMORY = 'oom', + RESTARTING = 'restarting', + SHUTTING_DOWN = 'shutting-down', + SHUT_DOWN = 'shut-down', +} diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index f297f1c0df0a..645b714cf54f 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -14,7 +14,6 @@ import { CHILD_MESSAGE_INITIALIZE, CHILD_MESSAGE_MEM_USAGE, ChildMessage, - ChildProcessWorkerOptions, OnCustomMessage, OnEnd, OnStart, @@ -26,6 +25,7 @@ import { ParentMessage, WorkerInterface, WorkerOptions, + WorkerStates, } from '../types'; const SIGNAL_BASE_EXIT_CODE = 128; @@ -35,14 +35,6 @@ const SIGTERM_EXIT_CODE = SIGNAL_BASE_EXIT_CODE + 15; // How long to wait after SIGTERM before sending SIGKILL const SIGKILL_DELAY = 500; -enum WorkerStates { - STARTING = 'starting', - OK = 'ok', - OUT_OF_MEMORY = 'oom', - RESTARTING = 'restarting', - SHUTTING_DOWN = 'shutdown', -} - /** * This class wraps the child process and provides a nice interface to * communicate with. It takes care of: @@ -86,9 +78,8 @@ export default class ChildProcessWorker implements WorkerInterface { private _state: WorkerStates; private _childWorkerPath: string; - private _silent = true; - constructor(options: ChildProcessWorkerOptions) { + constructor(options: WorkerOptions) { this._options = options; this._request = null; @@ -105,7 +96,6 @@ export default class ChildProcessWorker implements WorkerInterface { this._childWorkerPath = options.childWorkerPath || require.resolve('./processChild'); - this._silent = options.silent ?? true; this._state = WorkerStates.STARTING; this.initialize(); @@ -114,7 +104,8 @@ export default class ChildProcessWorker implements WorkerInterface { initialize(): void { if ( this._state === WorkerStates.OUT_OF_MEMORY || - this._state === WorkerStates.SHUTTING_DOWN + this._state === WorkerStates.SHUTTING_DOWN || + this._state === WorkerStates.SHUT_DOWN ) { return; } @@ -137,7 +128,7 @@ export default class ChildProcessWorker implements WorkerInterface { execArgv: process.execArgv.filter(v => !/^--(debug|inspect)/.test(v)), // default to advanced serialization in order to match worker threads serialization: 'advanced', - silent: this._silent, + silent: this._options.silent ?? true, ...this._options.forkOptions, }; @@ -217,7 +208,7 @@ export default class ChildProcessWorker implements WorkerInterface { stderrStr += str; } - if (stderrStr.includes('JavaScript heap out of memory')) { + if (stderrStr.includes('heap out of memory')) { this._state = WorkerStates.OUT_OF_MEMORY; } } @@ -356,13 +347,14 @@ export default class ChildProcessWorker implements WorkerInterface { onProcessStart(this); this._onProcessEnd = (...args) => { + // Clean the request to avoid sending past requests to workers that fail + // while waiting for a new request (timers, unhandled rejections...) + this._request = null; + if (this._childIdleMemoryUsageLimit && this._child.connected) { this.checkMemoryUsage(); } - // Clean the request to avoid sending past requests to workers that fail - // while waiting for a new request (timers, unhandled rejections...) - this._request = null; return onProcessEnd(...args); }; @@ -399,7 +391,7 @@ export default class ChildProcessWorker implements WorkerInterface { * * @returns Process id. */ - getWorkerPid(): number { + getWorkerSystemId(): number { return this._child.pid; } diff --git a/packages/jest-worker/src/workers/NodeThreadsWorker.ts b/packages/jest-worker/src/workers/NodeThreadsWorker.ts index 9cf59441f810..c013eb7cb051 100644 --- a/packages/jest-worker/src/workers/NodeThreadsWorker.ts +++ b/packages/jest-worker/src/workers/NodeThreadsWorker.ts @@ -5,22 +5,26 @@ * LICENSE file in the root directory of this source tree. */ +import {totalmem} from 'os'; import {PassThrough} from 'stream'; import {Worker} from 'worker_threads'; import mergeStream = require('merge-stream'); import { CHILD_MESSAGE_INITIALIZE, + CHILD_MESSAGE_MEM_USAGE, ChildMessage, OnCustomMessage, OnEnd, OnStart, PARENT_MESSAGE_CLIENT_ERROR, PARENT_MESSAGE_CUSTOM, + PARENT_MESSAGE_MEM_USAGE, PARENT_MESSAGE_OK, PARENT_MESSAGE_SETUP_ERROR, ParentMessage, WorkerInterface, WorkerOptions, + WorkerStates, } from '../types'; export default class ExperimentalWorker implements WorkerInterface { @@ -38,7 +42,16 @@ export default class ExperimentalWorker implements WorkerInterface { private _exitPromise: Promise; private _resolveExitPromise!: () => void; - private _forceExited: boolean; + + private _memoryUsagePromise: Promise | undefined; + private _resolveMemoryUsage: ((arg0: number) => void) | undefined; + + private _childWorkerPath: string; + + private _childIdleMemoryUsage: number | null; + private _childIdleMemoryUsageLimit: number | null; + private _memoryUsageCheck = false; + private _state: WorkerStates; constructor(options: WorkerOptions) { this._options = options; @@ -49,16 +62,36 @@ export default class ExperimentalWorker implements WorkerInterface { this._stdout = null; this._stderr = null; + this._childWorkerPath = + options.childWorkerPath || require.resolve('./threadChild'); + + this._childIdleMemoryUsage = null; + this._childIdleMemoryUsageLimit = options.idleMemoryLimit || null; + this._exitPromise = new Promise(resolve => { this._resolveExitPromise = resolve; }); - this._forceExited = false; + this._state = WorkerStates.STARTING; this.initialize(); } initialize(): void { - this._worker = new Worker(require.resolve('./threadChild'), { + if ( + this._state === WorkerStates.OUT_OF_MEMORY || + this._state === WorkerStates.SHUTTING_DOWN || + this._state === WorkerStates.SHUT_DOWN + ) { + return; + } + + if (this._worker) { + this._worker.terminate(); + } + + this._state = WorkerStates.STARTING; + + this._worker = new Worker(this._childWorkerPath, { eval: false, resourceLimits: this._options.resourceLimits, stderr: true, @@ -87,8 +120,17 @@ export default class ExperimentalWorker implements WorkerInterface { this._stderr.add(this._worker.stderr); } + // This can be useful for debugging. + if (!(this._options.silent ?? true)) { + this._worker.stdout.setEncoding('utf8'); + this._worker.stdout.on('data', console.log); + this._worker.stderr.setEncoding('utf8'); + this._worker.stderr.on('data', console.error); + } + this._worker.on('message', this._onMessage.bind(this)); this._worker.on('exit', this._onExit.bind(this)); + this._worker.on('error', this._onError.bind(this)); this._worker.postMessage([ CHILD_MESSAGE_INITIALIZE, @@ -126,6 +168,12 @@ export default class ExperimentalWorker implements WorkerInterface { this._resolveExitPromise(); } + private _onError(error: Error) { + if (error.message.includes('heap out of memory')) { + this._state = WorkerStates.OUT_OF_MEMORY; + } + } + private _onMessage(response: ParentMessage) { let error; @@ -155,6 +203,7 @@ export default class ExperimentalWorker implements WorkerInterface { this._onProcessEnd(error, null); break; + case PARENT_MESSAGE_SETUP_ERROR: error = new Error(`Error when calling setup: ${response[2]}`); @@ -164,16 +213,43 @@ export default class ExperimentalWorker implements WorkerInterface { this._onProcessEnd(error, null); break; + case PARENT_MESSAGE_CUSTOM: this._onCustomMessage(response[1]); break; + + case PARENT_MESSAGE_MEM_USAGE: + this._childIdleMemoryUsage = response[1]; + + if (this._resolveMemoryUsage) { + this._resolveMemoryUsage(response[1]); + + this._resolveMemoryUsage = undefined; + this._memoryUsagePromise = undefined; + } + + this._performRestartIfRequired(); + break; + default: throw new TypeError(`Unexpected response from worker: ${response[0]}`); } } private _onExit(exitCode: number) { - if (exitCode !== 0 && !this._forceExited) { + if (exitCode !== 0 && this._state === WorkerStates.OUT_OF_MEMORY) { + this._onProcessEnd( + new Error('Jest worker ran out of memory and crashed'), + null, + ); + + this._shutdown(); + } else if ( + (exitCode !== 0 && + this._state !== WorkerStates.SHUTTING_DOWN && + this._state !== WorkerStates.SHUT_DOWN) || + this._state === WorkerStates.RESTARTING + ) { this.initialize(); if (this._request) { @@ -189,7 +265,7 @@ export default class ExperimentalWorker implements WorkerInterface { } forceExit(): void { - this._forceExited = true; + this._state = WorkerStates.SHUTTING_DOWN; this._worker.terminate(); } @@ -205,6 +281,10 @@ export default class ExperimentalWorker implements WorkerInterface { // while waiting for a new request (timers, unhandled rejections...) this._request = null; + if (this._childIdleMemoryUsageLimit) { + this.checkMemoryUsage(); + } + const res = onProcessEnd?.(...args); // Clean up the reference so related closures can be garbage collected. @@ -233,6 +313,90 @@ export default class ExperimentalWorker implements WorkerInterface { return this._stderr; } + private _performRestartIfRequired(): void { + if (this._memoryUsageCheck) { + this._memoryUsageCheck = false; + + let limit = this._childIdleMemoryUsageLimit; + + if (limit && limit > 0 && limit <= 1) { + limit = Math.floor(totalmem() * limit); + } + + if ( + limit && + this._childIdleMemoryUsage && + this._childIdleMemoryUsage > limit + ) { + this._state = WorkerStates.RESTARTING; + + this._worker.terminate(); + } + } + } + + /** + * Gets the last reported memory usage. + * + * @returns Memory usage in bytes. + */ + getMemoryUsage(): Promise { + if (!this._memoryUsagePromise) { + let rejectCallback!: (err: Error) => void; + + const promise = new Promise((resolve, reject) => { + this._resolveMemoryUsage = resolve; + rejectCallback = reject; + }); + this._memoryUsagePromise = promise; + + if (!this._worker.threadId) { + rejectCallback(new Error('Child process is not running.')); + + this._memoryUsagePromise = undefined; + this._resolveMemoryUsage = undefined; + + return promise; + } + + try { + this._worker.postMessage([CHILD_MESSAGE_MEM_USAGE]); + } catch (err: any) { + this._memoryUsagePromise = undefined; + this._resolveMemoryUsage = undefined; + + rejectCallback(err); + } + + return promise; + } + + return this._memoryUsagePromise; + } + + /** + * Gets updated memory usage and restarts if required + */ + checkMemoryUsage(): void { + if (this._childIdleMemoryUsageLimit) { + this._memoryUsageCheck = true; + this._worker.postMessage([CHILD_MESSAGE_MEM_USAGE]); + } else { + console.warn( + 'Memory usage of workers can only be checked if a limit is set', + ); + } + } + + /** + * Gets the thread id of the worker. + * + * @returns Thread id. + */ + getWorkerSystemId(): number { + return this._worker.threadId; + } + private _getFakeStream() { if (!this._fakeStream) { this._fakeStream = new PassThrough(); diff --git a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js deleted file mode 100644 index 58cbf1be515c..000000000000 --- a/packages/jest-worker/src/workers/__tests__/ChildProcessWorkerEdgeCases.test.js +++ /dev/null @@ -1,259 +0,0 @@ -/** - * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. - * - * This source code is licensed under the MIT license found in the - * LICENSE file in the root directory of this source tree. - */ - -import {access, mkdir, rm, writeFile} from 'fs/promises'; -import {dirname, join} from 'path'; -import {transformFileAsync} from '@babel/core'; -import {CHILD_MESSAGE_CALL, CHILD_MESSAGE_MEM_USAGE} from '../../types'; -import ChildProcessWorker from '../ChildProcessWorker'; - -// These tests appear to be slow/flakey on Windows -jest.retryTimes(10); - -/** @type ChildProcessWorker */ -let worker; -let int; - -const root = join('../../'); -const filesToBuild = ['workers/processChild', 'types']; -const writeDestination = join(__dirname, '__temp__'); -const childWorkerPath = join(writeDestination, 'workers/processChild.js'); - -beforeAll(async () => { - await mkdir(writeDestination, {recursive: true}); - - for (const file of filesToBuild) { - const sourcePath = join(__dirname, root, `${file}.ts`); - const writePath = join(writeDestination, `${file}.js`); - - await mkdir(dirname(writePath), {recursive: true}); - - const result = await transformFileAsync(sourcePath); - - await writeFile(writePath, result.code, { - encoding: 'utf-8', - }); - } -}); - -afterAll(async () => { - await rm(writeDestination, {force: true, recursive: true}); -}); - -afterEach(async () => { - if (worker) { - worker.forceExit(); - await worker.waitForExit(); - } - - clearInterval(int); -}); - -function waitForChange(fn, limit = 100) { - const inital = fn(); - - return new Promise((resolve, reject) => { - let count = 0; - - int = setInterval(() => { - const updated = fn(); - - if (inital !== updated) { - resolve(updated); - clearInterval(int); - } - - if (count > limit) { - reject(new Error('Timeout waiting for change')); - } - - count++; - }, 50); - }); -} - -test.each(filesToBuild)('%s.js should exist', async file => { - const path = join(writeDestination, `${file}.js`); - - await expect(async () => await access(path)).not.toThrowError(); -}); - -test('should get memory usage', async () => { - worker = new ChildProcessWorker({ - childWorkerPath, - maxRetries: 0, - workerPath: join( - __dirname, - '__fixtures__', - 'ChildProcessWorkerEdgeCasesWorker', - ), - }); - - const memoryUsagePromise = worker.getMemoryUsage(); - expect(memoryUsagePromise).toBeInstanceOf(Promise); - expect(await memoryUsagePromise).toBeGreaterThan(0); -}); - -test('should recycle on idle limit breach', async () => { - worker = new ChildProcessWorker({ - childWorkerPath, - // There is no way this is fitting into 1000 bytes, so it should restart - // after requesting a memory usage update - idleMemoryLimit: 1000, - maxRetries: 0, - workerPath: join( - __dirname, - '__fixtures__', - 'ChildProcessWorkerEdgeCasesWorker', - ), - }); - - const startPid = worker.getWorkerPid(); - expect(startPid).toBeGreaterThanOrEqual(0); - - worker.checkMemoryUsage(); - - await waitForChange(() => worker.getWorkerPid()); - - const endPid = worker.getWorkerPid(); - expect(endPid).toBeGreaterThanOrEqual(0); - expect(endPid).not.toEqual(startPid); -}); - -test('should automatically recycle on idle limit breach', async () => { - worker = new ChildProcessWorker({ - childWorkerPath, - // There is no way this is fitting into 1000 bytes, so it should restart - // after requesting a memory usage update - idleMemoryLimit: 1000, - maxRetries: 0, - workerPath: join( - __dirname, - '__fixtures__', - 'ChildProcessWorkerEdgeCasesWorker', - ), - }); - - const startPid = worker.getWorkerPid(); - expect(startPid).toBeGreaterThanOrEqual(0); - - const onStart = jest.fn(); - const onEnd = jest.fn(); - const onCustom = jest.fn(); - - worker.send( - [CHILD_MESSAGE_CALL, true, 'safeFunction', []], - onStart, - onEnd, - onCustom, - ); - - await waitForChange(() => worker.getWorkerPid()); - - const endPid = worker.getWorkerPid(); - expect(endPid).toBeGreaterThanOrEqual(0); - expect(endPid).not.toEqual(startPid); -}); - -test('should cleanly exit on crash', async () => { - worker = new ChildProcessWorker({ - childWorkerPath, - forkOptions: { - // Forcibly set the heap limit so we can crash the process easily. - execArgv: ['--max-old-space-size=50'], - }, - maxRetries: 0, - silent: true, - workerPath: join( - __dirname, - '__fixtures__', - 'ChildProcessWorkerEdgeCasesWorker', - ), - }); - - const pid = worker.getWorkerPid(); - expect(pid).toBeGreaterThanOrEqual(0); - - const onStart = jest.fn(); - const onEnd = jest.fn(); - const onCustom = jest.fn(); - - worker.send( - [CHILD_MESSAGE_CALL, true, 'leakMemory', []], - onStart, - onEnd, - onCustom, - ); - - await worker.waitForExit(); -}, 15000); - -test('should handle regular fatal crashes', async () => { - worker = new ChildProcessWorker({ - childWorkerPath, - maxRetries: 4, - workerPath: join( - __dirname, - '__fixtures__', - 'ChildProcessWorkerEdgeCasesWorker', - ), - }); - - const startPid = worker.getWorkerPid(); - expect(startPid).toBeGreaterThanOrEqual(0); - - const onStart = jest.fn(); - const onEnd = jest.fn(); - const onCustom = jest.fn(); - - worker.send( - [CHILD_MESSAGE_CALL, true, 'fatalExitCode', []], - onStart, - onEnd, - onCustom, - ); - - let pidChanges = 0; - - while (true) { - // Ideally this would use Promise.any but it's not supported in Node 14 - // so doing this instead. Essentially what we're doing is looping and - // capturing the pid every time it changes. When it stops changing the - // timeout will be hit and we should be left with a collection of all - // the pids used by the worker. - const newPid = await new Promise(resolve => { - const resolved = false; - - const to = setTimeout(() => { - if (!resolved) { - this.resolved = true; - resolve(undefined); - } - }, 250); - - waitForChange(() => worker.getWorkerPid()).then(() => { - clearTimeout(to); - - if (!resolved) { - resolve(worker.getWorkerPid()); - } - }); - }); - - if (typeof newPid === 'number') { - pidChanges++; - } else { - break; - } - } - - // Expect the pids to be retries + 1 because it is restarted - // one last time at the end ready for the next request. - expect(pidChanges).toEqual(5); - - worker.forceExit(); -}); diff --git a/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js new file mode 100644 index 000000000000..2c50d13f59f9 --- /dev/null +++ b/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js @@ -0,0 +1,278 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import {access, mkdir, rm, writeFile} from 'fs/promises'; +import {dirname, join} from 'path'; +import {transformFileAsync} from '@babel/core'; +import { + CHILD_MESSAGE_CALL, + CHILD_MESSAGE_MEM_USAGE, + WorkerInterface, + WorkerOptions, +} from '../../types'; +import ChildProcessWorker from '../ChildProcessWorker'; +import ThreadsWorker from '../NodeThreadsWorker'; + +// These tests appear to be slow/flakey on Windows +jest.retryTimes(10); + +const root = join('../../'); +const filesToBuild = ['workers/processChild', 'workers/threadChild', 'types']; +const writeDestination = join(__dirname, '__temp__'); +const processChildWorkerPath = join( + writeDestination, + 'workers/processChild.js', +); +const threadChildWorkerPath = join(writeDestination, 'workers/threadChild.js'); + +beforeAll(async () => { + await mkdir(writeDestination, {recursive: true}); + + for (const file of filesToBuild) { + const sourcePath = join(__dirname, root, `${file}.ts`); + const writePath = join(writeDestination, `${file}.js`); + + await mkdir(dirname(writePath), {recursive: true}); + + const result = await transformFileAsync(sourcePath); + + await writeFile(writePath, result.code, { + encoding: 'utf-8', + }); + } +}); + +afterAll(async () => { + await rm(writeDestination, {force: true, recursive: true}); +}); + +test.each(filesToBuild)('%s.js should exist', async file => { + const path = join(writeDestination, `${file}.js`); + + await expect(async () => await access(path)).not.toThrowError(); +}); + +describe.each([ + { + name: 'ProcessWorker', + workerClass: ChildProcessWorker, + workerPath: processChildWorkerPath, + }, + { + name: 'ThreadWorker', + workerClass: ThreadsWorker, + workerPath: threadChildWorkerPath, + }, +])('$name', ({workerClass, workerPath}) => { + /** @type WorkerInterface */ + let worker; + let int; + + afterEach(async () => { + if (worker) { + worker.forceExit(); + await worker.waitForExit(); + } + + clearInterval(int); + }); + + function waitForChange(fn, limit = 100) { + const inital = fn(); + + return new Promise((resolve, reject) => { + let count = 0; + + int = setInterval(() => { + const updated = fn(); + + if (inital !== updated) { + resolve(updated); + clearInterval(int); + } + + if (count > limit) { + reject(new Error('Timeout waiting for change')); + } + + count++; + }, 50); + }); + } + + test('should get memory usage', async () => { + worker = new workerClass({ + childWorkerPath: workerPath, + maxRetries: 0, + workerPath: join(__dirname, '__fixtures__', 'EdgeCasesWorker'), + }); + + const memoryUsagePromise = worker.getMemoryUsage(); + expect(memoryUsagePromise).toBeInstanceOf(Promise); + + expect(await memoryUsagePromise).toBeGreaterThan(0); + }); + + test('should recycle on idle limit breach', async () => { + worker = new workerClass({ + childWorkerPath: workerPath, + // There is no way this is fitting into 1000 bytes, so it should restart + // after requesting a memory usage update + idleMemoryLimit: 1000, + maxRetries: 0, + workerPath: join(__dirname, '__fixtures__', 'EdgeCasesWorker'), + }); + + const startSystemId = worker.getWorkerSystemId(); + expect(startSystemId).toBeGreaterThanOrEqual(0); + + worker.checkMemoryUsage(); + + await waitForChange(() => worker.getWorkerSystemId()); + + const systemId = worker.getWorkerSystemId(); + expect(systemId).toBeGreaterThanOrEqual(0); + expect(systemId).not.toEqual(startSystemId); + }); + + test('should automatically recycle on idle limit breach', async () => { + worker = new workerClass({ + childWorkerPath: workerPath, + // There is no way this is fitting into 1000 bytes, so it should restart + // after requesting a memory usage update + idleMemoryLimit: 1000, + maxRetries: 0, + workerPath: join(__dirname, '__fixtures__', 'EdgeCasesWorker'), + }); + + const startPid = worker.getWorkerSystemId(); + expect(startPid).toBeGreaterThanOrEqual(0); + + const onStart = jest.fn(); + const onEnd = jest.fn(); + const onCustom = jest.fn(); + + worker.send( + [CHILD_MESSAGE_CALL, true, 'safeFunction', []], + onStart, + onEnd, + onCustom, + ); + + await waitForChange(() => worker.getWorkerSystemId()); + + const endPid = worker.getWorkerSystemId(); + expect(endPid).toBeGreaterThanOrEqual(0); + expect(endPid).not.toEqual(startPid); + }); + + test('should cleanly exit on crash', async () => { + const workerHeapLimit = 10; + + /** @type WorkerOptions */ + const options = { + childWorkerPath: workerPath, + maxRetries: 0, + silent: true, + workerPath: join(__dirname, '__fixtures__', 'EdgeCasesWorker'), + }; + + if (workerClass === ThreadsWorker) { + options.resourceLimits = { + codeRangeSizeMb: workerHeapLimit * 2, + maxOldGenerationSizeMb: workerHeapLimit, + maxYoungGenerationSizeMb: workerHeapLimit * 2, + stackSizeMb: workerHeapLimit * 2, + }; + } else if (workerClass === ChildProcessWorker) { + options.forkOptions = { + // Forcibly set the heap limit so we can crash the process easily. + execArgv: [`--max-old-space-size=${workerHeapLimit}`], + }; + } + + worker = new workerClass(options); + + const pid = worker.getWorkerSystemId(); + expect(pid).toBeGreaterThanOrEqual(0); + + const onStart = jest.fn(); + const onEnd = jest.fn(); + const onCustom = jest.fn(); + + worker.send( + [CHILD_MESSAGE_CALL, true, 'leakMemory', []], + onStart, + onEnd, + onCustom, + ); + + await worker.waitForExit(); + }, 15000); + + test('should handle regular fatal crashes', async () => { + worker = new workerClass({ + childWorkerPath: workerPath, + maxRetries: 4, + workerPath: join(__dirname, '__fixtures__', 'EdgeCasesWorker'), + }); + + const startPid = worker.getWorkerSystemId(); + expect(startPid).toBeGreaterThanOrEqual(0); + + const onStart = jest.fn(); + const onEnd = jest.fn(); + const onCustom = jest.fn(); + + worker.send( + [CHILD_MESSAGE_CALL, true, 'fatalExitCode', []], + onStart, + onEnd, + onCustom, + ); + + let pidChanges = 0; + + while (true) { + // Ideally this would use Promise.any but it's not supported in Node 14 + // so doing this instead. Essentially what we're doing is looping and + // capturing the pid every time it changes. When it stops changing the + // timeout will be hit and we should be left with a collection of all + // the pids used by the worker. + const newPid = await new Promise(resolve => { + const resolved = false; + + const to = setTimeout(() => { + if (!resolved) { + this.resolved = true; + resolve(undefined); + } + }, 250); + + waitForChange(() => worker.getWorkerSystemId()).then(() => { + clearTimeout(to); + + if (!resolved) { + resolve(worker.getWorkerSystemId()); + } + }); + }); + + if (typeof newPid === 'number') { + pidChanges++; + } else { + break; + } + } + + // Expect the pids to be retries + 1 because it is restarted + // one last time at the end ready for the next request. + expect(pidChanges).toEqual(5); + + worker.forceExit(); + }); +}); diff --git a/packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js b/packages/jest-worker/src/workers/__tests__/__fixtures__/EdgeCasesWorker.js similarity index 65% rename from packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js rename to packages/jest-worker/src/workers/__tests__/__fixtures__/EdgeCasesWorker.js index 9b87a49db61f..482fd7c33508 100644 --- a/packages/jest-worker/src/workers/__tests__/__fixtures__/ChildProcessWorkerEdgeCasesWorker.js +++ b/packages/jest-worker/src/workers/__tests__/__fixtures__/EdgeCasesWorker.js @@ -5,14 +5,25 @@ * LICENSE file in the root directory of this source tree. */ -let leakStore = ''; +const leakStore = []; /** * This exists to force a memory leak in the worker tests. */ -function leakMemory() { +async function leakMemory() { + console.log( + `Intentionally leaking memory: ${( + process.memoryUsage().heapUsed / + 1024 / + 1024 + ).toFixed(2)}MB at start`, + ); + + let i = 0; while (true) { - leakStore += '#'.repeat(1000); + i++; + + leakStore.push(i); } } diff --git a/packages/jest-worker/src/workers/threadChild.ts b/packages/jest-worker/src/workers/threadChild.ts index f0c41563cf15..8abfbde77382 100644 --- a/packages/jest-worker/src/workers/threadChild.ts +++ b/packages/jest-worker/src/workers/threadChild.ts @@ -10,12 +10,15 @@ import { CHILD_MESSAGE_CALL, CHILD_MESSAGE_END, CHILD_MESSAGE_INITIALIZE, + CHILD_MESSAGE_MEM_USAGE, ChildMessageCall, ChildMessageInitialize, PARENT_MESSAGE_CLIENT_ERROR, PARENT_MESSAGE_ERROR, + PARENT_MESSAGE_MEM_USAGE, PARENT_MESSAGE_OK, PARENT_MESSAGE_SETUP_ERROR, + ParentMessageMemUsage, } from '../types'; type UnknownFunction = (...args: Array) => unknown | Promise; @@ -55,6 +58,10 @@ const messageListener = (request: any) => { end(); break; + case CHILD_MESSAGE_MEM_USAGE: + reportMemoryUsage(); + break; + default: throw new TypeError( `Unexpected request from parent process: ${request[0]}`, @@ -63,6 +70,19 @@ const messageListener = (request: any) => { }; parentPort!.on('message', messageListener); +function reportMemoryUsage() { + if (isMainThread) { + throw new Error('Child can only be used on a forked process'); + } + + const msg: ParentMessageMemUsage = [ + PARENT_MESSAGE_MEM_USAGE, + process.memoryUsage().heapUsed, + ]; + + parentPort!.postMessage(msg); +} + function reportSuccess(result: unknown) { if (isMainThread) { throw new Error('Child can only be used on a forked process'); From 7239650f581567c8a0b87efd3abd9fdf6c9d8d11 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Thu, 28 Jul 2022 15:40:59 +0100 Subject: [PATCH 43/49] chore: docs --- docs/Configuration.md | 6 +++--- packages/jest-worker/README.md | 15 +++++++++++++++ .../src/workers/__tests__/WorkerEdgeCases.test.js | 2 +- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/docs/Configuration.md b/docs/Configuration.md index 7ad06d09cfdc..50a337f9fc9f 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -2260,10 +2260,10 @@ Default: `undefined` Specifies the memory limit for workers before they are recycled and is primarily a work-around for [this issue](https://github.com/facebook/jest/issues/11956); -After the worker has executed a test the memory usage of it is checked. If it exceeds the value specified the worker is killed and restarted. The limit can be specified in 2 ways +After the worker has executed a test the memory usage of it is checked. If it exceeds the value specified the worker is killed and restarted. The limit can be specified in 2 ways: -- < 1 - The value is assumed to be a percentage of system memory. So 0.5 sets the memory limit of the worker to half of the total system memory -- \> 1 - Assumed to be a fixed byte value +- `< 1` - The value is assumed to be a percentage of system memory. So 0.5 sets the memory limit of the worker to half of the total system memory +- `\> 1` - Assumed to be a fixed byte value ```js tab /** @type {import('jest').Config} */ diff --git a/packages/jest-worker/README.md b/packages/jest-worker/README.md index c8a29401c19d..68db9ce1792a 100644 --- a/packages/jest-worker/README.md +++ b/packages/jest-worker/README.md @@ -75,6 +75,15 @@ List of method names that can be called on the child processes from the parent p Allow customizing all options passed to `child_process.fork`. By default, some values are set (`cwd`, `env`, `execArgv` and `serialization`), but you can override them and customize the rest. For a list of valid values, check [the Node documentation](https://nodejs.org/api/child_process.html#child_process_child_process_fork_modulepath_args_options). +#### `idleMemoryLimit: number` (optional) + +Specifies the memory limit for workers before they are recycled and is primarily a work-around for [this issue](https://github.com/facebook/jest/issues/11956); + +After the worker has executed a task the memory usage of it is checked. If it exceeds the value specified the worker is killed and restarted. If no limit is set this process does not occur. The limit can be specified in 2 ways: + +- `< 1` - The value is assumed to be a percentage of system memory. So 0.5 sets the memory limit of the worker to half of the total system memory +- `\> 1` - Assumed to be a fixed byte value. So 5000 would set an idle limit of 5000 bytes. + #### `maxRetries: number` (optional) Maximum amount of times that a dead child can be re-spawned, per call. Defaults to `3`, pass `Infinity` to allow endless retries. @@ -87,6 +96,12 @@ Amount of workers to spawn. Defaults to the number of CPUs minus 1. The `resourceLimits` option which will be passed to `worker_threads` workers. +#### `silent: Boolean` (optional) + +Set to false for `stdout` and `stderr` to be logged to console. + +By default this is true. + #### `setupArgs: Array` (optional) The arguments that will be passed to the `setup` method during initialization. diff --git a/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js index 2c50d13f59f9..122267089d19 100644 --- a/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js @@ -251,7 +251,7 @@ describe.each([ this.resolved = true; resolve(undefined); } - }, 250); + }, 500); waitForChange(() => worker.getWorkerSystemId()).then(() => { clearTimeout(to); From fdb4a6610c17fa56483ed0334a940d39686eb008 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Thu, 28 Jul 2022 15:50:48 +0100 Subject: [PATCH 44/49] chore: linting --- packages/jest-worker/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/jest-worker/README.md b/packages/jest-worker/README.md index 68db9ce1792a..19f07ff91073 100644 --- a/packages/jest-worker/README.md +++ b/packages/jest-worker/README.md @@ -98,9 +98,9 @@ The `resourceLimits` option which will be passed to `worker_threads` workers. #### `silent: Boolean` (optional) -Set to false for `stdout` and `stderr` to be logged to console. +Set to false for `stdout` and `stderr` to be logged to console. -By default this is true. +By default this is true. #### `setupArgs: Array` (optional) From 3e3be9917fc7901897ba5ddee0125a8ace96e0d4 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 1 Aug 2022 17:31:01 +0100 Subject: [PATCH 45/49] feat: add shorthand idle limit parsing --- docs/Configuration.md | 17 ++- .../src/__tests__/normalize.test.ts | 12 ++ .../src/__tests__/stringToBytes.test.ts | 120 ++++++++++++++++++ packages/jest-config/src/normalize.ts | 6 +- packages/jest-config/src/stringToBytes.ts | 83 ++++++++++++ packages/jest-worker/README.md | 4 +- .../src/workers/ChildProcessWorker.ts | 6 + .../src/workers/NodeThreadsWorker.ts | 6 + .../workers/__tests__/WorkerEdgeCases.test.js | 5 +- 9 files changed, 250 insertions(+), 9 deletions(-) create mode 100644 packages/jest-config/src/__tests__/stringToBytes.test.ts create mode 100644 packages/jest-config/src/stringToBytes.ts diff --git a/docs/Configuration.md b/docs/Configuration.md index 50a337f9fc9f..2049cf2548f3 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -2260,10 +2260,19 @@ Default: `undefined` Specifies the memory limit for workers before they are recycled and is primarily a work-around for [this issue](https://github.com/facebook/jest/issues/11956); -After the worker has executed a test the memory usage of it is checked. If it exceeds the value specified the worker is killed and restarted. The limit can be specified in 2 ways: - -- `< 1` - The value is assumed to be a percentage of system memory. So 0.5 sets the memory limit of the worker to half of the total system memory -- `\> 1` - Assumed to be a fixed byte value +After the worker has executed a test the memory usage of it is checked. If it exceeds the value specified the worker is killed and restarted. The limit can be specified in a number of different ways and whatever the result is `Math.floor` is used to turn it into an integer value: + +- `<= 1` - The value is assumed to be a percentage of system memory. So 0.5 sets the memory limit of the worker to half of the total system memory +- `\> 1` - Assumed to be a fixed byte value. Because of the previous rule if you wanted a value of 1 byte (I don't know why) you could use `1.1`. +- With units + - `50%` - As above, a percentage of total system memory + - `100KB`, `65MB`, etc - With units to denote a fixed memory limit. + - `K` / `KB` - Kilobytes (x1000) + - `KiB` - Kibibytes (x1024) + - `M` / `MB` - Megabytes + - `MiB` - Mebibytes + - `G` / `GB` - Gigabytes + - `GiB` - Gibibytes ```js tab /** @type {import('jest').Config} */ diff --git a/packages/jest-config/src/__tests__/normalize.test.ts b/packages/jest-config/src/__tests__/normalize.test.ts index 4ec6c24d5963..39f4c0da60a1 100644 --- a/packages/jest-config/src/__tests__/normalize.test.ts +++ b/packages/jest-config/src/__tests__/normalize.test.ts @@ -2112,3 +2112,15 @@ describe('logs a deprecation warning', () => { expect(console.warn).toMatchSnapshot(); }); }); + +it('parses workerIdleMemoryLimit', async () => { + const {options} = await normalize( + { + rootDir: '/root/', + workerIdleMemoryLimit: '45MiB', + }, + {} as Config.Argv, + ); + + expect(options.workerIdleMemoryLimit).toEqual(47185920); +}); diff --git a/packages/jest-config/src/__tests__/stringToBytes.test.ts b/packages/jest-config/src/__tests__/stringToBytes.test.ts new file mode 100644 index 000000000000..ea86876fa03f --- /dev/null +++ b/packages/jest-config/src/__tests__/stringToBytes.test.ts @@ -0,0 +1,120 @@ +import stringToBytes from '../stringToBytes'; + +describe('numeric input', () => { + test('> 1 represents bytes', () => { + expect(stringToBytes(50.8)).toEqual(50); + }); + + test('1.1 should be a 1', () => { + expect(stringToBytes(1.1, 54)).toEqual(1); + }); + + test('< 1 represents a %', () => { + expect(stringToBytes(0.3, 51)).toEqual(15); + }); + + test('should throw when no reference supplied', () => { + expect(() => stringToBytes(0.3)).toThrowError(); + }); + + test('should throw on a bad input', () => { + expect(() => stringToBytes(-0.3, 51)).toThrowError(); + }); +}); + +describe('string input', () => { + describe('numeric passthrough', () => { + test('> 1 represents bytes', () => { + expect(stringToBytes('50.8')).toEqual(50); + }); + + test('< 1 represents a %', () => { + expect(stringToBytes('0.3', 51)).toEqual(15); + }); + + test('should throw when no reference supplied', () => { + expect(() => stringToBytes('0.3')).toThrowError(); + }); + + test('should throw on a bad input', () => { + expect(() => stringToBytes('-0.3', 51)).toThrowError(); + }); + }); + + describe('parsing', () => { + test('0% should throw an error', () => { + expect(() => stringToBytes('0%', 51)).toThrowError(); + }); + + test('30%', () => { + expect(stringToBytes('30%', 51)).toEqual(15); + }); + + test('80%', () => { + expect(stringToBytes('80%', 51)).toEqual(40); + }); + + test('100%', () => { + expect(stringToBytes('100%', 51)).toEqual(51); + }); + + // The units caps is intentionally janky to test for forgiving string parsing. + describe('k', () => { + test('30k', () => { + expect(stringToBytes('30K')).toEqual(30000); + }); + + test('30KB', () => { + expect(stringToBytes('30kB')).toEqual(30000); + }); + + test('30KiB', () => { + expect(stringToBytes('30kIb')).toEqual(30720); + }); + }); + + describe('m', () => { + test('30M', () => { + expect(stringToBytes('30M')).toEqual(30000000); + }); + + test('30MB', () => { + expect(stringToBytes('30MB')).toEqual(30000000); + }); + + test('30MiB', () => { + expect(stringToBytes('30MiB')).toEqual(31457280); + }); + }); + + describe('g', () => { + test('30G', () => { + expect(stringToBytes('30G')).toEqual(30000000000); + }); + + test('30GB', () => { + expect(stringToBytes('30gB')).toEqual(30000000000); + }); + + test('30GiB', () => { + expect(stringToBytes('30GIB')).toEqual(32212254720); + }); + }); + + test('unknown unit', () => { + expect(() => stringToBytes('50XX')).toThrowError(); + }); + }); +}); + +test('nesting', () => { + expect(stringToBytes(stringToBytes(stringToBytes('30%', 51)))).toEqual(15); +}); + +test('null', () => { + expect(stringToBytes(null)).toEqual(null); +}); + +test('undefined', () => { + expect(stringToBytes(undefined)).toEqual(undefined); +}); diff --git a/packages/jest-config/src/normalize.ts b/packages/jest-config/src/normalize.ts index 421fdec11e27..753955c89be2 100644 --- a/packages/jest-config/src/normalize.ts +++ b/packages/jest-config/src/normalize.ts @@ -6,6 +6,7 @@ */ import {createHash} from 'crypto'; +import {totalmem} from 'os'; import * as path from 'path'; import chalk = require('chalk'); import merge = require('deepmerge'); @@ -36,6 +37,7 @@ import {DEFAULT_JS_PATTERN} from './constants'; import getMaxWorkers from './getMaxWorkers'; import {parseShardPair} from './parseShardPair'; import setFromArgv from './setFromArgv'; +import stringToBytes from './stringToBytes'; import { BULLET, DOCUMENTATION_NOTE, @@ -967,9 +969,11 @@ export default async function normalize( case 'watch': case 'watchAll': case 'watchman': - case 'workerIdleMemoryLimit': value = oldOptions[key]; break; + case 'workerIdleMemoryLimit': + value = stringToBytes(oldOptions[key], totalmem()); + break; case 'watchPlugins': value = (oldOptions[key] || []).map(watchPlugin => { if (typeof watchPlugin === 'string') { diff --git a/packages/jest-config/src/stringToBytes.ts b/packages/jest-config/src/stringToBytes.ts new file mode 100644 index 000000000000..65176e0ce250 --- /dev/null +++ b/packages/jest-config/src/stringToBytes.ts @@ -0,0 +1,83 @@ +function stringToBytes( + input: undefined, + percentageReference?: number, +): undefined; +function stringToBytes(input: null, percentageReference?: number): null; +function stringToBytes( + input: string | number, + percentageReference?: number, +): number; + +/** + * Converts a string representing an amount of memory to bytes. + * + * @param input The value to convert to bytes. + * @param percentageReference The reference value to use when a '%' value is supplied. + */ +function stringToBytes( + input: string | number | null | undefined, + percentageReference?: number, +): number | null | undefined { + if (input === null || input === undefined) { + return input; + } + + if (typeof input === 'string') { + if (isNaN(Number.parseFloat(input.slice(-1)))) { + // eslint-disable-next-line prefer-const + let [, numericString, trailingChars] = + input.match(/(.*?)([^0-9.-]+)$/i) || []; + + if (trailingChars && numericString) { + const numericValue = Number.parseFloat(numericString); + trailingChars = trailingChars.toLowerCase(); + + switch (trailingChars) { + case '%': + input = numericValue / 100; + break; + case 'kb': + case 'k': + return numericValue * 1000; + case 'kib': + return numericValue * 1024; + case 'mb': + case 'm': + return numericValue * 1000 * 1000; + case 'mib': + return numericValue * 1024 * 1024; + case 'gb': + case 'g': + return numericValue * 1000 * 1000 * 1000; + case 'gib': + return numericValue * 1024 * 1024 * 1024; + } + } + + // It ends in some kind of char so we need to do some parsing + } else { + input = Number.parseFloat(input); + } + } + + if (typeof input === 'number') { + if (input <= 1 && input > 0) { + if (percentageReference) { + return Math.floor(input * percentageReference); + } else { + throw new Error( + 'For a percentage based memory limit a percentageReference must be supplied', + ); + } + } else if (input > 1) { + return Math.floor(input); + } else { + throw new Error('Unexpected numerical input'); + } + } + + throw new Error('Unexpected input'); +} + +// https://github.com/import-js/eslint-plugin-import/issues/1590 +export default stringToBytes; diff --git a/packages/jest-worker/README.md b/packages/jest-worker/README.md index 19f07ff91073..1277ed824bf2 100644 --- a/packages/jest-worker/README.md +++ b/packages/jest-worker/README.md @@ -81,8 +81,8 @@ Specifies the memory limit for workers before they are recycled and is primarily After the worker has executed a task the memory usage of it is checked. If it exceeds the value specified the worker is killed and restarted. If no limit is set this process does not occur. The limit can be specified in 2 ways: -- `< 1` - The value is assumed to be a percentage of system memory. So 0.5 sets the memory limit of the worker to half of the total system memory -- `\> 1` - Assumed to be a fixed byte value. So 5000 would set an idle limit of 5000 bytes. +- `<= 1` - The value is assumed to be a percentage of system memory. So 0.5 sets the memory limit of the worker to half of the total system memory +- `\> 1` - Assumed to be a fixed byte value. Because of the previous rule if you wanted a value of 1 byte (I don't know why) you could use `1.1`. #### `maxRetries: number` (optional) diff --git a/packages/jest-worker/src/workers/ChildProcessWorker.ts b/packages/jest-worker/src/workers/ChildProcessWorker.ts index 645b714cf54f..4ef4d51b87cc 100644 --- a/packages/jest-worker/src/workers/ChildProcessWorker.ts +++ b/packages/jest-worker/src/workers/ChildProcessWorker.ts @@ -296,8 +296,14 @@ export default class ChildProcessWorker implements WorkerInterface { let limit = this._childIdleMemoryUsageLimit; + // TODO: At some point it would make sense to make use of + // stringToBytes found in jest-config, however as this + // package does not have any dependencies on an other jest + // packages that can wait until some other time. if (limit && limit > 0 && limit <= 1) { limit = Math.floor(totalmem() * limit); + } else if (limit) { + limit = Math.floor(limit); } if ( diff --git a/packages/jest-worker/src/workers/NodeThreadsWorker.ts b/packages/jest-worker/src/workers/NodeThreadsWorker.ts index c013eb7cb051..f45d1e1044b2 100644 --- a/packages/jest-worker/src/workers/NodeThreadsWorker.ts +++ b/packages/jest-worker/src/workers/NodeThreadsWorker.ts @@ -319,8 +319,14 @@ export default class ExperimentalWorker implements WorkerInterface { let limit = this._childIdleMemoryUsageLimit; + // TODO: At some point it would make sense to make use of + // stringToBytes found in jest-config, however as this + // package does not have any dependencies on an other jest + // packages that can wait until some other time. if (limit && limit > 0 && limit <= 1) { limit = Math.floor(totalmem() * limit); + } else if (limit) { + limit = Math.floor(limit); } if ( diff --git a/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js index 122267089d19..3285126e0ec6 100644 --- a/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js @@ -17,8 +17,9 @@ import { import ChildProcessWorker from '../ChildProcessWorker'; import ThreadsWorker from '../NodeThreadsWorker'; -// These tests appear to be slow/flakey on Windows -jest.retryTimes(10); +// These tests appear to be slow/flaky. Allowing it to retry quite a few times +// will cut down on this noise and they're fast tests anyway. +jest.retryTimes(30); const root = join('../../'); const filesToBuild = ['workers/processChild', 'workers/threadChild', 'types']; From 9f9dff1ae2e4eaca2f33d9e1d0e3ed3001415764 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 1 Aug 2022 17:40:08 +0100 Subject: [PATCH 46/49] docs: copyright header --- packages/jest-config/src/__tests__/stringToBytes.test.ts | 7 +++++++ packages/jest-config/src/stringToBytes.ts | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/packages/jest-config/src/__tests__/stringToBytes.test.ts b/packages/jest-config/src/__tests__/stringToBytes.test.ts index ea86876fa03f..7c5902ec1acb 100644 --- a/packages/jest-config/src/__tests__/stringToBytes.test.ts +++ b/packages/jest-config/src/__tests__/stringToBytes.test.ts @@ -1,3 +1,10 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + import stringToBytes from '../stringToBytes'; describe('numeric input', () => { diff --git a/packages/jest-config/src/stringToBytes.ts b/packages/jest-config/src/stringToBytes.ts index 65176e0ce250..f9fa6d72b37f 100644 --- a/packages/jest-config/src/stringToBytes.ts +++ b/packages/jest-config/src/stringToBytes.ts @@ -1,3 +1,10 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + function stringToBytes( input: undefined, percentageReference?: number, From 43f105459283cf193c4cef9692324d148b2d5097 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Mon, 1 Aug 2022 18:53:45 +0100 Subject: [PATCH 47/49] chore: improve test flake --- .../src/workers/__tests__/WorkerEdgeCases.test.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js b/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js index 3285126e0ec6..8005048f48f4 100644 --- a/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js +++ b/packages/jest-worker/src/workers/__tests__/WorkerEdgeCases.test.js @@ -82,7 +82,7 @@ describe.each([ clearInterval(int); }); - function waitForChange(fn, limit = 100) { + function waitForChange(fn) { const inital = fn(); return new Promise((resolve, reject) => { @@ -96,12 +96,12 @@ describe.each([ clearInterval(int); } - if (count > limit) { + if (count > 100000) { reject(new Error('Timeout waiting for change')); } count++; - }, 50); + }, 1); }); } @@ -169,7 +169,7 @@ describe.each([ const endPid = worker.getWorkerSystemId(); expect(endPid).toBeGreaterThanOrEqual(0); expect(endPid).not.toEqual(startPid); - }); + }, 10000); test('should cleanly exit on crash', async () => { const workerHeapLimit = 10; From 791b7a5197061f70de71a6f9dc2593f43b6e0b40 Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Tue, 2 Aug 2022 09:21:37 +0100 Subject: [PATCH 48/49] chore: fix typing --- packages/jest-runner/src/index.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/jest-runner/src/index.ts b/packages/jest-runner/src/index.ts index 3debd22e1b2e..c646c432232c 100644 --- a/packages/jest-runner/src/index.ts +++ b/packages/jest-runner/src/index.ts @@ -107,7 +107,12 @@ export default class TestRunner extends EmittingTestRunner { const worker = new Worker(require.resolve('./testWorker'), { exposedMethods: ['worker'], forkOptions: {serialization: 'json', stdio: 'pipe'}, - idleMemoryLimit: this._globalConfig.workerIdleMemoryLimit, + // The workerIdleMemoryLimit should've been converted to a number during + // the normalization phase. + idleMemoryLimit: + typeof this._globalConfig.workerIdleMemoryLimit === 'number' + ? this._globalConfig.workerIdleMemoryLimit + : undefined, maxRetries: 3, numWorkers: this._globalConfig.maxWorkers, setupArgs: [{serializableResolvers: Array.from(resolvers.values())}], From f3db1cce1274aeadef1106a9a39fec283bf895bd Mon Sep 17 00:00:00 2001 From: Paul Hawxby Date: Thu, 4 Aug 2022 22:25:14 +0100 Subject: [PATCH 49/49] chore: pr feedback --- .github/workflows/test.yml | 2 -- docs/Configuration.md | 2 +- e2e/__tests__/workerForceExit.test.ts | 3 --- packages/jest-worker/src/workers/NodeThreadsWorker.ts | 1 + 4 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3952d84931f6..d7060b18db3a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,7 +32,6 @@ jobs: id: cpu-cores uses: SimenB/github-actions-cpu-cores@v1 - name: run tests - timeout-minutes: 20 run: yarn test-ci-partial:parallel --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} test-jasmine: @@ -58,5 +57,4 @@ jobs: id: cpu-cores uses: SimenB/github-actions-cpu-cores@v1 - name: run tests using jest-jasmine - timeout-minutes: 20 run: yarn jest-jasmine-ci --max-workers ${{ steps.cpu-cores.outputs.count }} --shard=${{ matrix.shard }} diff --git a/docs/Configuration.md b/docs/Configuration.md index 2049cf2548f3..b6d56b9ec376 100644 --- a/docs/Configuration.md +++ b/docs/Configuration.md @@ -2254,7 +2254,7 @@ Default: `true` Whether to use [`watchman`](https://facebook.github.io/watchman/) for file crawling. -### `workerIdleMemoryLimit` \[number] +### `workerIdleMemoryLimit` \[number|string] Default: `undefined` diff --git a/e2e/__tests__/workerForceExit.test.ts b/e2e/__tests__/workerForceExit.test.ts index 3ade72d2b3db..045c1ce702fc 100644 --- a/e2e/__tests__/workerForceExit.test.ts +++ b/e2e/__tests__/workerForceExit.test.ts @@ -15,9 +15,6 @@ import { } from '../Utils'; import runJest from '../runJest'; -// These tests appear to be slow/flakey on Windows -jest.retryTimes(10); - const DIR = resolve(tmpdir(), 'worker-force-exit'); beforeEach(() => cleanup(DIR)); diff --git a/packages/jest-worker/src/workers/NodeThreadsWorker.ts b/packages/jest-worker/src/workers/NodeThreadsWorker.ts index f45d1e1044b2..ad1bb822cd56 100644 --- a/packages/jest-worker/src/workers/NodeThreadsWorker.ts +++ b/packages/jest-worker/src/workers/NodeThreadsWorker.ts @@ -123,6 +123,7 @@ export default class ExperimentalWorker implements WorkerInterface { // This can be useful for debugging. if (!(this._options.silent ?? true)) { this._worker.stdout.setEncoding('utf8'); + // eslint-disable-next-line no-console this._worker.stdout.on('data', console.log); this._worker.stderr.setEncoding('utf8'); this._worker.stderr.on('data', console.error);