From 412b0b7e0c203edd1aae67491b1330ec2593d4fc Mon Sep 17 00:00:00 2001 From: Jonny Harris Date: Thu, 27 May 2021 14:07:33 +0100 Subject: [PATCH 1/3] Retry e2e tests --- packages/e2e-tests/src/config/bootstrap.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/e2e-tests/src/config/bootstrap.js b/packages/e2e-tests/src/config/bootstrap.js index 2ad85aea85fe..680c0068ca7a 100644 --- a/packages/e2e-tests/src/config/bootstrap.js +++ b/packages/e2e-tests/src/config/bootstrap.js @@ -112,6 +112,11 @@ const pageEvents = []; // The Jest timeout is increased because these tests are a bit slow jest.setTimeout(PUPPETEER_TIMEOUT || 100000); +// Retry flaky tests at most 2 times in CI. +if ('true' === process.env.CI) { + jest.retryTimes(2); +} + // Set default timeout for individual expect-puppeteer assertions. (Default: 500) setDefaultOptions({ timeout: EXPECT_PUPPETEER_TIMEOUT || 500 }); From 47d5d4f1313a2cec9ac2f5f6fb18c9ecaf32786a Mon Sep 17 00:00:00 2001 From: Jonny Harris Date: Thu, 27 May 2021 17:09:22 +0100 Subject: [PATCH 2/3] Retry in test. --- packages/e2e-tests/src/config/bootstrap.js | 2 +- .../e2e-tests/src/specs/wordpress/retries.js | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 packages/e2e-tests/src/specs/wordpress/retries.js diff --git a/packages/e2e-tests/src/config/bootstrap.js b/packages/e2e-tests/src/config/bootstrap.js index 680c0068ca7a..5586123e4eb1 100644 --- a/packages/e2e-tests/src/config/bootstrap.js +++ b/packages/e2e-tests/src/config/bootstrap.js @@ -114,7 +114,7 @@ jest.setTimeout(PUPPETEER_TIMEOUT || 100000); // Retry flaky tests at most 2 times in CI. if ('true' === process.env.CI) { - jest.retryTimes(2); + jest.retryTimes(3); } // Set default timeout for individual expect-puppeteer assertions. (Default: 500) diff --git a/packages/e2e-tests/src/specs/wordpress/retries.js b/packages/e2e-tests/src/specs/wordpress/retries.js new file mode 100644 index 000000000000..a5dd0d11cd35 --- /dev/null +++ b/packages/e2e-tests/src/specs/wordpress/retries.js @@ -0,0 +1,33 @@ +/* + * Copyright 2021 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +/** + * External dependencies + */ +import { join } from 'path'; +import { tmpdir } from 'os'; +import { readFileSync, writeFileSync } from 'fs'; + +describe('Retries', () => { + const countPath = join(tmpdir(), 'test.txt'); + beforeAll(() => { + writeFileSync(countPath, '0', 'utf8'); + }); + it('retries', () => { + const tries = Number(readFileSync(countPath, 'utf8')); + writeFileSync(countPath, String(tries + 1), 'utf8'); + expect(tries).toBeGreaterThanOrEqual(3); + }); +}); From d3e9bf1e4ec77a195847a80466e1164b490190a2 Mon Sep 17 00:00:00 2001 From: Jonny Harris Date: Thu, 27 May 2021 17:22:53 +0100 Subject: [PATCH 3/3] Remove test. --- packages/e2e-tests/src/config/bootstrap.js | 2 +- .../e2e-tests/src/specs/wordpress/retries.js | 33 ------------------- 2 files changed, 1 insertion(+), 34 deletions(-) delete mode 100644 packages/e2e-tests/src/specs/wordpress/retries.js diff --git a/packages/e2e-tests/src/config/bootstrap.js b/packages/e2e-tests/src/config/bootstrap.js index 5586123e4eb1..cad69debb61f 100644 --- a/packages/e2e-tests/src/config/bootstrap.js +++ b/packages/e2e-tests/src/config/bootstrap.js @@ -112,7 +112,7 @@ const pageEvents = []; // The Jest timeout is increased because these tests are a bit slow jest.setTimeout(PUPPETEER_TIMEOUT || 100000); -// Retry flaky tests at most 2 times in CI. +// Retry flaky tests at most 2 times in CI (off by 1). if ('true' === process.env.CI) { jest.retryTimes(3); } diff --git a/packages/e2e-tests/src/specs/wordpress/retries.js b/packages/e2e-tests/src/specs/wordpress/retries.js deleted file mode 100644 index a5dd0d11cd35..000000000000 --- a/packages/e2e-tests/src/specs/wordpress/retries.js +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Copyright 2021 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -/** - * External dependencies - */ -import { join } from 'path'; -import { tmpdir } from 'os'; -import { readFileSync, writeFileSync } from 'fs'; - -describe('Retries', () => { - const countPath = join(tmpdir(), 'test.txt'); - beforeAll(() => { - writeFileSync(countPath, '0', 'utf8'); - }); - it('retries', () => { - const tries = Number(readFileSync(countPath, 'utf8')); - writeFileSync(countPath, String(tries + 1), 'utf8'); - expect(tries).toBeGreaterThanOrEqual(3); - }); -});