From 721d68a8908c0ed5519a4f4e9a5769050ca340ec Mon Sep 17 00:00:00 2001 From: Gerard Soldevila Date: Wed, 22 Nov 2023 08:53:35 +0100 Subject: [PATCH] Avoid ECONNRESET errors on idle timeout (#162947) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### Summary Address https://github.com/elastic/kibana/issues/82002 and https://github.com/elastic/kibana/issues/75440 I think I found a breakthrough for this flaky behavior. I run the integration test 800x locally, with different settings: Adjusting both the delayed emission (send 1 char at a time), and the socket idle timeout to have exacly the same value (e.g. `10 millis`), I managed to get the `ECONNRESET` 100% of the times. Thus, IIUC the ECONNRESET happens when the client tries to send a character over the socket and at the same time the server responds with the idle timeout. Adjusting the values so that the delay between character emissions is significantly larger than the idle timeout, e.g. 20 vs 5, I get `socket hang up` 100% of the times. Flaky Test Runner Pipeline - 300x 🟢 https://buildkite.com/elastic/kibana-flaky-test-suite-runner/builds/4030 --- .../integration_tests/http/router.test.ts | 4 +- .../server/plugin.ts | 2 +- .../test_suites/core/route.ts | 56 ++++++------------- 3 files changed, 20 insertions(+), 42 deletions(-) diff --git a/src/core/server/integration_tests/http/router.test.ts b/src/core/server/integration_tests/http/router.test.ts index 1e5b284134515..f66cab0a07f5e 100644 --- a/src/core/server/integration_tests/http/router.test.ts +++ b/src/core/server/integration_tests/http/router.test.ts @@ -408,7 +408,7 @@ describe('Options', () => { }); describe('idleSocket', () => { - it.skip('should timeout if payload sending has too long of an idle period', async () => { + it('should timeout if payload sending has too long of an idle period', async () => { const { server: innerServer, createRouter } = await server.setup(setupDeps); const router = createRouter('/'); @@ -420,7 +420,7 @@ describe('Options', () => { body: { accepts: ['application/json'], }, - timeout: { idleSocket: 10 }, + timeout: { idleSocket: 5 }, }, }, async (context, req, res) => { diff --git a/test/plugin_functional/plugins/core_plugin_route_timeouts/server/plugin.ts b/test/plugin_functional/plugins/core_plugin_route_timeouts/server/plugin.ts index fce258c1b8b00..5ecdc0a03dd08 100644 --- a/test/plugin_functional/plugins/core_plugin_route_timeouts/server/plugin.ts +++ b/test/plugin_functional/plugins/core_plugin_route_timeouts/server/plugin.ts @@ -53,7 +53,7 @@ export class CorePluginRouteTimeoutsPlugin implements Plugin { body: { accepts: ['application/json'], }, - timeout: { idleSocket: 10 }, + timeout: { idleSocket: 5 }, }, path: '/short_idle_socket_timeout', validate: { diff --git a/test/plugin_functional/test_suites/core/route.ts b/test/plugin_functional/test_suites/core/route.ts index 13bd4190d32f9..597189dd5faf3 100644 --- a/test/plugin_functional/test_suites/core/route.ts +++ b/test/plugin_functional/test_suites/core/route.ts @@ -7,14 +7,13 @@ */ import expect from '@kbn/expect'; -import { Test } from 'supertest'; -import { PluginFunctionalProviderContext } from '../../services'; +import type { Test } from 'supertest'; +import type { PluginFunctionalProviderContext } from '../../services'; export default function ({ getService }: PluginFunctionalProviderContext) { const supertest = getService('supertest'); - // FLAKY: https://github.com/elastic/kibana/issues/75440 - describe.skip('route', function () { + describe('route', function () { describe('timeouts', function () { const writeBodyCharAtATime = (request: Test, body: string, interval: number) => { return new Promise((resolve, reject) => { @@ -45,7 +44,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) { .set('Transfer-Encoding', 'chunked') .set('kbn-xsrf', 'true'); - const result = writeBodyCharAtATime(request, '{"foo":"bar"}', 10); + const result = writeBodyCharAtATime(request, '{"foo":"bar"}', 20); await result.then( (res) => { @@ -65,7 +64,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) { .set('Transfer-Encoding', 'chunked') .set('kbn-xsrf', 'true'); - const result = writeBodyCharAtATime(request, '{"foo":"bar"}', 10); + const result = writeBodyCharAtATime(request, '{"foo":"bar"}', 20); await result.then( (res) => { @@ -107,7 +106,7 @@ export default function ({ getService }: PluginFunctionalProviderContext) { .set('Transfer-Encoding', 'chunked') .set('kbn-xsrf', 'true'); - const result = writeBodyCharAtATime(request, '{"responseDelay":0}', 10); + const result = writeBodyCharAtATime(request, '{"responseDelay":0}', 20); await result.then( (res) => { @@ -119,44 +118,23 @@ export default function ({ getService }: PluginFunctionalProviderContext) { ); }); - it('should timeout if servers response is too slow', async function () { - // start the request - const request = supertest + it('should timeout if servers response is too slow', async () => { + await supertest .post('/short_idle_socket_timeout') + .send({ responseDelay: 100 }) .set('Content-Type', 'application/json') - .set('Transfer-Encoding', 'chunked') - .set('kbn-xsrf', 'true'); - - const result = writeBodyCharAtATime(request, '{"responseDelay":100}', 0); - - await result.then( - (res) => { - expect(res).to.be(undefined); - }, - (err) => { - expect(err.message).to.be('socket hang up'); - } - ); + .set('kbn-xsrf', 'true') + .then(() => expect('to throw').to.be('but it did NOT')) + .catch((error) => expect(error.message).to.be('socket hang up')); }); - it('should not timeout if servers response is fast enough', async function () { - // start the request - const request = supertest + it('should not timeout if servers response is fast enough', async () => { + await supertest .post('/longer_idle_socket_timeout') + .send({ responseDelay: 100 }) .set('Content-Type', 'application/json') - .set('Transfer-Encoding', 'chunked') - .set('kbn-xsrf', 'true'); - - const result = writeBodyCharAtATime(request, '{"responseDelay":100}', 0); - - await result.then( - (res) => { - expect(res).to.have.property('statusCode', 200); - }, - (err) => { - expect(err).to.be(undefined); - } - ); + .set('kbn-xsrf', 'true') + .expect(200); }); }); });