[Console] Proxy fallback (#50185) (#51814)

* First iteration of liveness manager for Console

* First iteration of PoC working

* Updated console proxy fallback behaviour after feedback

* remove @types/node-fetch

* If all hosts failed due to connection refused errors 502

* Remove unnecessary existence check

src/legacy/core_plugins/console/index.ts

@@ -141,7 +141,7 @@ export default function(kibana: any) {
 
       server.route(
         createProxyRoute({
-          baseUrl: head(legacyEsConfig.hosts),
+          hosts: legacyEsConfig.hosts,
           pathFilters: proxyPathFilters,
           getConfigForReq(req: any, uri: any) {
             const filteredHeaders = filterHeaders(

src/legacy/core_plugins/console/server/__tests__/proxy_route/body.js

@@ -36,7 +36,7 @@ describe('Console Proxy Route', () => {
       const server = new Server();
       server.route(
         createProxyRoute({
-          baseUrl: 'http://localhost:9200',
+          hosts: ['http://localhost:9200'],
         })
       );
 

src/legacy/core_plugins/console/server/__tests__/proxy_route/headers.js

@@ -40,7 +40,7 @@ describe('Console Proxy Route', () => {
       const server = new Server();
       server.route(
         createProxyRoute({
-          baseUrl: 'http://localhost:9200',
+          hosts: ['http://localhost:9200'],
         })
       );
 

src/legacy/core_plugins/console/server/__tests__/proxy_route/params.js

@@ -72,7 +72,7 @@ describe('Console Proxy Route', () => {
           const { server } = setup();
           server.route(
             createProxyRoute({
-              baseUrl: 'http://localhost:9200',
+              hosts: ['http://localhost:9200'],
               pathFilters: [/^\/foo\//, /^\/bar\//],
             })
           );
@@ -91,7 +91,7 @@ describe('Console Proxy Route', () => {
           const { server } = setup();
           server.route(
             createProxyRoute({
-              baseUrl: 'http://localhost:9200',
+              hosts: ['http://localhost:9200'],
               pathFilters: [/^\/foo\//, /^\/bar\//],
             })
           );
@@ -113,7 +113,7 @@ describe('Console Proxy Route', () => {
 
         const getConfigForReq = sinon.stub().returns({});
 
-        server.route(createProxyRoute({ baseUrl: 'http://localhost:9200', getConfigForReq }));
+        server.route(createProxyRoute({ hosts: ['http://localhost:9200'], getConfigForReq }));
         await server.inject({
           method: 'POST',
           url: '/api/console/proxy?method=HEAD&path=/index/type/id',
@@ -142,7 +142,7 @@ describe('Console Proxy Route', () => {
 
         server.route(
           createProxyRoute({
-            baseUrl: 'http://localhost:9200',
+            hosts: ['http://localhost:9200'],
             getConfigForReq: () => ({
               timeout,
               agent,
@@ -166,19 +166,5 @@ describe('Console Proxy Route', () => {
         expect(opts.headers).to.have.property('baz', 'bop');
       });
     });
-
-    describe('baseUrl', () => {
-      describe('default', () => {
-        it('ensures that the path starts with a /');
-      });
-      describe('url ends with a slash', () => {
-        it('combines clean with paths that start with a slash');
-        it(`combines clean with paths that don't start with a slash`);
-      });
-      describe(`url doesn't end with a slash`, () => {
-        it('combines clean with paths that start with a slash');
-        it(`combines clean with paths that don't start with a slash`);
-      });
-    });
   });
 });

src/legacy/core_plugins/console/server/__tests__/proxy_route/query_string.js

@@ -38,7 +38,7 @@ describe('Console Proxy Route', () => {
       const server = new Server();
       server.route(
         createProxyRoute({
-          baseUrl: 'http://localhost:9200',
+          hosts: ['http://localhost:9200'],
         })
       );
 

src/legacy/core_plugins/console/server/proxy_route.js → src/legacy/core_plugins/console/server/proxy_route.ts

@@ -18,12 +18,13 @@
  */
 
 import Joi from 'joi';
+import * as url from 'url';
+import { IncomingMessage } from 'http';
 import Boom from 'boom';
 import { trimLeft, trimRight } from 'lodash';
 import { sendRequest } from './request';
-import * as url from 'url';
 
-function toURL(base, path) {
+function toURL(base: string, path: string) {
   const urlResult = new url.URL(`${trimRight(base, '/')}/${trimLeft(path, '/')}`);
   // Appending pretty here to have Elasticsearch do the JSON formatting, as doing
   // in JS can lead to data loss (7.0 will get munged into 7, thus losing indication of
@@ -34,11 +35,11 @@ function toURL(base, path) {
   return urlResult;
 }
 
-function getProxyHeaders(req) {
+function getProxyHeaders(req: any) {
   const headers = Object.create(null);
 
   // Scope this proto-unsafe functionality to where it is being used.
-  function extendCommaList(obj, property, value) {
+  function extendCommaList(obj: Record<string, any>, property: string, value: any) {
     obj[property] = (obj[property] ? obj[property] + ',' : '') + value;
   }
 
@@ -58,9 +59,13 @@ function getProxyHeaders(req) {
 }
 
 export const createProxyRoute = ({
-  baseUrl = '/',
+  hosts,
   pathFilters = [/.*/],
   getConfigForReq = () => ({}),
+}: {
+  hosts: string[];
+  pathFilters: RegExp[];
+  getConfigForReq: (...args: any[]) => any;
 }) => ({
   path: '/api/console/proxy',
   method: 'POST',
@@ -84,63 +89,82 @@ export const createProxyRoute = ({
     },
 
     pre: [
-      function filterPath(req) {
+      function filterPath(req: any) {
         const { path } = req.query;
 
         if (pathFilters.some(re => re.test(path))) {
           return null;
         }
 
         const err = Boom.forbidden();
-        err.output.payload = `Error connecting to '${path}':\n\nUnable to send requests to that path.`;
+        err.output.payload = `Error connecting to '${path}':\n\nUnable to send requests to that path.` as any;
         err.output.headers['content-type'] = 'text/plain';
         throw err;
       },
     ],
 
-    handler: async (req, h) => {
+    handler: async (req: any, h: any) => {
       const { payload, query } = req;
       const { path, method } = query;
-      const uri = toURL(baseUrl, path);
-
-      // Because this can technically be provided by a settings-defined proxy config, we need to
-      // preserve these property names to maintain BWC.
-      const { timeout, agent, headers, rejectUnauthorized } = getConfigForReq(req, uri.toString());
-
-      const requestHeaders = {
-        ...headers,
-        ...getProxyHeaders(req),
-      };
-
-      const esIncomingMessage = await sendRequest({
-        method,
-        headers: requestHeaders,
-        uri,
-        timeout,
-        payload,
-        rejectUnauthorized,
-        agent,
-      });
+
+      let esIncomingMessage: IncomingMessage;
+
+      for (let idx = 0; idx < hosts.length; ++idx) {
+        const host = hosts[idx];
+        try {
+          const uri = toURL(host, path);
+
+          // Because this can technically be provided by a settings-defined proxy config, we need to
+          // preserve these property names to maintain BWC.
+          const { timeout, agent, headers, rejectUnauthorized } = getConfigForReq(
+            req,
+            uri.toString()
+          );
+
+          const requestHeaders = {
+            ...headers,
+            ...getProxyHeaders(req),
+          };
+
+          esIncomingMessage = await sendRequest({
+            method,
+            headers: requestHeaders,
+            uri,
+            timeout,
+            payload,
+            rejectUnauthorized,
+            agent,
+          });
+
+          break;
+        } catch (e) {
+          if (e.code !== 'ECONNREFUSED') {
+            throw Boom.boomify(e);
+          }
+          if (idx === hosts.length - 1) {
+            throw Boom.badGateway('Could not reach any configured nodes.');
+          }
+          // Otherwise, try the next host...
+        }
+      }
 
       const {
         statusCode,
         statusMessage,
-        headers: responseHeaders,
-      } = esIncomingMessage;
-
-      const { warning } = responseHeaders;
+        headers: { warning },
+      } = esIncomingMessage!;
 
       if (method.toUpperCase() !== 'HEAD') {
         return h
-          .response(esIncomingMessage)
+          .response(esIncomingMessage!)
           .code(statusCode)
-          .header('warning', warning);
+          .header('warning', warning!);
       } else {
         return h
           .response(`${statusCode} - ${statusMessage}`)
           .code(statusCode)
           .type('text/plain')
-          .header('warning', warning);
+          .header('warning', warning!);
       }
     },
   },

src/legacy/core_plugins/console/server/request.test.ts

@@ -24,7 +24,7 @@ import { fail } from 'assert';
 
 describe(`Console's send request`, () => {
   let sandbox: sinon.SinonSandbox;
-  let stub: sinon.SinonStub<Parameters<typeof http['request']>, ClientRequest>;
+  let stub: sinon.SinonStub<Parameters<typeof http.request>, ClientRequest>;
   let fakeRequest: http.ClientRequest;
 
   beforeEach(() => {
@@ -52,7 +52,7 @@ describe(`Console's send request`, () => {
         method: 'get',
         payload: null as any,
         timeout: 0, // immediately timeout
-        uri: new URL('http://noone.nowhere.com'),
+        uri: new URL('http://noone.nowhere.none'),
       });
       fail('Should not reach here!');
     } catch (e) {

src/legacy/core_plugins/console/server/request.ts

@@ -89,7 +89,7 @@ export const sendRequest = ({
     }
   });
 
-  const onError = () => reject();
+  const onError = (e: Error) => reject(e);
   req.once('error', onError);
 
   const timeoutPromise = new Promise<any>((timeoutResolve, timeoutReject) => {
@@ -103,5 +103,5 @@ export const sendRequest = ({
     }, timeout);
   });
 
-  return Promise.race<http.ServerResponse>([reqPromise, timeoutPromise]);
+  return Promise.race<http.IncomingMessage>([reqPromise, timeoutPromise]);
 };