From de65b3624106f4a8963efb1de60ef977edba1624 Mon Sep 17 00:00:00 2001 From: Thomas Vendetta Date: Wed, 29 Mar 2017 18:37:25 -0400 Subject: [PATCH 01/12] Fixes #5211 --- package.json | 3 +- src/cli_plugin/install/downloaders/http.js | 32 +++++++++++++++++++++- 2 files changed, 33 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index a1892dc7b4a9f..e09c0083d0b68 100644 --- a/package.json +++ b/package.json @@ -129,8 +129,8 @@ "expose-loader": "0.7.0", "extract-text-webpack-plugin": "0.8.2", "file-loader": "0.8.4", - "font-awesome": "4.4.0", "flot-charts": "0.8.3", + "font-awesome": "4.4.0", "glob": "5.0.13", "glob-all": "3.0.1", "good-squeeze": "2.1.0", @@ -138,6 +138,7 @@ "h2o2": "5.1.1", "handlebars": "4.0.5", "hapi": "14.2.0", + "http-proxy-agent": "1.0.0", "imports-loader": "0.6.4", "inert": "4.0.2", "jade": "1.11.0", diff --git a/src/cli_plugin/install/downloaders/http.js b/src/cli_plugin/install/downloaders/http.js index f8c28d65b7733..0a44f477331f4 100644 --- a/src/cli_plugin/install/downloaders/http.js +++ b/src/cli_plugin/install/downloaders/http.js @@ -2,11 +2,41 @@ import Wreck from 'wreck'; import Progress from '../progress'; import { fromNode as fn } from 'bluebird'; import { createWriteStream } from 'fs'; +import HttpProxyAgent from 'http-proxy-agent'; +import URL from 'url'; + +function getProxyAgent(sourceUrl) { + const httpProxy = process.env.HTTP_PROXY; + // we have a proxy detected, lets use it + if (httpProxy && httpProxy !== '') { + // get the hostname of the sourceUrl + const hostname = URL.parse(sourceUrl).hostname; + const noProxy = process.env.NO_PROXY || ''; + + // proxy if the hostname is not in noProxy + const shouldProxy = (noProxy.indexOf(hostname) === -1); + + if (shouldProxy) { + return new HttpProxyAgent(httpProxy); + } else { + return false; + } + } else { + return false; + } +} function sendRequest({ sourceUrl, timeout }) { const maxRedirects = 11; //Because this one goes to 11. return fn(cb => { - const req = Wreck.request('GET', sourceUrl, { timeout, redirects: maxRedirects }, (err, resp) => { + const reqOptions = { timeout, redirects: maxRedirects }; + const proxyAgent = getProxyAgent(sourceUrl); + + if (proxyAgent) { + reqOptions.agent = proxyAgent; + } + + const req = Wreck.request('GET', sourceUrl, reqOptions, (err, resp) => { if (err) { if (err.code === 'ECONNREFUSED') { err = new Error('ENOTFOUND'); From a4990338ac20e9531eddd8a5f394d8be44a4e0d6 Mon Sep 17 00:00:00 2001 From: Thomas Vendetta Date: Tue, 27 Jun 2017 11:37:50 -0400 Subject: [PATCH 02/12] Cleanup conditional logic --- src/cli_plugin/install/downloaders/http.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/cli_plugin/install/downloaders/http.js b/src/cli_plugin/install/downloaders/http.js index 0a44f477331f4..f179a2ebd807c 100644 --- a/src/cli_plugin/install/downloaders/http.js +++ b/src/cli_plugin/install/downloaders/http.js @@ -18,12 +18,10 @@ function getProxyAgent(sourceUrl) { if (shouldProxy) { return new HttpProxyAgent(httpProxy); - } else { - return false; } - } else { - return false; } + + return false; } function sendRequest({ sourceUrl, timeout }) { From 4434ca088a009fd14439f637446dbedc7f110c62 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Tue, 11 Jul 2017 10:23:03 +0200 Subject: [PATCH 03/12] Improve proxy handling in http downloader This will now also accept http_proxy/no_proxy (lower case) env variables. It also logs information about the used (or skipped) proxy to the console. --- src/cli_plugin/install/downloaders/http.js | 26 ++++++++++++++-------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/src/cli_plugin/install/downloaders/http.js b/src/cli_plugin/install/downloaders/http.js index f179a2ebd807c..d5feb347b3947 100644 --- a/src/cli_plugin/install/downloaders/http.js +++ b/src/cli_plugin/install/downloaders/http.js @@ -5,30 +5,38 @@ import { createWriteStream } from 'fs'; import HttpProxyAgent from 'http-proxy-agent'; import URL from 'url'; -function getProxyAgent(sourceUrl) { - const httpProxy = process.env.HTTP_PROXY; +function getProxyAgent(sourceUrl, logger) { + const httpProxy = process.env.http_proxy || process.env.HTTP_PROXY; // we have a proxy detected, lets use it - if (httpProxy && httpProxy !== '') { + if (httpProxy) { + logger.log(`Picked up proxy ${httpProxy} from http_proxy environment variable.`); // get the hostname of the sourceUrl const hostname = URL.parse(sourceUrl).hostname; - const noProxy = process.env.NO_PROXY || ''; + const noProxy = process.env.no_proxy || process.env.NO_PROXY || ''; + const excludedHosts = noProxy.split(','); // proxy if the hostname is not in noProxy - const shouldProxy = (noProxy.indexOf(hostname) === -1); + const shouldProxy = (excludedHosts.indexOf(hostname) === -1); if (shouldProxy) { + logger.log(`Use proxy to download plugin.`); + logger.log(`Hint: you can add ${hostname} to the no_proxy environment variable, ` + + `to exclude that host from proxying.`); return new HttpProxyAgent(httpProxy); + } else { + logger.log(`Found exception for host ${hostname} in no_proxy environment variable. ` + + `Skipping proxy.`); } } - return false; + return null; } -function sendRequest({ sourceUrl, timeout }) { +function sendRequest({ sourceUrl, timeout }, logger) { const maxRedirects = 11; //Because this one goes to 11. return fn(cb => { const reqOptions = { timeout, redirects: maxRedirects }; - const proxyAgent = getProxyAgent(sourceUrl); + const proxyAgent = getProxyAgent(sourceUrl, logger); if (proxyAgent) { reqOptions.agent = proxyAgent; @@ -78,7 +86,7 @@ Responsible for managing http transfers */ export default async function downloadUrl(logger, sourceUrl, targetPath, timeout) { try { - const { req, resp } = await sendRequest({ sourceUrl, timeout }); + const { req, resp } = await sendRequest({ sourceUrl, timeout }, logger); try { const totalSize = parseFloat(resp.headers['content-length']) || 0; From 2aecac8df3679ba654ff6c382074c4f1ae4803a4 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Tue, 11 Jul 2017 14:53:36 +0200 Subject: [PATCH 04/12] Use includes instead of indexOf --- src/cli_plugin/install/downloaders/http.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli_plugin/install/downloaders/http.js b/src/cli_plugin/install/downloaders/http.js index d5feb347b3947..97ebf73a349e9 100644 --- a/src/cli_plugin/install/downloaders/http.js +++ b/src/cli_plugin/install/downloaders/http.js @@ -16,7 +16,7 @@ function getProxyAgent(sourceUrl, logger) { const excludedHosts = noProxy.split(','); // proxy if the hostname is not in noProxy - const shouldProxy = (excludedHosts.indexOf(hostname) === -1); + const shouldProxy = !excludedHosts.includes(hostname); if (shouldProxy) { logger.log(`Use proxy to download plugin.`); From eb4291a8e5c8fc99def353f1f0ddbc9d9c672b22 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Tue, 11 Jul 2017 14:55:02 +0200 Subject: [PATCH 05/12] Fix wording in downloader log --- src/cli_plugin/install/downloaders/http.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cli_plugin/install/downloaders/http.js b/src/cli_plugin/install/downloaders/http.js index 97ebf73a349e9..5be26ed81096a 100644 --- a/src/cli_plugin/install/downloaders/http.js +++ b/src/cli_plugin/install/downloaders/http.js @@ -19,7 +19,7 @@ function getProxyAgent(sourceUrl, logger) { const shouldProxy = !excludedHosts.includes(hostname); if (shouldProxy) { - logger.log(`Use proxy to download plugin.`); + logger.log(`Using proxy to download plugin.`); logger.log(`Hint: you can add ${hostname} to the no_proxy environment variable, ` + `to exclude that host from proxying.`); return new HttpProxyAgent(httpProxy); From 6b7b090f71c17c8d7f4841aab296456c29c3171a Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Tue, 11 Jul 2017 15:08:31 +0200 Subject: [PATCH 06/12] Refactor http downloader Extract getEnv method and use destructuring. --- src/cli_plugin/install/downloaders/http.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/cli_plugin/install/downloaders/http.js b/src/cli_plugin/install/downloaders/http.js index 5be26ed81096a..ecb24667cde73 100644 --- a/src/cli_plugin/install/downloaders/http.js +++ b/src/cli_plugin/install/downloaders/http.js @@ -5,14 +5,18 @@ import { createWriteStream } from 'fs'; import HttpProxyAgent from 'http-proxy-agent'; import URL from 'url'; +function getEnv(name) { + return process.env[name.toLowerCase()] || process.env[name.toUpperCase()]; +} + function getProxyAgent(sourceUrl, logger) { - const httpProxy = process.env.http_proxy || process.env.HTTP_PROXY; + const httpProxy = getEnv('http_proxy') || ''; // we have a proxy detected, lets use it if (httpProxy) { logger.log(`Picked up proxy ${httpProxy} from http_proxy environment variable.`); // get the hostname of the sourceUrl - const hostname = URL.parse(sourceUrl).hostname; - const noProxy = process.env.no_proxy || process.env.NO_PROXY || ''; + const { hostname } = URL.parse(sourceUrl); + const noProxy = getEnv('no_proxy') || ''; const excludedHosts = noProxy.split(','); // proxy if the hostname is not in noProxy From 3a67239f7c9ee62f56feb06ad23c066e1623c0a5 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Tue, 11 Jul 2017 15:16:48 +0200 Subject: [PATCH 07/12] Use early return in plugin downloader --- src/cli_plugin/install/downloaders/http.js | 40 ++++++++++++---------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/src/cli_plugin/install/downloaders/http.js b/src/cli_plugin/install/downloaders/http.js index ecb24667cde73..843445e33f0a7 100644 --- a/src/cli_plugin/install/downloaders/http.js +++ b/src/cli_plugin/install/downloaders/http.js @@ -12,25 +12,27 @@ function getEnv(name) { function getProxyAgent(sourceUrl, logger) { const httpProxy = getEnv('http_proxy') || ''; // we have a proxy detected, lets use it - if (httpProxy) { - logger.log(`Picked up proxy ${httpProxy} from http_proxy environment variable.`); - // get the hostname of the sourceUrl - const { hostname } = URL.parse(sourceUrl); - const noProxy = getEnv('no_proxy') || ''; - const excludedHosts = noProxy.split(','); - - // proxy if the hostname is not in noProxy - const shouldProxy = !excludedHosts.includes(hostname); - - if (shouldProxy) { - logger.log(`Using proxy to download plugin.`); - logger.log(`Hint: you can add ${hostname} to the no_proxy environment variable, ` - + `to exclude that host from proxying.`); - return new HttpProxyAgent(httpProxy); - } else { - logger.log(`Found exception for host ${hostname} in no_proxy environment variable. ` - + `Skipping proxy.`); - } + if (!httpProxy) { + return null; + } + + logger.log(`Picked up proxy ${httpProxy} from http_proxy environment variable.`); + // get the hostname of the sourceUrl + const { hostname } = URL.parse(sourceUrl); + const noProxy = getEnv('no_proxy') || ''; + const excludedHosts = noProxy.split(','); + + // proxy if the hostname is not in noProxy + const shouldProxy = !excludedHosts.includes(hostname); + + if (shouldProxy) { + logger.log(`Using proxy to download plugin.`); + logger.log(`Hint: you can add ${hostname} to the no_proxy environment variable, ` + + `to exclude that host from proxying.`); + return new HttpProxyAgent(httpProxy); + } else { + logger.log(`Found exception for host ${hostname} in no_proxy environment variable. ` + + `Skipping proxy.`); } return null; From fe339a8931f695779d17cef3d086ab3bc1a742d2 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Wed, 19 Jul 2017 20:27:23 +0200 Subject: [PATCH 08/12] Use proxy-from-env for plugin proxy support --- package.json | 1 + src/cli_plugin/install/__tests__/download.js | 117 +++++++++++++++++++ src/cli_plugin/install/downloaders/http.js | 34 +----- 3 files changed, 124 insertions(+), 28 deletions(-) diff --git a/package.json b/package.json index e09c0083d0b68..ca1f18828facb 100644 --- a/package.json +++ b/package.json @@ -163,6 +163,7 @@ "node-uuid": "1.4.7", "pegjs": "0.9.0", "postcss-loader": "1.2.1", + "proxy-from-env": "^1.0.0", "pui-react-overlay-trigger": "7.5.4", "pui-react-tooltip": "7.5.4", "querystring-browser": "1.0.4", diff --git a/src/cli_plugin/install/__tests__/download.js b/src/cli_plugin/install/__tests__/download.js index 63cddbfc3fdd8..92ff496303aef 100644 --- a/src/cli_plugin/install/__tests__/download.js +++ b/src/cli_plugin/install/__tests__/download.js @@ -8,6 +8,7 @@ import Logger from '../../lib/logger'; import { UnsupportedProtocolError } from '../../lib/errors'; import { download, _downloadSingle, _getFilePath, _checkFilePathDeprecation } from '../download'; import { join } from 'path'; +import http from 'http'; describe('kibana cli', function () { @@ -253,6 +254,122 @@ describe('kibana cli', function () { }); + describe('proxy support', function () { + + const proxyPort = 2626; + const proxyUrl = `http://localhost:${proxyPort}`; + + let proxyHit = false; + + const proxy = http.createServer(function (req, res) { + proxyHit = true; + // Our test proxy simply returns an empty 200 response, since we only + // care about the download promise being resolved. + res.writeHead(200); + res.end(); + }).listen(proxyPort); + + function expectProxyHit() { + expect(proxyHit).to.be(true); + } + + function expectNoProxyHit() { + expect(proxyHit).to.be(false); + } + + function nockPluginForUrl(url) { + nock(url) + .get('/plugin.zip') + .replyWithFile(200, join(__dirname, 'replies/test_plugin.zip')); + } + + beforeEach(function () { + proxyHit = false; + }); + + afterEach(function () { + delete process.env.http_proxy; + delete process.env.https_proxy; + delete process.env.no_proxy; + }); + + it('should use http_proxy env variable', function () { + process.env.http_proxy = proxyUrl; + settings.urls = ['http://example.com/plugin.zip']; + + return download(settings, logger) + .then(expectProxyHit); + }); + + it('should use https_proxy for secure URLs', function () { + process.env.https_proxy = proxyUrl; + settings.urls = ['https://example.com/plugin.zip']; + + return download(settings, logger) + .then(expectProxyHit); + }); + + it('should support domains in no_proxy', function () { + process.env.http_proxy = proxyUrl; + process.env.no_proxy = 'foo.bar, example.com'; + settings.urls = ['http://example.com/plugin.zip']; + + nockPluginForUrl('http://example.com'); + + return download(settings, logger) + .then(expectNoProxyHit); + }); + + it('should support subdomains in no_proxy', function () { + process.env.http_proxy = proxyUrl; + process.env.no_proxy = 'foo.bar,plugins.example.com'; + settings.urls = ['http://plugins.example.com/plugin.zip']; + + nockPluginForUrl('http://plugins.example.com'); + + return download(settings, logger) + .then(expectNoProxyHit); + }); + + it('should accept wildcard subdomains in no_proxy', function () { + process.env.http_proxy = proxyUrl; + process.env.no_proxy = 'foo.bar, .example.com'; + settings.urls = ['http://plugins.example.com/plugin.zip']; + + nockPluginForUrl('http://plugins.example.com'); + + return download(settings, logger) + .then(expectNoProxyHit); + }); + + it('should support asterisk wildcard no_proxy syntax', function () { + process.env.http_proxy = proxyUrl; + process.env.no_proxy = '*.example.com'; + settings.urls = ['http://plugins.example.com/plugin.zip']; + + nockPluginForUrl('http://plugins.example.com'); + + return download(settings, logger) + .then(expectNoProxyHit); + }); + + it('should support implicit ports in no_proxy', function () { + process.env.https_proxy = proxyUrl; + process.env.no_proxy = 'example.com:443'; + settings.urls = ['https://example.com/plugin.zip']; + + nockPluginForUrl('https://example.com'); + + return download(settings, logger) + .then(expectNoProxyHit); + }); + + after(function () { + proxy.close(); + }); + + }); + }); }); diff --git a/src/cli_plugin/install/downloaders/http.js b/src/cli_plugin/install/downloaders/http.js index 843445e33f0a7..1c261be5643e5 100644 --- a/src/cli_plugin/install/downloaders/http.js +++ b/src/cli_plugin/install/downloaders/http.js @@ -3,39 +3,17 @@ import Progress from '../progress'; import { fromNode as fn } from 'bluebird'; import { createWriteStream } from 'fs'; import HttpProxyAgent from 'http-proxy-agent'; -import URL from 'url'; - -function getEnv(name) { - return process.env[name.toLowerCase()] || process.env[name.toUpperCase()]; -} +import { getProxyForUrl } from 'proxy-from-env'; function getProxyAgent(sourceUrl, logger) { - const httpProxy = getEnv('http_proxy') || ''; - // we have a proxy detected, lets use it - if (!httpProxy) { - return null; - } + const proxy = getProxyForUrl(sourceUrl); - logger.log(`Picked up proxy ${httpProxy} from http_proxy environment variable.`); - // get the hostname of the sourceUrl - const { hostname } = URL.parse(sourceUrl); - const noProxy = getEnv('no_proxy') || ''; - const excludedHosts = noProxy.split(','); - - // proxy if the hostname is not in noProxy - const shouldProxy = !excludedHosts.includes(hostname); - - if (shouldProxy) { - logger.log(`Using proxy to download plugin.`); - logger.log(`Hint: you can add ${hostname} to the no_proxy environment variable, ` - + `to exclude that host from proxying.`); - return new HttpProxyAgent(httpProxy); - } else { - logger.log(`Found exception for host ${hostname} in no_proxy environment variable. ` - + `Skipping proxy.`); + if (!proxy) { + return null; } - return null; + logger.log(`Picked up proxy ${proxy} from environment variable.`); + return new HttpProxyAgent(proxy); } function sendRequest({ sourceUrl, timeout }, logger) { From e08c35a6af2e0d0125655a86bcbf3fc9445c2d8b Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Wed, 19 Jul 2017 20:34:40 +0200 Subject: [PATCH 09/12] Add documentation on plugin proxy support --- docs/plugins.asciidoc | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/plugins.asciidoc b/docs/plugins.asciidoc index 0c2d8355dc4e2..b9365dd54b075 100644 --- a/docs/plugins.asciidoc +++ b/docs/plugins.asciidoc @@ -66,6 +66,20 @@ If plugins were installed as a different user and the server is not starting, th [source,shell] $ chown -R kibana:kibana /path/to/kibana/optimize +[float] +=== Proxy support for plugin installation + +Kibana supports plugin installation via a proxy. It uses the `http_proxy` and `https_proxy` +environment variables to detect a proxy for HTTP and HTTPS URLs. + +It also respects the `no_proxy` environment variable to exclude specific URLs from proxying. + +You can specify the environment variable directly when installing plugins: + +[source,shell] +$ http_proxy="http://proxy.local:4242" bin/kibana-plugin install + + == Updating & Removing Plugins To update a plugin, remove the current version and reinstall the plugin. From 4a422d84b4d1b220a09d7bacd74a0dc41644f4de Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Thu, 20 Jul 2017 08:46:16 +0200 Subject: [PATCH 10/12] Resolve issue with dirty state after previous tests --- src/cli_plugin/install/__tests__/download.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/cli_plugin/install/__tests__/download.js b/src/cli_plugin/install/__tests__/download.js index 92ff496303aef..394a0ceea0ba0 100644 --- a/src/cli_plugin/install/__tests__/download.js +++ b/src/cli_plugin/install/__tests__/download.js @@ -252,6 +252,10 @@ describe('kibana cli', function () { }); }); + after(function () { + nock.cleanAll(); + }); + }); describe('proxy support', function () { From 04d7ffa1d18862fa310612c93bac685990bb0bb9 Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Thu, 20 Jul 2017 08:46:48 +0200 Subject: [PATCH 11/12] Wait for server listen and close in before/after --- src/cli_plugin/install/__tests__/download.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/cli_plugin/install/__tests__/download.js b/src/cli_plugin/install/__tests__/download.js index 394a0ceea0ba0..e1f45bc08161d 100644 --- a/src/cli_plugin/install/__tests__/download.js +++ b/src/cli_plugin/install/__tests__/download.js @@ -271,7 +271,7 @@ describe('kibana cli', function () { // care about the download promise being resolved. res.writeHead(200); res.end(); - }).listen(proxyPort); + }); function expectProxyHit() { expect(proxyHit).to.be(true); @@ -287,6 +287,10 @@ describe('kibana cli', function () { .replyWithFile(200, join(__dirname, 'replies/test_plugin.zip')); } + before(function (done) { + proxy.listen(proxyPort, done); + }); + beforeEach(function () { proxyHit = false; }); @@ -368,8 +372,8 @@ describe('kibana cli', function () { .then(expectNoProxyHit); }); - after(function () { - proxy.close(); + after(function (done) { + proxy.close(done); }); }); From 151cb7952960735738c2e4a5f86435201742d42a Mon Sep 17 00:00:00 2001 From: Tim Roes Date: Tue, 25 Jul 2017 11:47:40 +0200 Subject: [PATCH 12/12] Add test for interchanged proxy usage --- src/cli_plugin/install/__tests__/download.js | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/src/cli_plugin/install/__tests__/download.js b/src/cli_plugin/install/__tests__/download.js index e1f45bc08161d..4693ed10e853b 100644 --- a/src/cli_plugin/install/__tests__/download.js +++ b/src/cli_plugin/install/__tests__/download.js @@ -317,6 +317,26 @@ describe('kibana cli', function () { .then(expectProxyHit); }); + it('should not use http_proxy for HTTPS urls', function () { + process.env.http_proxy = proxyUrl; + settings.urls = ['https://example.com/plugin.zip']; + + nockPluginForUrl('https://example.com'); + + return download(settings, logger) + .then(expectNoProxyHit); + }); + + it('should not use https_proxy for HTTP urls', function () { + process.env.https_proxy = proxyUrl; + settings.urls = ['http://example.com/plugin.zip']; + + nockPluginForUrl('http://example.com'); + + return download(settings, logger) + .then(expectNoProxyHit); + }); + it('should support domains in no_proxy', function () { process.env.http_proxy = proxyUrl; process.env.no_proxy = 'foo.bar, example.com';