Replaces zip library and small refactor of plugin install (#11145)

Previously our plugin installation used over 5,000 file descriptors when analyzing and extracting the archive. To mitigate this, we are moving to a more supported zip library, Yauzl.

In addition to the replacement of the zip library, this refactors the getPackData method. It was extracting the plugin package.json files, reading them, then finally deleting them. This commit performs this all in memory, removing the need for any file system operations during this step.

These changes reduced the installation time by an average of 35%, from 211.18 seconds to 137.8 seconds. The majority of this time is now spent during optimize.

Signed-off-by: Tyler Smalley <[email protected]>

package.json

@@ -70,7 +70,6 @@
     "url": "https://github.com/elastic/kibana.git"
   },
   "dependencies": {
-    "@bigfunger/decompress-zip": "0.2.0-stripfix3",
     "@bigfunger/jsondiffpatch": "0.1.38-webpack",
     "@elastic/datemath": "2.3.0",
     "@elastic/httpolyglot": "0.1.2-elasticpatch1",
@@ -197,7 +196,8 @@
     "vision": "4.1.0",
     "webpack": "github:elastic/webpack#fix/query-params-for-aliased-loaders",
     "whatwg-fetch": "0.9.0",
-    "wreck": "6.2.0"
+    "wreck": "6.2.0",
+    "yauzl": "2.7.0"
   },
   "devDependencies": {
     "@elastic/eslint-config-kibana": "0.5.0",
@@ -257,7 +257,7 @@
     "makelogs": "3.2.3",
     "marked-text-renderer": "0.1.0",
     "mocha": "2.5.3",
-    "mock-fs": "4.0.0",
+    "mock-fs": "4.2.0",
     "murmurhash3js": "3.0.1",
     "ncp": "2.0.0",
     "nock": "8.0.0",

src/cli_plugin/install/__tests__/kibana.js

@@ -1,5 +1,6 @@
 import expect from 'expect.js';
 import sinon from 'sinon';
+import mockFs from 'mock-fs';
 import Logger from '../../lib/logger';
 import { join } from 'path';
 import rimraf from 'rimraf';
@@ -13,13 +14,15 @@ describe('kibana cli', function () {
     describe('kibana', function () {
       const testWorkingPath = join(__dirname, '.test.data');
       const tempArchiveFilePath = join(testWorkingPath, 'archive.part');
+      const pluginDir = join(__dirname, 'plugins');
 
       const settings = {
         workingPath: testWorkingPath,
         tempArchiveFile: tempArchiveFilePath,
         plugin: 'test-plugin',
         version: '1.0.0',
-        plugins: [ { name: 'foo', path: join(testWorkingPath, 'foo') } ]
+        plugins: [ { name: 'foo' } ],
+        pluginDir
       };
 
       const logger = new Logger(settings);
@@ -130,13 +133,10 @@ describe('kibana cli', function () {
       });
 
       describe('existingInstall', function () {
-        let testWorkingPath;
         let processExitStub;
 
         beforeEach(function () {
           processExitStub = sinon.stub(process, 'exit');
-          testWorkingPath = join(__dirname, '.test.data');
-          rimraf.sync(testWorkingPath);
           sinon.stub(logger, 'log');
           sinon.stub(logger, 'error');
         });
@@ -145,26 +145,23 @@ describe('kibana cli', function () {
           processExitStub.restore();
           logger.log.restore();
           logger.error.restore();
-          rimraf.sync(testWorkingPath);
         });
 
-        it('should throw an error if the workingPath already exists.', function () {
-          mkdirp.sync(settings.plugins[0].path);
-          existingInstall(settings, logger);
+        it('should throw an error if the plugin already exists.', function () {
+          mockFs({ [`${pluginDir}/foo`]: {} });
 
+          existingInstall(settings, logger);
           expect(logger.error.firstCall.args[0]).to.match(/already exists/);
           expect(process.exit.called).to.be(true);
+
+          mockFs.restore();
         });
 
-        it('should not throw an error if the workingPath does not exist.', function () {
+        it('should not throw an error if the plugin does not exist.', function () {
           existingInstall(settings, logger);
           expect(logger.error.called).to.be(false);
         });
-
       });
-
     });
-
   });
-
 });

src/cli_plugin/install/__tests__/pack.js

@@ -98,10 +98,9 @@ describe('kibana cli', function () {
         })
         .then(() => {
           expect(settings.plugins[0].name).to.be('test-plugin');
-          expect(settings.plugins[0].folder).to.be('test-plugin');
+          expect(settings.plugins[0].archivePath).to.be('kibana/test-plugin');
           expect(settings.plugins[0].version).to.be('1.0.0');
           expect(settings.plugins[0].kibanaVersion).to.be('1.0.0');
-          expect(settings.plugins[0].platform).to.be(undefined);
         });
       });
 
@@ -134,40 +133,28 @@ describe('kibana cli', function () {
         })
         .then(() => {
           expect(settings.plugins[0].name).to.be('funger-plugin');
-          expect(settings.plugins[0].file).to.be('kibana/funger-plugin/package.json');
-          expect(settings.plugins[0].folder).to.be('funger-plugin');
+          expect(settings.plugins[0].archivePath).to.be('kibana/funger-plugin');
           expect(settings.plugins[0].version).to.be('1.0.0');
-          expect(settings.plugins[0].platform).to.be(undefined);
 
           expect(settings.plugins[1].name).to.be('pdf');
-          expect(settings.plugins[1].file).to.be('kibana/pdf-linux/package.json');
-          expect(settings.plugins[1].folder).to.be('pdf-linux');
+          expect(settings.plugins[1].archivePath).to.be('kibana/pdf-linux');
           expect(settings.plugins[1].version).to.be('1.0.0');
-          expect(settings.plugins[1].platform).to.be('linux');
 
           expect(settings.plugins[2].name).to.be('pdf');
-          expect(settings.plugins[2].file).to.be('kibana/pdf-win32/package.json');
-          expect(settings.plugins[2].folder).to.be('pdf-win32');
+          expect(settings.plugins[2].archivePath).to.be('kibana/pdf-win32');
           expect(settings.plugins[2].version).to.be('1.0.0');
-          expect(settings.plugins[2].platform).to.be('win32');
 
           expect(settings.plugins[3].name).to.be('pdf');
-          expect(settings.plugins[3].file).to.be('kibana/pdf-win64/package.json');
-          expect(settings.plugins[3].folder).to.be('pdf-win64');
+          expect(settings.plugins[3].archivePath).to.be('kibana/pdf-win64');
           expect(settings.plugins[3].version).to.be('1.0.0');
-          expect(settings.plugins[3].platform).to.be('win64');
 
           expect(settings.plugins[4].name).to.be('pdf');
-          expect(settings.plugins[4].file).to.be('kibana/pdf/package.json');
-          expect(settings.plugins[4].folder).to.be('pdf');
+          expect(settings.plugins[4].archivePath).to.be('kibana/pdf');
           expect(settings.plugins[4].version).to.be('1.0.0');
-          expect(settings.plugins[4].platform).to.be(undefined);
 
           expect(settings.plugins[5].name).to.be('test-plugin');
-          expect(settings.plugins[5].file).to.be('kibana/test-plugin/package.json');
-          expect(settings.plugins[5].folder).to.be('test-plugin');
+          expect(settings.plugins[5].archivePath).to.be('kibana/test-plugin');
           expect(settings.plugins[5].version).to.be('1.0.0');
-          expect(settings.plugins[5].platform).to.be(undefined);
         });
       });