[build] Use docker to manage dependencies

.dockerignore

@@ -0,0 +1,10 @@
+.aws-config.json
+.signing-config.json
+.node_binaries
+node_modules
+optimize
+target
+build
+html_docs
+esvm
+plugins

CONTRIBUTING.md

@@ -307,11 +307,9 @@ npm run test:browser -- --dev # remove the --dev flag to run them once and close
 
 ### Building OS packages
 
-Packages are built using fpm, dpkg, and rpm.  Package building has only been tested on Linux and is not supported on any other platform.
+Packages are built in a Docker container.
 
 ```bash
-apt-get install ruby-dev rpm
-gem install fpm -v 1.5.0
 npm run build -- --skip-archives
 ```
 

package.json

@@ -51,7 +51,7 @@
     "test:coverage": "grunt test:coverage",
     "test:visualRegression": "grunt test:visualRegression:buildGallery",
     "checkLicenses": "grunt licenses",
-    "build": "grunt build",
+    "build": "grunt docker:packages",
     "release": "grunt release",
     "start": "sh ./bin/kibana --dev",
     "precommit": "grunt precommit",

tasks/build/archives.js

@@ -13,17 +13,12 @@ export default (grunt) => {
   }
 
   async function archives({ name, buildName, zipPath, tarPath }) {
+    const tarArguments = ['-zchf', tarPath, buildName];
     if (/windows/.test(name)) {
       await exec('zip', ['-rq', '-ll', zipPath, buildName]);
     } else {
-      const tarArguments = ['-zchf', tarPath, buildName];
-
-      // Add a flag to handle filepaths with colons (i.e. C://...) on windows
-      if (/^win/.test(process.platform)) {
-        tarArguments.push('--force-local');
-      }
-
       await exec('tar', tarArguments);
+
     }
   }
 

tasks/build/docker/packages/Dockerfile

@@ -0,0 +1,35 @@
+ARG node_version
+FROM node:$node_version
+
+ARG kibana_folder
+
+RUN set -x && \
+  apt-get update && \
+  apt-get install -y \
+    ruby-dev \
+    ruby \
+    zip \
+    rpm
+RUN gem install bundler
+
+RUN groupadd --gid 1010 kibana \
+  && useradd --uid 1010 --gid kibana --shell /bin/bash --create-home kibana
+
+RUN mkdir -p $kibana_folder
+COPY . $kibana_folder
+RUN chown -R kibana:kibana $kibana_folder
+
+USER kibana
+
+WORKDIR /tmp
+COPY tasks/build/docker/packages/Gemfile .
+COPY tasks/build/docker/packages/Gemfile.lock .
+RUN bundle install --deployment
+ENV BUNDLE_GEMFILE=/tmp/Gemfile
+
+WORKDIR $kibana_folder
+ENV NPM_CONFIG_LOGLEVEL error
+RUN npm install
+
+
+CMD node_modules/.bin/grunt build $BUILD_OPTIONS

tasks/build/docker/packages/Gemfile

@@ -0,0 +1,3 @@
+source "https://rubygems.org"
+gem "pleaserun", "= 0.0.24"
+gem "fpm", "=1.5.0"

tasks/build/docker/packages/Gemfile.lock

@@ -0,0 +1,57 @@
+GEM
+  remote: https://rubygems.org/
+  specs:
+    archive-tar-minitar (0.6.1)
+      minitar (~> 0.6)
+      minitar-cli (~> 0.6)
+    arr-pm (0.0.10)
+      cabin (> 0)
+    backports (3.8.0)
+    cabin (0.9.0)
+    childprocess (0.7.1)
+      ffi (~> 1.0, >= 1.0.11)
+    clamp (0.6.5)
+    corefines (1.10.0)
+    ffi (1.9.18)
+    fpm (1.5.0)
+      archive-tar-minitar
+      arr-pm (~> 0.0.10)
+      backports (>= 2.6.2)
+      cabin (>= 0.6.0)
+      childprocess
+      clamp (~> 0.6)
+      corefines (~> 1.9)
+      ffi
+      json (>= 1.7.7)
+      ruby-xz
+    hashie (3.5.6)
+    insist (1.0.0)
+    io-like (0.3.0)
+    json (2.1.0)
+    minitar (0.6.1)
+    minitar-cli (0.6.1)
+      minitar (~> 0.6.0)
+      powerbar (~> 1.0)
+    mustache (0.99.8)
+    pleaserun (0.0.24)
+      cabin (> 0)
+      clamp
+      insist
+      mustache (= 0.99.8)
+      stud
+    powerbar (1.0.18)
+      hashie (>= 1.1.0)
+    ruby-xz (0.2.3)
+      ffi (~> 1.9)
+      io-like (~> 0.3)
+    stud (0.0.23)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  fpm (= 1.5.0)
+  pleaserun (= 0.0.24)
+
+BUNDLED WITH
+   1.15.1

tasks/build/docker/packages/docker-compose.yml

@@ -0,0 +1,12 @@
+version: '2'
+services:
+  kibana_build:
+    container_name: $KIBANA_BUILD_CONTAINER_NAME
+    environment:
+      - BUILD_OPTIONS=$KIBANA_BUILD_OPTIONS
+    build:
+      context: $KIBANA_BUILD_CONTEXT
+      dockerfile: $KIBANA_BUILD_CONTEXT/tasks/build/docker/packages/Dockerfile
+      args:
+        - node_version=$KIBANA_NODE_VERSION
+        - kibana_folder=$KIBANA_FOLDER

tasks/build/docker_build.js

@@ -0,0 +1,59 @@
+import rimraf from 'rimraf';
+import { join } from 'path';
+import { execFileSync as exec } from 'child_process';
+
+module.exports = function (grunt) {
+  grunt.registerTask('docker:packages', 'Build packages from docker', function () {
+    const composePath = join(grunt.config.get('root'), 'tasks/build/docker/packages/docker-compose.yml');
+    const env = Object.assign(process.env, {
+      KIBANA_NODE_VERSION:  grunt.config.get('nodeVersion'),
+      KIBANA_BUILD_CONTEXT: grunt.config.get('root'),
+      KIBANA_BUILD_OPTIONS: grunt.option.flags().join(' '),
+      KIBANA_BUILD_CONTAINER_NAME: 'kibana_build',
+      KIBANA_FOLDER: '/home/kibana/repo'
+    });
+    const stdio = [0, 1, 2];
+    const execOptions = { env, stdio };
+
+    const useCache = grunt.option('cache');
+
+    const targetDir = join(grunt.config.get('root'), 'target');
+    const buildDir = join(grunt.config.get('root'), 'build');
+
+    exec('docker-compose', [
+      '-f', composePath,
+      'build',
+      useCache ? '' : '--no-cache',
+    ].filter(Boolean), execOptions);
+    exec('docker-compose', [
+      '-f', composePath,
+      'up'
+    ], execOptions);
+
+    const containerId = String(exec('docker-compose', [
+      '-f', composePath,
+      'ps',
+      '-q', env.KIBANA_BUILD_CONTAINER_NAME
+    ], { env })).trim();
+
+    rimraf.sync(targetDir);
+    rimraf.sync(buildDir);
+
+    exec('docker', [
+      'cp',
+      `${containerId}:${env.KIBANA_FOLDER}/target`,
+      targetDir
+    ]);
+    exec('docker', [
+      'cp',
+      `${containerId}:${env.KIBANA_FOLDER}/build`,
+      buildDir
+    ], execOptions);
+
+    exec('docker-compose', [
+      '-f', composePath,
+      'rm',
+      '--force'
+    ], execOptions);
+  });
+};

tasks/build/index.js

@@ -1,4 +1,5 @@
 import { flatten } from 'lodash';
+
 module.exports = function (grunt) {
   grunt.registerTask('build', 'Build packages', function () {
     grunt.task.run(flatten([

tasks/build/os_packages.js

@@ -8,7 +8,7 @@ export default (grunt) => {
   const packageScriptsDir = grunt.config.get('packageScriptsDir');
   const servicesByName = indexBy(config.get('services'), 'name');
   const packages = config.get('packages');
-  const fpm = args => exec('fpm', args);
+  const fpm = (args, options = {}) => exec('bundle', args, options);
 
   grunt.registerTask('_build:osPackages', function () {
     grunt.file.mkdir(targetDir);
@@ -17,6 +17,8 @@ export default (grunt) => {
     .filter(({ name }) => /linux-x86_64$/.test(name))
     .forEach(({ buildDir, debArch, rpmArch }) => {
       const baseOptions = [
+        'exec',
+        'fpm',
         '--force',
         // we force dashes in the version file name because otherwise fpm uses
         // the filtered package version, which would have dashes replaced with
@@ -64,15 +66,19 @@ export default (grunt) => {
         `${servicesByName.systemd.outputDir}/etc/=/etc/`
       ];
 
+      const options = {
+        cwd: grunt.config.get('root')
+      };
+
       //Manually find flags, multiple args without assignment are not entirely parsed
       const flags = grunt.option.flags().filter(flag => /deb|rpm/.test(flag)).join(',');
       const buildDeb = flags.includes('deb') || !flags.length;
       const buildRpm = flags.includes('rpm') || !flags.length;
       if (buildDeb) {
-        fpm([...baseOptions, ...debOptions, ...args]);
+        fpm([...baseOptions, ...debOptions, ...args], options);
       }
       if (buildRpm) {
-        fpm([...baseOptions, ...rpmOptions, ...args]);
+        fpm([...baseOptions, ...rpmOptions, ...args], options);
       }
 
     });

tasks/build/shasums.js

@@ -1,8 +1,6 @@
 import { promisify } from 'bluebird';
 const readdir = promisify(require('fs').readdir);
 const exec = promisify(require('child_process').exec);
-const platform = require('os').platform();
-const cmd = /^win/.test(platform) ? 'sha1sum ' : 'shasum ';
 
 module.exports = function (grunt) {
   grunt.registerTask('_build:shasums', function () {
@@ -16,7 +14,7 @@ module.exports = function (grunt) {
       // only sha the archives and packages
       if (!archive.match(/\.zip$|\.tar.gz$|\.deb$|\.rpm$/)) return;
 
-      return exec(cmd + archive + ' > ' + archive + '.sha1.txt', {
+      return exec(`shasum ${archive} > ${archive}.sha1.txt`, {
         cwd: targetDir
       });
     })

test/scripts/jenkins_selenium.sh

@@ -3,6 +3,6 @@
 set -e
 source "$(dirname $0)/_jenkins_setup.sh"
 
-"$(npm bin)/grunt" build --release;
+"$(npm bin)/grunt" docker:packages --release;
 
 xvfb-run "$(npm bin)/grunt" jenkins:selenium;