From 5fa48633eed091ebbc20795b244de7967c78d906 Mon Sep 17 00:00:00 2001 From: Jonathan Budzenski Date: Fri, 4 Aug 2017 13:50:21 -0500 Subject: [PATCH] [build] Use docker to manage dependencies --- .dockerignore | 10 ++++ CONTRIBUTING.md | 4 +- package.json | 2 +- tasks/build/archives.js | 9 +-- tasks/build/docker/packages/Dockerfile | 35 +++++++++++ tasks/build/docker/packages/Gemfile | 3 + tasks/build/docker/packages/Gemfile.lock | 57 ++++++++++++++++++ .../build/docker/packages/docker-compose.yml | 12 ++++ tasks/build/docker_build.js | 59 +++++++++++++++++++ tasks/build/index.js | 1 + tasks/build/os_packages.js | 12 +++- tasks/build/shasums.js | 4 +- test/scripts/jenkins_selenium.sh | 2 +- 13 files changed, 192 insertions(+), 18 deletions(-) create mode 100644 .dockerignore create mode 100644 tasks/build/docker/packages/Dockerfile create mode 100644 tasks/build/docker/packages/Gemfile create mode 100644 tasks/build/docker/packages/Gemfile.lock create mode 100644 tasks/build/docker/packages/docker-compose.yml create mode 100644 tasks/build/docker_build.js diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000000000..50f100ce092f1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,10 @@ +.aws-config.json +.signing-config.json +.node_binaries +node_modules +optimize +target +build +html_docs +esvm +plugins diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b1c403ae945a8..13079b56556ec 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -307,11 +307,9 @@ npm run test:browser -- --dev # remove the --dev flag to run them once and close ### Building OS packages -Packages are built using fpm, dpkg, and rpm. Package building has only been tested on Linux and is not supported on any other platform. +Packages are built in a Docker container. ```bash -apt-get install ruby-dev rpm -gem install fpm -v 1.5.0 npm run build -- --skip-archives ``` diff --git a/package.json b/package.json index e682686a9fc00..bc3c3cecfc153 100644 --- a/package.json +++ b/package.json @@ -51,7 +51,7 @@ "test:coverage": "grunt test:coverage", "test:visualRegression": "grunt test:visualRegression:buildGallery", "checkLicenses": "grunt licenses", - "build": "grunt build", + "build": "grunt docker:packages", "release": "grunt release", "start": "sh ./bin/kibana --dev", "precommit": "grunt precommit", diff --git a/tasks/build/archives.js b/tasks/build/archives.js index 1c8de1d8f7e0b..f4223c1ad61c6 100644 --- a/tasks/build/archives.js +++ b/tasks/build/archives.js @@ -13,17 +13,12 @@ export default (grunt) => { } async function archives({ name, buildName, zipPath, tarPath }) { + const tarArguments = ['-zchf', tarPath, buildName]; if (/windows/.test(name)) { await exec('zip', ['-rq', '-ll', zipPath, buildName]); } else { - const tarArguments = ['-zchf', tarPath, buildName]; - - // Add a flag to handle filepaths with colons (i.e. C://...) on windows - if (/^win/.test(process.platform)) { - tarArguments.push('--force-local'); - } - await exec('tar', tarArguments); + } } diff --git a/tasks/build/docker/packages/Dockerfile b/tasks/build/docker/packages/Dockerfile new file mode 100644 index 0000000000000..c742680145fcd --- /dev/null +++ b/tasks/build/docker/packages/Dockerfile @@ -0,0 +1,35 @@ +ARG node_version +FROM node:$node_version + +ARG kibana_folder + +RUN set -x && \ + apt-get update && \ + apt-get install -y \ + ruby-dev \ + ruby \ + zip \ + rpm +RUN gem install bundler + +RUN groupadd --gid 1010 kibana \ + && useradd --uid 1010 --gid kibana --shell /bin/bash --create-home kibana + +RUN mkdir -p $kibana_folder +COPY . $kibana_folder +RUN chown -R kibana:kibana $kibana_folder + +USER kibana + +WORKDIR /tmp +COPY tasks/build/docker/packages/Gemfile . +COPY tasks/build/docker/packages/Gemfile.lock . +RUN bundle install --deployment +ENV BUNDLE_GEMFILE=/tmp/Gemfile + +WORKDIR $kibana_folder +ENV NPM_CONFIG_LOGLEVEL error +RUN npm install + + +CMD node_modules/.bin/grunt build $BUILD_OPTIONS diff --git a/tasks/build/docker/packages/Gemfile b/tasks/build/docker/packages/Gemfile new file mode 100644 index 0000000000000..5e1c2fbc23448 --- /dev/null +++ b/tasks/build/docker/packages/Gemfile @@ -0,0 +1,3 @@ +source "https://rubygems.org" +gem "pleaserun", "= 0.0.24" +gem "fpm", "=1.5.0" diff --git a/tasks/build/docker/packages/Gemfile.lock b/tasks/build/docker/packages/Gemfile.lock new file mode 100644 index 0000000000000..8af63284216c4 --- /dev/null +++ b/tasks/build/docker/packages/Gemfile.lock @@ -0,0 +1,57 @@ +GEM + remote: https://rubygems.org/ + specs: + archive-tar-minitar (0.6.1) + minitar (~> 0.6) + minitar-cli (~> 0.6) + arr-pm (0.0.10) + cabin (> 0) + backports (3.8.0) + cabin (0.9.0) + childprocess (0.7.1) + ffi (~> 1.0, >= 1.0.11) + clamp (0.6.5) + corefines (1.10.0) + ffi (1.9.18) + fpm (1.5.0) + archive-tar-minitar + arr-pm (~> 0.0.10) + backports (>= 2.6.2) + cabin (>= 0.6.0) + childprocess + clamp (~> 0.6) + corefines (~> 1.9) + ffi + json (>= 1.7.7) + ruby-xz + hashie (3.5.6) + insist (1.0.0) + io-like (0.3.0) + json (2.1.0) + minitar (0.6.1) + minitar-cli (0.6.1) + minitar (~> 0.6.0) + powerbar (~> 1.0) + mustache (0.99.8) + pleaserun (0.0.24) + cabin (> 0) + clamp + insist + mustache (= 0.99.8) + stud + powerbar (1.0.18) + hashie (>= 1.1.0) + ruby-xz (0.2.3) + ffi (~> 1.9) + io-like (~> 0.3) + stud (0.0.23) + +PLATFORMS + ruby + +DEPENDENCIES + fpm (= 1.5.0) + pleaserun (= 0.0.24) + +BUNDLED WITH + 1.15.1 diff --git a/tasks/build/docker/packages/docker-compose.yml b/tasks/build/docker/packages/docker-compose.yml new file mode 100644 index 0000000000000..e885ea4b37b46 --- /dev/null +++ b/tasks/build/docker/packages/docker-compose.yml @@ -0,0 +1,12 @@ +version: '2' +services: + kibana_build: + container_name: $KIBANA_BUILD_CONTAINER_NAME + environment: + - BUILD_OPTIONS=$KIBANA_BUILD_OPTIONS + build: + context: $KIBANA_BUILD_CONTEXT + dockerfile: $KIBANA_BUILD_CONTEXT/tasks/build/docker/packages/Dockerfile + args: + - node_version=$KIBANA_NODE_VERSION + - kibana_folder=$KIBANA_FOLDER diff --git a/tasks/build/docker_build.js b/tasks/build/docker_build.js new file mode 100644 index 0000000000000..9bcddee3366c2 --- /dev/null +++ b/tasks/build/docker_build.js @@ -0,0 +1,59 @@ +import rimraf from 'rimraf'; +import { join } from 'path'; +import { execFileSync as exec } from 'child_process'; + +module.exports = function (grunt) { + grunt.registerTask('docker:packages', 'Build packages from docker', function () { + const composePath = join(grunt.config.get('root'), 'tasks/build/docker/packages/docker-compose.yml'); + const env = Object.assign(process.env, { + KIBANA_NODE_VERSION: grunt.config.get('nodeVersion'), + KIBANA_BUILD_CONTEXT: grunt.config.get('root'), + KIBANA_BUILD_OPTIONS: grunt.option.flags().join(' '), + KIBANA_BUILD_CONTAINER_NAME: 'kibana_build', + KIBANA_FOLDER: '/home/kibana/repo' + }); + const stdio = [0, 1, 2]; + const execOptions = { env, stdio }; + + const useCache = grunt.option('cache'); + + const targetDir = join(grunt.config.get('root'), 'target'); + const buildDir = join(grunt.config.get('root'), 'build'); + + exec('docker-compose', [ + '-f', composePath, + 'build', + useCache ? '' : '--no-cache', + ].filter(Boolean), execOptions); + exec('docker-compose', [ + '-f', composePath, + 'up' + ], execOptions); + + const containerId = String(exec('docker-compose', [ + '-f', composePath, + 'ps', + '-q', env.KIBANA_BUILD_CONTAINER_NAME + ], { env })).trim(); + + rimraf.sync(targetDir); + rimraf.sync(buildDir); + + exec('docker', [ + 'cp', + `${containerId}:${env.KIBANA_FOLDER}/target`, + targetDir + ]); + exec('docker', [ + 'cp', + `${containerId}:${env.KIBANA_FOLDER}/build`, + buildDir + ], execOptions); + + exec('docker-compose', [ + '-f', composePath, + 'rm', + '--force' + ], execOptions); + }); +}; diff --git a/tasks/build/index.js b/tasks/build/index.js index 4d0227016ee88..f6f97442ed9f1 100644 --- a/tasks/build/index.js +++ b/tasks/build/index.js @@ -1,4 +1,5 @@ import { flatten } from 'lodash'; + module.exports = function (grunt) { grunt.registerTask('build', 'Build packages', function () { grunt.task.run(flatten([ diff --git a/tasks/build/os_packages.js b/tasks/build/os_packages.js index 1109c6d53d7ec..9fd7f785d9525 100644 --- a/tasks/build/os_packages.js +++ b/tasks/build/os_packages.js @@ -8,7 +8,7 @@ export default (grunt) => { const packageScriptsDir = grunt.config.get('packageScriptsDir'); const servicesByName = indexBy(config.get('services'), 'name'); const packages = config.get('packages'); - const fpm = args => exec('fpm', args); + const fpm = (args, options = {}) => exec('bundle', args, options); grunt.registerTask('_build:osPackages', function () { grunt.file.mkdir(targetDir); @@ -17,6 +17,8 @@ export default (grunt) => { .filter(({ name }) => /linux-x86_64$/.test(name)) .forEach(({ buildDir, debArch, rpmArch }) => { const baseOptions = [ + 'exec', + 'fpm', '--force', // we force dashes in the version file name because otherwise fpm uses // the filtered package version, which would have dashes replaced with @@ -64,15 +66,19 @@ export default (grunt) => { `${servicesByName.systemd.outputDir}/etc/=/etc/` ]; + const options = { + cwd: grunt.config.get('root') + }; + //Manually find flags, multiple args without assignment are not entirely parsed const flags = grunt.option.flags().filter(flag => /deb|rpm/.test(flag)).join(','); const buildDeb = flags.includes('deb') || !flags.length; const buildRpm = flags.includes('rpm') || !flags.length; if (buildDeb) { - fpm([...baseOptions, ...debOptions, ...args]); + fpm([...baseOptions, ...debOptions, ...args], options); } if (buildRpm) { - fpm([...baseOptions, ...rpmOptions, ...args]); + fpm([...baseOptions, ...rpmOptions, ...args], options); } }); diff --git a/tasks/build/shasums.js b/tasks/build/shasums.js index 7506e67ffb6a4..03f5c6d9a35af 100644 --- a/tasks/build/shasums.js +++ b/tasks/build/shasums.js @@ -1,8 +1,6 @@ import { promisify } from 'bluebird'; const readdir = promisify(require('fs').readdir); const exec = promisify(require('child_process').exec); -const platform = require('os').platform(); -const cmd = /^win/.test(platform) ? 'sha1sum ' : 'shasum '; module.exports = function (grunt) { grunt.registerTask('_build:shasums', function () { @@ -16,7 +14,7 @@ module.exports = function (grunt) { // only sha the archives and packages if (!archive.match(/\.zip$|\.tar.gz$|\.deb$|\.rpm$/)) return; - return exec(cmd + archive + ' > ' + archive + '.sha1.txt', { + return exec(`shasum ${archive} > ${archive}.sha1.txt`, { cwd: targetDir }); }) diff --git a/test/scripts/jenkins_selenium.sh b/test/scripts/jenkins_selenium.sh index d035147563a89..db42225ed91fb 100755 --- a/test/scripts/jenkins_selenium.sh +++ b/test/scripts/jenkins_selenium.sh @@ -3,6 +3,6 @@ set -e source "$(dirname $0)/_jenkins_setup.sh" -"$(npm bin)/grunt" build --release; +"$(npm bin)/grunt" docker:packages --release; xvfb-run "$(npm bin)/grunt" jenkins:selenium;