From 53c38cc8febba2860694716991710f951c49e253 Mon Sep 17 00:00:00 2001 From: Ryan Ernst Date: Tue, 13 Feb 2018 22:49:53 -0800 Subject: [PATCH] Build: Group archive and package distribution projects (#28673) This commit adds intermediate gradle projects for archive based distributions (zip, tar) and package based distributions (rpm, deb). The grouping allows the common distribution build file to be considerably shorter and clearly separated from the common zip/tar and rpm/deb configuration. --- build.gradle | 14 +- .../gradle/vagrant/VagrantTestPlugin.groovy | 5 + distribution/archives/build.gradle | 86 ++++++ .../integ-test-zip/build.gradle | 0 .../test/rest/CreatedLocationHeaderIT.java | 0 .../IntegTestZipClientYamlTestSuiteIT.java | 0 .../test/rest/NodeRestUsageIT.java | 0 .../test/rest/RequestsWithoutContentIT.java | 0 .../rest/WaitForRefreshAndCloseTests.java | 0 distribution/{ => archives}/tar/build.gradle | 0 .../test/rest/TarClientYamlTestSuiteIT.java | 0 .../test/smoke_test_plugins/10_modules.yml | 0 distribution/{ => archives}/zip/build.gradle | 0 .../test/rest/ZipClientYamlTestSuiteIT.java | 0 .../test/smoke_test_plugins/10_modules.yml | 0 distribution/build.gradle | 278 +----------------- distribution/bwc/build.gradle | 16 +- distribution/packages/build.gradle | 239 +++++++++++++++ distribution/{ => packages}/deb/build.gradle | 0 .../deb/src/main/packaging/copyright | 0 .../src/main/packaging/init.d/elasticsearch | 0 .../src/main/packaging/lintian/elasticsearch | 0 distribution/{ => packages}/rpm/build.gradle | 0 .../src/main/packaging/init.d/elasticsearch | 0 .../rpm/src/main/resources/logo/elastic.gif | Bin settings.gradle | 12 +- 26 files changed, 360 insertions(+), 290 deletions(-) create mode 100644 distribution/archives/build.gradle rename distribution/{ => archives}/integ-test-zip/build.gradle (100%) rename distribution/{ => archives}/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java (100%) rename distribution/{ => archives}/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java (100%) rename distribution/{ => archives}/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java (100%) rename distribution/{ => archives}/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java (100%) rename distribution/{ => archives}/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java (100%) rename distribution/{ => archives}/tar/build.gradle (100%) rename distribution/{ => archives}/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java (100%) rename distribution/{ => archives}/tar/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml (100%) rename distribution/{ => archives}/zip/build.gradle (100%) rename distribution/{ => archives}/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java (100%) rename distribution/{ => archives}/zip/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml (100%) create mode 100644 distribution/packages/build.gradle rename distribution/{ => packages}/deb/build.gradle (100%) rename distribution/{ => packages}/deb/src/main/packaging/copyright (100%) rename distribution/{ => packages}/deb/src/main/packaging/init.d/elasticsearch (100%) rename distribution/{ => packages}/deb/src/main/packaging/lintian/elasticsearch (100%) rename distribution/{ => packages}/rpm/build.gradle (100%) rename distribution/{ => packages}/rpm/src/main/packaging/init.d/elasticsearch (100%) rename distribution/{ => packages}/rpm/src/main/resources/logo/elastic.gif (100%) diff --git a/build.gradle b/build.gradle index ab5da713d8cac..c80df706f11ad 100644 --- a/build.gradle +++ b/build.gradle @@ -144,8 +144,8 @@ task verifyVersions { * the enabled state of every bwc task. It should be set back to true * after the backport of the backcompat code is complete. */ -final boolean bwc_tests_enabled = true -final String bwc_tests_disabled_issue = "" /* place a PR link here when commiting bwc changes */ +final boolean bwc_tests_enabled = false +final String bwc_tests_disabled_issue = "https://github.com/elastic/elasticsearch/pull/28673" /* place a PR link here when commiting bwc changes */ if (bwc_tests_enabled == false) { if (bwc_tests_disabled_issue.isEmpty()) { throw new GradleException("bwc_tests_disabled_issue must be set when bwc_tests_enabled == false") @@ -203,11 +203,11 @@ subprojects { "org.elasticsearch.client:test:${version}": ':client:test', "org.elasticsearch.client:transport:${version}": ':client:transport', "org.elasticsearch.test:framework:${version}": ':test:framework', - "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:integ-test-zip', - "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:zip', - "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:tar', - "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:rpm', - "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:deb', + "org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip', + "org.elasticsearch.distribution.zip:elasticsearch:${version}": ':distribution:archives:zip', + "org.elasticsearch.distribution.tar:elasticsearch:${version}": ':distribution:archives:tar', + "org.elasticsearch.distribution.rpm:elasticsearch:${version}": ':distribution:packages:rpm', + "org.elasticsearch.distribution.deb:elasticsearch:${version}": ':distribution:packages:deb', "org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage', // for transport client "org.elasticsearch.plugin:transport-netty4-client:${version}": ':modules:transport-netty4', diff --git a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy index 669291884f40b..2d5f985a2c118 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy @@ -114,6 +114,11 @@ class VagrantTestPlugin implements Plugin { DISTRIBUTION_ARCHIVES.each { // Adds a dependency for the current version + if (it == 'tar') { + it = 'archives:tar' + } else { + it = "packages:${it}" + } project.dependencies.add(BATS, project.dependencies.project(path: ":distribution:${it}", configuration: 'archives')) } diff --git a/distribution/archives/build.gradle b/distribution/archives/build.gradle new file mode 100644 index 0000000000000..8bf31afce1b6f --- /dev/null +++ b/distribution/archives/build.gradle @@ -0,0 +1,86 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.tools.ant.filters.FixCrLfFilter +import org.apache.tools.ant.taskdefs.condition.Os +import org.elasticsearch.gradle.BuildPlugin +import org.elasticsearch.gradle.EmptyDirTask +import org.elasticsearch.gradle.ConcatFilesTask +import org.elasticsearch.gradle.MavenFilteringHack +import org.elasticsearch.gradle.NoticeTask +import org.elasticsearch.gradle.precommit.DependencyLicensesTask +import org.elasticsearch.gradle.precommit.UpdateShasTask +import org.elasticsearch.gradle.test.RunTask + +subprojects { + // CopySpec does not make it easy to create an empty director so we create the directory that we want, and then point CopySpec to its + // parent to copy to the root of the distribution + File logs = new File(buildDir, 'logs-hack/logs') + task createLogDir(type: EmptyDirTask) { + dir "${logs}" + dirMode 0755 + } + File plugins = new File(buildDir, 'plugins-hack/plugins') + task createPluginsDir(type: EmptyDirTask) { + dir "${plugins}" + dirMode 0755 + } + project.ext.archivesFiles = copySpec { + into("elasticsearch-${version}") { + with libFiles + into('config') { + dirMode 0750 + fileMode 0660 + with configFiles + } + into('bin') { + with copySpec { + with binFiles + from('../../src/main/resources/bin') { + include '*.bat' + filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf')) + } + MavenFilteringHack.filter(it, expansions) + } + } + into('') { + from { + dirMode 0755 + logs.getParent() + } + } + into('') { + from { + dirMode 0755 + plugins.getParent() + } + } + with commonFiles + with noticeFile + from('../../src/main/resources') { + include 'bin/*.exe' + } + if (project.name != 'integ-test-zip') { + with modulesFiles + } else { + with transportModulesFiles + } + } + } +} diff --git a/distribution/integ-test-zip/build.gradle b/distribution/archives/integ-test-zip/build.gradle similarity index 100% rename from distribution/integ-test-zip/build.gradle rename to distribution/archives/integ-test-zip/build.gradle diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java similarity index 100% rename from distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java rename to distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/CreatedLocationHeaderIT.java diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java similarity index 100% rename from distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java rename to distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/IntegTestZipClientYamlTestSuiteIT.java diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java similarity index 100% rename from distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java rename to distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/NodeRestUsageIT.java diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java similarity index 100% rename from distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java rename to distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/RequestsWithoutContentIT.java diff --git a/distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java b/distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java similarity index 100% rename from distribution/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java rename to distribution/archives/integ-test-zip/src/test/java/org/elasticsearch/test/rest/WaitForRefreshAndCloseTests.java diff --git a/distribution/tar/build.gradle b/distribution/archives/tar/build.gradle similarity index 100% rename from distribution/tar/build.gradle rename to distribution/archives/tar/build.gradle diff --git a/distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java b/distribution/archives/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java similarity index 100% rename from distribution/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java rename to distribution/archives/tar/src/test/java/org/elasticsearch/test/rest/TarClientYamlTestSuiteIT.java diff --git a/distribution/tar/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml b/distribution/archives/tar/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml similarity index 100% rename from distribution/tar/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml rename to distribution/archives/tar/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml diff --git a/distribution/zip/build.gradle b/distribution/archives/zip/build.gradle similarity index 100% rename from distribution/zip/build.gradle rename to distribution/archives/zip/build.gradle diff --git a/distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java b/distribution/archives/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java similarity index 100% rename from distribution/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java rename to distribution/archives/zip/src/test/java/org/elasticsearch/test/rest/ZipClientYamlTestSuiteIT.java diff --git a/distribution/zip/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml b/distribution/archives/zip/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml similarity index 100% rename from distribution/zip/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml rename to distribution/archives/zip/src/test/resources/rest-api-spec/test/smoke_test_plugins/10_modules.yml diff --git a/distribution/build.gradle b/distribution/build.gradle index d322aa9c1ff12..c47da920ecf4d 100644 --- a/distribution/build.gradle +++ b/distribution/build.gradle @@ -29,20 +29,7 @@ import org.elasticsearch.gradle.precommit.DependencyLicensesTask import org.elasticsearch.gradle.precommit.UpdateShasTask import org.elasticsearch.gradle.test.RunTask -// for deb/rpm -buildscript { - repositories { - maven { - url "https://plugins.gradle.org/m2/" - } - } - dependencies { - classpath 'com.netflix.nebula:gradle-ospackage-plugin:3.4.0' - } -} - -Collection distributions = project.subprojects.findAll { - it.path.contains(':tools') == false && it.path.contains(':bwc') == false } +Collection distributions = project('archives').subprojects + project('packages').subprojects /***************************************************************************** * Third party dependencies report * @@ -106,7 +93,7 @@ project.rootProject.subprojects.findAll { it.parent.path == ':modules' }.each { } // also want to make sure the module's integration tests run after the integ-test-zip (ie rest tests) module.afterEvaluate({ - module.integTest.mustRunAfter(':distribution:integ-test-zip:integTest') + module.integTest.mustRunAfter(':distribution:archives:integ-test-zip:integTest') }) restTestExpansions['expected.modules.count'] += 1 } @@ -141,7 +128,7 @@ configure(distributions) { project.integTest { includePackaged project.name == 'integ-test-zip' if (project.name != 'integ-test-zip') { - mustRunAfter ':distribution:integ-test-zip:integTest' + mustRunAfter ':distribution:archives:integ-test-zip:integTest' } } project.integTestCluster { @@ -192,13 +179,13 @@ configure(distributions) { } configFiles = copySpec { - from '../src/main/resources/config' + from '../../src/main/resources/config' MavenFilteringHack.filter(it, expansions) } binFiles = copySpec { // everything except windows files - from '../src/main/resources/bin' + from '../../src/main/resources/bin' exclude '*.bat' exclude '*.exe' eachFile { it.setMode(0755) } @@ -231,261 +218,6 @@ configure(distributions) { } } -/***************************************************************************** - * Zip and tgz configuration * - *****************************************************************************/ -configure(distributions.findAll { ['zip', 'tar', 'integ-test-zip'].contains(it.name) }) { - // CopySpec does not make it easy to create an empty director so we create the directory that we want, and then point CopySpec to its - // parent to copy to the root of the distribution - File logs = new File(buildDir, 'logs-hack/logs') - task createLogDir(type: EmptyDirTask) { - dir "${logs}" - dirMode 0755 - } - File plugins = new File(buildDir, 'plugins-hack/plugins') - task createPluginsDir(type: EmptyDirTask) { - dir "${plugins}" - dirMode 0755 - } - project.ext.archivesFiles = copySpec { - into("elasticsearch-${version}") { - with libFiles - into('config') { - dirMode 0750 - fileMode 0660 - with configFiles - } - into('bin') { - with copySpec { - with binFiles - from('../src/main/resources/bin') { - include '*.bat' - filter(FixCrLfFilter, eol: FixCrLfFilter.CrLf.newInstance('crlf')) - } - MavenFilteringHack.filter(it, expansions) - } - } - into('') { - from { - dirMode 0755 - logs.getParent() - } - } - into('') { - from { - dirMode 0755 - plugins.getParent() - } - } - with commonFiles - with noticeFile - from('../src/main/resources') { - include 'bin/*.exe' - } - if (project.name != 'integ-test-zip') { - with modulesFiles - } else { - with transportModulesFiles - } - } - } -} - -/***************************************************************************** - * Deb and rpm configuration * - ***************************************************************************** - * - * The general strategy here is to build a directory on disk, packagingFiles - * that contains stuff that needs to be copied into the distributions. This is - * important for two reasons: - * 1. ospackage wants to copy the directory permissions that it sees off of the - * filesystem. If you ask it to create a directory that doesn't already - * exist on disk it petulantly creates it with 0755 permissions, no matter - * how hard you try to convince it otherwise. - * 2. Convincing ospackage to pick up an empty directory as part of a set of - * directories on disk is reasonably easy. Convincing it to just create an - * empty directory requires more wits than I have. - * 3. ospackage really wants to suck up some of the debian control scripts - * directly from the filesystem. It doesn't want to process them through - * MavenFilteringHack or any other copy-style action. - * - * The following commands are useful when it comes to check the user/group - * and files permissions set within the RPM and DEB packages: - * - * rpm -qlp --dump path/to/elasticsearch.rpm - * dpkg -c path/to/elasticsearch.deb - */ -configure(distributions.findAll { ['deb', 'rpm'].contains(it.name) }) { - integTest.enabled = Os.isFamily(Os.FAMILY_WINDOWS) == false - File packagingFiles = new File(buildDir, 'packaging') - project.ext.packagingFiles = packagingFiles - task processPackagingFiles(type: Copy) { - from '../src/main/packaging' - from 'src/main/packaging' - - MavenFilteringHack.filter(it, expansions) - into packagingFiles - /* Explicitly declare the outputs so that gradle won't skip this task if - one of the other tasks like createEtc run first and create the packaging - directory as a side effect. */ - outputs.dir("${packagingFiles}/env") - outputs.dir("${packagingFiles}/systemd") - } - - task createEtc(type: EmptyDirTask) { - dir "${packagingFiles}/etc/elasticsearch" - dirMode 0750 - outputs.dir dir - } - - task fillEtc(type: Copy) { - dependsOn createEtc - with configFiles - into "${packagingFiles}/etc/elasticsearch" - /* Explicitly declare the output files so this task doesn't consider itself - up to date when the directory is created, which it would by default. And - that'll happen when createEtc runs. */ - outputs.file "${packagingFiles}/etc/elasticsearch/elasticsearch.yml" - outputs.file "${packagingFiles}/etc/elasticsearch/jvm.options" - outputs.file "${packagingFiles}/etc/elasticsearch/log4j2.properties" - } - - task createPidDir(type: EmptyDirTask) { - dir "${packagingFiles}/var/run/elasticsearch" - } - task createLogDir(type: EmptyDirTask) { - dir "${packagingFiles}/var/log/elasticsearch" - } - task createDataDir(type: EmptyDirTask) { - dir "${packagingFiles}/var/lib/elasticsearch" - } - task createPluginsDir(type: EmptyDirTask) { - dir "${packagingFiles}/usr/share/elasticsearch/plugins" - } - - /** - * Setup the build/packaging directory to be like the target filesystem - * because ospackage will use arbitrary permissions if you try to create a - * directory that doesn't exist on the filesystem. - */ - task preparePackagingFiles { - dependsOn processPackagingFiles, fillEtc, createPidDir, createLogDir, - createDataDir, createPluginsDir - } - - apply plugin: 'nebula.ospackage-base' - ospackage { - packageName 'elasticsearch' - maintainer 'Elasticsearch Team ' - summary ''' - Elasticsearch is a distributed RESTful search engine built for the cloud. - Reference documentation can be found at - https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html - and the 'Elasticsearch: The Definitive Guide' book can be found at - https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html - '''.stripIndent().replace('\n', ' ').trim() - url 'https://www.elastic.co/' - - // signing setup - if (project.hasProperty('signing.password') && System.getProperty('build.snapshot', 'true') == 'false') { - signingKeyId = project.hasProperty('signing.keyId') ? project.property('signing.keyId') : 'D88E42B4' - signingKeyPassphrase = project.property('signing.password') - signingKeyRingFile = project.hasProperty('signing.secretKeyRingFile') ? - project.file(project.property('signing.secretKeyRingFile')) : - new File(new File(System.getProperty('user.home'), '.gnupg'), 'secring.gpg') - } - - String scripts = "${packagingFiles}/scripts" - preInstall file("${scripts}/preinst") - postInstall file("${scripts}/postinst") - preUninstall file("${scripts}/prerm") - postUninstall file("${scripts}/postrm") - - if (project.name == 'rpm') { - requires('/bin/bash') - } else if (project.name == 'deb') { - requires('bash') - } - requires('coreutils') - - into '/usr/share/elasticsearch' - fileMode 0644 - dirMode 0755 - user 'root' - permissionGroup 'root' - with libFiles - with modulesFiles - into('bin') { - with binFiles - } - with copySpec { - with commonFiles - if (project.name == 'deb') { - // Deb gets a copyright file instead. - exclude 'LICENSE.txt' - } - } - with noticeFile - - configurationFile '/etc/elasticsearch/elasticsearch.yml' - configurationFile '/etc/elasticsearch/jvm.options' - configurationFile '/etc/elasticsearch/log4j2.properties' - into('/etc/elasticsearch') { - dirMode 0750 - fileMode 0660 - permissionGroup 'elasticsearch' - includeEmptyDirs true - createDirectoryEntry true - fileType CONFIG | NOREPLACE - from "${packagingFiles}/etc/elasticsearch" - } - - into('/usr/lib/tmpfiles.d') { - from "${packagingFiles}/systemd/elasticsearch.conf" - } - configurationFile '/usr/lib/systemd/system/elasticsearch.service' - into('/usr/lib/systemd/system') { - fileType CONFIG | NOREPLACE - from "${packagingFiles}/systemd/elasticsearch.service" - } - into('/usr/lib/sysctl.d') { - fileType CONFIG | NOREPLACE - from "${packagingFiles}/systemd/sysctl/elasticsearch.conf" - } - configurationFile '/etc/init.d/elasticsearch' - into('/etc/init.d') { - fileMode 0750 - fileType CONFIG | NOREPLACE - from "${packagingFiles}/init.d/elasticsearch" - } - configurationFile project.expansions['path.env'] - into(new File(project.expansions['path.env']).getParent()) { - fileType CONFIG | NOREPLACE - fileMode 0660 - from "${project.packagingFiles}/env/elasticsearch" - } - - /** - * Suck up all the empty directories that we need to install into the path. - */ - Closure suckUpEmptyDirectories = { path, u, g, mode -> - into(path) { - from "${packagingFiles}/${path}" - includeEmptyDirs true - createDirectoryEntry true - user u - permissionGroup g - dirMode mode - fileMode mode - } - } - suckUpEmptyDirectories('/var/run', 'elasticsearch', 'elasticsearch', 0755) - suckUpEmptyDirectories('/var/log', 'elasticsearch', 'elasticsearch', 0750) - suckUpEmptyDirectories('/var/lib', 'elasticsearch', 'elasticsearch', 0750) - suckUpEmptyDirectories('/usr/share/elasticsearch', 'root', 'root', 0755) - } -} - task run(type: RunTask) { distribution = 'zip' } diff --git a/distribution/bwc/build.gradle b/distribution/bwc/build.gradle index 21a9dbab2eb8c..840c69742a0c7 100644 --- a/distribution/bwc/build.gradle +++ b/distribution/bwc/build.gradle @@ -120,9 +120,17 @@ subprojects { } } - File bwcDeb = file("${checkoutDir}/distribution/deb/build/distributions/elasticsearch-${bwcVersion}.deb") - File bwcRpm = file("${checkoutDir}/distribution/rpm/build/distributions/elasticsearch-${bwcVersion}.rpm") - File bwcZip = file("${checkoutDir}/distribution/zip/build/distributions/elasticsearch-${bwcVersion}.zip") + String debDir = 'distribution/packages/deb' + String rpmDir = 'distribution/packages/rpm' + String zipDir = 'distribution/archives/zip' + if (bwcVersion.before('6.3.0')) { + debDir = 'distribution/deb' + rpmDir = 'distribution/rpm' + zipDir = 'distribution/zip' + } + File bwcDeb = file("${checkoutDir}/${debDir}/build/distributions/elasticsearch-${bwcVersion}.deb") + File bwcRpm = file("${checkoutDir}/${rpmDir}/build/distributions/elasticsearch-${bwcVersion}.rpm") + File bwcZip = file("${checkoutDir}/${zipDir}/build/distributions/elasticsearch-${bwcVersion}.zip") task buildBwcVersion(type: Exec) { dependsOn checkoutBwcBranch, writeBuildMetadata workingDir = checkoutDir @@ -139,7 +147,7 @@ subprojects { } else { executable new File(checkoutDir, 'gradlew').toString() } - args ":distribution:deb:assemble", ":distribution:rpm:assemble", ":distribution:zip:assemble", "-Dbuild.snapshot=true" + args ":${debDir.replace('/', ':')}:assemble", ":${rpmDir.replace('/', ':')}:assemble", ":${zipDir.replace('/', ':')}:assemble", "-Dbuild.snapshot=true" final LogLevel logLevel = gradle.startParameter.logLevel if ([LogLevel.QUIET, LogLevel.WARN, LogLevel.INFO, LogLevel.DEBUG].contains(logLevel)) { args "--${logLevel.name().toLowerCase(Locale.ENGLISH)}" diff --git a/distribution/packages/build.gradle b/distribution/packages/build.gradle new file mode 100644 index 0000000000000..6eb9716dcd260 --- /dev/null +++ b/distribution/packages/build.gradle @@ -0,0 +1,239 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import org.apache.tools.ant.filters.FixCrLfFilter +import org.apache.tools.ant.taskdefs.condition.Os +import org.elasticsearch.gradle.BuildPlugin +import org.elasticsearch.gradle.EmptyDirTask +import org.elasticsearch.gradle.ConcatFilesTask +import org.elasticsearch.gradle.MavenFilteringHack +import org.elasticsearch.gradle.NoticeTask +import org.elasticsearch.gradle.precommit.DependencyLicensesTask +import org.elasticsearch.gradle.precommit.UpdateShasTask +import org.elasticsearch.gradle.test.RunTask + +/***************************************************************************** + * Deb and rpm configuration * + ***************************************************************************** + * + * The general strategy here is to build a directory on disk, packagingFiles + * that contains stuff that needs to be copied into the distributions. This is + * important for two reasons: + * 1. ospackage wants to copy the directory permissions that it sees off of the + * filesystem. If you ask it to create a directory that doesn't already + * exist on disk it petulantly creates it with 0755 permissions, no matter + * how hard you try to convince it otherwise. + * 2. Convincing ospackage to pick up an empty directory as part of a set of + * directories on disk is reasonably easy. Convincing it to just create an + * empty directory requires more wits than I have. + * 3. ospackage really wants to suck up some of the debian control scripts + * directly from the filesystem. It doesn't want to process them through + * MavenFilteringHack or any other copy-style action. + * + * The following commands are useful when it comes to check the user/group + * and files permissions set within the RPM and DEB packages: + * + * rpm -qlp --dump path/to/elasticsearch.rpm + * dpkg -c path/to/elasticsearch.deb + */ + +// for deb/rpm +buildscript { + repositories { + maven { + url "https://plugins.gradle.org/m2/" + } + } + dependencies { + classpath 'com.netflix.nebula:gradle-ospackage-plugin:3.4.0' + } +} + +subprojects { + // TODO: remove integ test for packages + integTest.enabled = Os.isFamily(Os.FAMILY_WINDOWS) == false + File packagingFiles = new File(buildDir, 'packaging') + project.ext.packagingFiles = packagingFiles + task processPackagingFiles(type: Copy) { + from '../../src/main/packaging' + from 'src/main/packaging' + + MavenFilteringHack.filter(it, expansions) + into packagingFiles + /* Explicitly declare the outputs so that gradle won't skip this task if + one of the other tasks like createEtc run first and create the packaging + directory as a side effect. */ + outputs.dir("${packagingFiles}/env") + outputs.dir("${packagingFiles}/systemd") + } + + task createEtc(type: EmptyDirTask) { + dir "${packagingFiles}/etc/elasticsearch" + dirMode 0750 + outputs.dir dir + } + + task fillEtc(type: Copy) { + dependsOn createEtc + with configFiles + into "${packagingFiles}/etc/elasticsearch" + /* Explicitly declare the output files so this task doesn't consider itself + up to date when the directory is created, which it would by default. And + that'll happen when createEtc runs. */ + outputs.file "${packagingFiles}/etc/elasticsearch/elasticsearch.yml" + outputs.file "${packagingFiles}/etc/elasticsearch/jvm.options" + outputs.file "${packagingFiles}/etc/elasticsearch/log4j2.properties" + } + + task createPidDir(type: EmptyDirTask) { + dir "${packagingFiles}/var/run/elasticsearch" + } + task createLogDir(type: EmptyDirTask) { + dir "${packagingFiles}/var/log/elasticsearch" + } + task createDataDir(type: EmptyDirTask) { + dir "${packagingFiles}/var/lib/elasticsearch" + } + task createPluginsDir(type: EmptyDirTask) { + dir "${packagingFiles}/usr/share/elasticsearch/plugins" + } + + /** + * Setup the build/packaging directory to be like the target filesystem + * because ospackage will use arbitrary permissions if you try to create a + * directory that doesn't exist on the filesystem. + */ + task preparePackagingFiles { + dependsOn processPackagingFiles, fillEtc, createPidDir, createLogDir, + createDataDir, createPluginsDir + } + + apply plugin: 'nebula.ospackage-base' + ospackage { + packageName 'elasticsearch' + maintainer 'Elasticsearch Team ' + summary ''' + Elasticsearch is a distributed RESTful search engine built for the cloud. + Reference documentation can be found at + https://www.elastic.co/guide/en/elasticsearch/reference/current/index.html + and the 'Elasticsearch: The Definitive Guide' book can be found at + https://www.elastic.co/guide/en/elasticsearch/guide/current/index.html + '''.stripIndent().replace('\n', ' ').trim() + url 'https://www.elastic.co/' + + // signing setup + if (project.hasProperty('signing.password') && System.getProperty('build.snapshot', 'true') == 'false') { + signingKeyId = project.hasProperty('signing.keyId') ? project.property('signing.keyId') : 'D88E42B4' + signingKeyPassphrase = project.property('signing.password') + signingKeyRingFile = project.hasProperty('signing.secretKeyRingFile') ? + project.file(project.property('signing.secretKeyRingFile')) : + new File(new File(System.getProperty('user.home'), '.gnupg'), 'secring.gpg') + } + + String scripts = "${packagingFiles}/scripts" + preInstall file("${scripts}/preinst") + postInstall file("${scripts}/postinst") + preUninstall file("${scripts}/prerm") + postUninstall file("${scripts}/postrm") + + if (project.name == 'rpm') { + requires('/bin/bash') + } else if (project.name == 'deb') { + requires('bash') + } + requires('coreutils') + + into '/usr/share/elasticsearch' + fileMode 0644 + dirMode 0755 + user 'root' + permissionGroup 'root' + with libFiles + with modulesFiles + into('bin') { + with binFiles + } + with copySpec { + with commonFiles + if (project.name == 'deb') { + // Deb gets a copyright file instead. + exclude 'LICENSE.txt' + } + } + with noticeFile + + configurationFile '/etc/elasticsearch/elasticsearch.yml' + configurationFile '/etc/elasticsearch/jvm.options' + configurationFile '/etc/elasticsearch/log4j2.properties' + into('/etc/elasticsearch') { + dirMode 0750 + fileMode 0660 + permissionGroup 'elasticsearch' + includeEmptyDirs true + createDirectoryEntry true + fileType CONFIG | NOREPLACE + from "${packagingFiles}/etc/elasticsearch" + } + + into('/usr/lib/tmpfiles.d') { + from "${packagingFiles}/systemd/elasticsearch.conf" + } + configurationFile '/usr/lib/systemd/system/elasticsearch.service' + into('/usr/lib/systemd/system') { + fileType CONFIG | NOREPLACE + from "${packagingFiles}/systemd/elasticsearch.service" + } + into('/usr/lib/sysctl.d') { + fileType CONFIG | NOREPLACE + from "${packagingFiles}/systemd/sysctl/elasticsearch.conf" + } + configurationFile '/etc/init.d/elasticsearch' + into('/etc/init.d') { + fileMode 0750 + fileType CONFIG | NOREPLACE + from "${packagingFiles}/init.d/elasticsearch" + } + configurationFile project.expansions['path.env'] + into(new File(project.expansions['path.env']).getParent()) { + fileType CONFIG | NOREPLACE + fileMode 0660 + from "${project.packagingFiles}/env/elasticsearch" + } + + /** + * Suck up all the empty directories that we need to install into the path. + */ + Closure suckUpEmptyDirectories = { path, u, g, mode -> + into(path) { + from "${packagingFiles}/${path}" + includeEmptyDirs true + createDirectoryEntry true + user u + permissionGroup g + dirMode mode + fileMode mode + } + } + suckUpEmptyDirectories('/var/run', 'elasticsearch', 'elasticsearch', 0755) + suckUpEmptyDirectories('/var/log', 'elasticsearch', 'elasticsearch', 0750) + suckUpEmptyDirectories('/var/lib', 'elasticsearch', 'elasticsearch', 0750) + suckUpEmptyDirectories('/usr/share/elasticsearch', 'root', 'root', 0755) + } +} + diff --git a/distribution/deb/build.gradle b/distribution/packages/deb/build.gradle similarity index 100% rename from distribution/deb/build.gradle rename to distribution/packages/deb/build.gradle diff --git a/distribution/deb/src/main/packaging/copyright b/distribution/packages/deb/src/main/packaging/copyright similarity index 100% rename from distribution/deb/src/main/packaging/copyright rename to distribution/packages/deb/src/main/packaging/copyright diff --git a/distribution/deb/src/main/packaging/init.d/elasticsearch b/distribution/packages/deb/src/main/packaging/init.d/elasticsearch similarity index 100% rename from distribution/deb/src/main/packaging/init.d/elasticsearch rename to distribution/packages/deb/src/main/packaging/init.d/elasticsearch diff --git a/distribution/deb/src/main/packaging/lintian/elasticsearch b/distribution/packages/deb/src/main/packaging/lintian/elasticsearch similarity index 100% rename from distribution/deb/src/main/packaging/lintian/elasticsearch rename to distribution/packages/deb/src/main/packaging/lintian/elasticsearch diff --git a/distribution/rpm/build.gradle b/distribution/packages/rpm/build.gradle similarity index 100% rename from distribution/rpm/build.gradle rename to distribution/packages/rpm/build.gradle diff --git a/distribution/rpm/src/main/packaging/init.d/elasticsearch b/distribution/packages/rpm/src/main/packaging/init.d/elasticsearch similarity index 100% rename from distribution/rpm/src/main/packaging/init.d/elasticsearch rename to distribution/packages/rpm/src/main/packaging/init.d/elasticsearch diff --git a/distribution/rpm/src/main/resources/logo/elastic.gif b/distribution/packages/rpm/src/main/resources/logo/elastic.gif similarity index 100% rename from distribution/rpm/src/main/resources/logo/elastic.gif rename to distribution/packages/rpm/src/main/resources/logo/elastic.gif diff --git a/settings.gradle b/settings.gradle index 8a63d37c1057b..f083612de10be 100644 --- a/settings.gradle +++ b/settings.gradle @@ -15,11 +15,11 @@ List projects = [ 'client:client-benchmark-noop-api-plugin', 'client:benchmark', 'benchmarks', - 'distribution:integ-test-zip', - 'distribution:zip', - 'distribution:tar', - 'distribution:deb', - 'distribution:rpm', + 'distribution:archives:integ-test-zip', + 'distribution:archives:zip', + 'distribution:archives:tar', + 'distribution:packages:deb', + 'distribution:packages:rpm', 'distribution:bwc:next-minor-snapshot', 'distribution:bwc:staged-minor-snapshot', 'distribution:bwc:next-bugfix-snapshot', @@ -112,4 +112,4 @@ if (extraProjects.exists()) { for (File extraProjectDir : extraProjects.listFiles()) { addSubProjects('', extraProjectDir) } -} \ No newline at end of file +}