Skip to content

Commit

Permalink
Merge branch 'master' into bad-rest-requests
Browse files Browse the repository at this point in the history 
* master:
  Do not optimize append-only if seen normal op with higher seqno (elastic#28787)
  [test] packaging: gradle tasks for groovy tests (elastic#29046)
  Prune only gc deletes below local checkpoint (elastic#28790)
  • Loading branch information
@jasontedor
jasontedor committed Mar 27, 2018
2 parents 696c00b + 0ac89a3 commit 17fbfc5
Show file tree
Hide file tree
Showing 12 changed files with 456 additions and 138 deletions.
6 changes: 3 additions & 3 deletions TESTING.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -414,16 +414,16 @@ and in another window:

----------------------------------------------------
vagrant up centos-7 --provider virtualbox && vagrant ssh centos-7
cd $BATS_ARCHIVES
cd $PACKAGING_ARCHIVES
sudo -E bats $BATS_TESTS/*rpm*.bats
----------------------------------------------------

If you wanted to retest all the release artifacts on a single VM you could:

-------------------------------------------------
./gradlew setupBats
./gradlew setupPackagingTest
cd qa/vagrant; vagrant up ubuntu-1404 --provider virtualbox && vagrant ssh ubuntu-1404
cd $BATS_ARCHIVES
cd $PACKAGING_ARCHIVES
sudo -E bats $BATS_TESTS/*.bats
-------------------------------------------------

Expand Down
8 changes: 4 additions & 4 deletions Vagrantfile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -334,9 +334,9 @@ export TAR=/elasticsearch/distribution/tar/build/distributions
export RPM=/elasticsearch/distribution/rpm/build/distributions
export DEB=/elasticsearch/distribution/deb/build/distributions
export BATS=/project/build/bats
export BATS_UTILS=/project/build/bats/utils
export BATS_TESTS=/project/build/bats/tests
export BATS_ARCHIVES=/project/build/bats/archives
export BATS_UTILS=/project/build/packaging/bats/utils
export BATS_TESTS=/project/build/packaging/bats/tests
export PACKAGING_ARCHIVES=/project/build/packaging/archives
VARS
cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars
Defaults env_keep += "ZIP"
Expand All @@ -346,7 +346,7 @@ Defaults env_keep += "DEB"
Defaults env_keep += "BATS"
Defaults env_keep += "BATS_UTILS"
Defaults env_keep += "BATS_TESTS"
Defaults env_keep += "BATS_ARCHIVES"
Defaults env_keep += "PACKAGING_ARCHIVES"
SUDOERS_VARS
chmod 0440 /etc/sudoers.d/elasticsearch_vars
SHELL
Expand Down
7 changes: 0 additions & 7 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,9 +37,6 @@ class VagrantPropertiesExtension {
@Input
Boolean inheritTests

@Input
Boolean inheritTestArchives

@Input
Boolean inheritTestUtils

Expand All @@ -60,10 +57,6 @@ class VagrantPropertiesExtension {
this.inheritTests = inheritTests
}

void setInheritTestArchives(Boolean inheritTestArchives) {
this.inheritTestArchives = inheritTestArchives
}

void setInheritTestUtils(Boolean inheritTestUtils) {
this.inheritTestUtils = inheritTestUtils
}
Expand Down
166 changes: 86 additions & 80 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
package org.elasticsearch.gradle.vagrant

import com.carrotsearch.gradle.junit4.RandomizedTestingPlugin
import org.apache.tools.ant.taskdefs.condition.Os
import org.elasticsearch.gradle.FileContentsTask
import org.elasticsearch.gradle.LoggedExec
Expand Down Expand Up @@ -43,8 +42,9 @@ class VagrantTestPlugin implements Plugin<Project> {
/** Packages onboarded for upgrade tests **/
static List<String> UPGRADE_FROM_ARCHIVES = ['rpm', 'deb']

private static final PACKAGING_CONFIGURATION = 'packaging'
private static final BATS = 'bats'
private static final String BATS_TEST_COMMAND ="cd \$BATS_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS"
private static final String BATS_TEST_COMMAND ="cd \$PACKAGING_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS"
private static final String PLATFORM_TEST_COMMAND ="rm -rf ~/elasticsearch && rsync -r /elasticsearch/ ~/elasticsearch && cd ~/elasticsearch && ./gradlew test integTest"

@Override
Expand All @@ -53,11 +53,11 @@ class VagrantTestPlugin implements Plugin<Project> {
// Creates the Vagrant extension for the project
project.extensions.create('esvagrant', VagrantPropertiesExtension, listVagrantBoxes(project))

// Add required repositories for Bats tests
configureBatsRepositories(project)
// Add required repositories for packaging tests
configurePackagingArchiveRepositories(project)

// Creates custom configurations for Bats testing files (and associated scripts and archives)
createBatsConfiguration(project)
createPackagingConfiguration(project)

// Creates all the main Vagrant tasks
createVagrantTasks(project)
Expand Down Expand Up @@ -87,7 +87,7 @@ class VagrantTestPlugin implements Plugin<Project> {
}
}

private static void configureBatsRepositories(Project project) {
private static void configurePackagingArchiveRepositories(Project project) {
RepositoryHandler repos = project.repositories

// Try maven central first, it'll have releases before 5.0.0
Expand All @@ -102,10 +102,10 @@ class VagrantTestPlugin implements Plugin<Project> {
}
}

private static void createBatsConfiguration(Project project) {
project.configurations.create(BATS)
private static void createPackagingConfiguration(Project project) {
project.configurations.create(PACKAGING_CONFIGURATION)

String upgradeFromVersion = System.getProperty("tests.packaging.upgradeVersion");
String upgradeFromVersion = System.getProperty("tests.packaging.upgradeVersion")
if (upgradeFromVersion == null) {
String firstPartOfSeed = project.rootProject.testSeed.tokenize(':').get(0)
final long seed = Long.parseUnsignedLong(firstPartOfSeed, 16)
Expand All @@ -120,12 +120,14 @@ class VagrantTestPlugin implements Plugin<Project> {
} else {
it = "packages:${it}"
}
project.dependencies.add(BATS, project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
project.dependencies.add(PACKAGING_CONFIGURATION,
project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
}

UPGRADE_FROM_ARCHIVES.each {
// The version of elasticsearch that we upgrade *from*
project.dependencies.add(BATS, "org.elasticsearch.distribution.${it}:elasticsearch:${upgradeFromVersion}@${it}")
project.dependencies.add(PACKAGING_CONFIGURATION,
"org.elasticsearch.distribution.${it}:elasticsearch:${upgradeFromVersion}@${it}")
}

project.extensions.esvagrant.upgradeFromVersion = upgradeFromVersion
Expand Down Expand Up @@ -154,73 +156,66 @@ class VagrantTestPlugin implements Plugin<Project> {
}

private static void createPrepareVagrantTestEnvTask(Project project) {
File batsDir = new File("${project.buildDir}/${BATS}")
File packagingDir = new File(project.buildDir, PACKAGING_CONFIGURATION)

Task createBatsDirsTask = project.tasks.create('createBatsDirs')
createBatsDirsTask.outputs.dir batsDir
createBatsDirsTask.doLast {
batsDir.mkdirs()
File archivesDir = new File(packagingDir, 'archives')
Copy copyPackagingArchives = project.tasks.create('copyPackagingArchives', Copy) {
into archivesDir
from project.configurations[PACKAGING_CONFIGURATION]
}

Copy copyBatsArchives = project.tasks.create('copyBatsArchives', Copy) {
dependsOn createBatsDirsTask
into "${batsDir}/archives"
from project.configurations[BATS]
Task createVersionFile = project.tasks.create('createVersionFile', FileContentsTask) {
dependsOn copyPackagingArchives
file "${archivesDir}/version"
contents project.version
}

Task createUpgradeFromFile = project.tasks.create('createUpgradeFromFile', FileContentsTask) {
dependsOn copyPackagingArchives
file "${archivesDir}/upgrade_from_version"
contents project.extensions.esvagrant.upgradeFromVersion
}

File batsDir = new File(packagingDir, BATS)
Copy copyBatsTests = project.tasks.create('copyBatsTests', Copy) {
dependsOn createBatsDirsTask
into "${batsDir}/tests"
from {
"${project.extensions.esvagrant.batsDir}/tests"
}
}

Copy copyBatsUtils = project.tasks.create('copyBatsUtils', Copy) {
dependsOn createBatsDirsTask
into "${batsDir}/utils"
from {
"${project.extensions.esvagrant.batsDir}/utils"
}
}

// Now we iterate over dependencies of the bats configuration. When a project dependency is found,
// we bring back its own archives, test files or test utils.
// we bring back its test files or test utils.
project.afterEvaluate {
project.configurations.bats.dependencies.findAll {it.targetConfiguration == BATS }.each { d ->
if (d instanceof DefaultProjectDependency) {
DefaultProjectDependency externalBatsDependency = (DefaultProjectDependency) d
Project externalBatsProject = externalBatsDependency.dependencyProject
String externalBatsDir = externalBatsProject.extensions.esvagrant.batsDir

if (project.extensions.esvagrant.inheritTests) {
copyBatsTests.from(externalBatsProject.files("${externalBatsDir}/tests"))
}
if (project.extensions.esvagrant.inheritTestArchives) {
copyBatsArchives.from(externalBatsDependency.projectConfiguration.files)
}
if (project.extensions.esvagrant.inheritTestUtils) {
copyBatsUtils.from(externalBatsProject.files("${externalBatsDir}/utils"))
project.configurations[PACKAGING_CONFIGURATION].dependencies
.findAll {it.targetConfiguration == PACKAGING_CONFIGURATION }
.each { d ->
if (d instanceof DefaultProjectDependency) {
DefaultProjectDependency externalBatsDependency = (DefaultProjectDependency) d
Project externalBatsProject = externalBatsDependency.dependencyProject
String externalBatsDir = externalBatsProject.extensions.esvagrant.batsDir

if (project.extensions.esvagrant.inheritTests) {
copyBatsTests.from(externalBatsProject.files("${externalBatsDir}/tests"))
}
if (project.extensions.esvagrant.inheritTestUtils) {
copyBatsUtils.from(externalBatsProject.files("${externalBatsDir}/utils"))
}
}
}
}
}

Task createVersionFile = project.tasks.create('createVersionFile', FileContentsTask) {
dependsOn createBatsDirsTask
file "${batsDir}/archives/version"
contents project.version
}

Task createUpgradeFromFile = project.tasks.create('createUpgradeFromFile', FileContentsTask) {
dependsOn createBatsDirsTask
file "${batsDir}/archives/upgrade_from_version"
contents project.extensions.esvagrant.upgradeFromVersion
}

Task vagrantSetUpTask = project.tasks.create('setupBats')
Task vagrantSetUpTask = project.tasks.create('setupPackagingTest')
vagrantSetUpTask.dependsOn 'vagrantCheckVersion'
vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils, copyBatsArchives, createVersionFile, createUpgradeFromFile
vagrantSetUpTask.dependsOn copyPackagingArchives, createVersionFile, createUpgradeFromFile
vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils
}

private static void createPackagingTestTask(Project project) {
Expand Down Expand Up @@ -270,8 +265,8 @@ class VagrantTestPlugin implements Plugin<Project> {
assert project.tasks.virtualboxCheckVersion != null
Task virtualboxCheckVersion = project.tasks.virtualboxCheckVersion

assert project.tasks.setupBats != null
Task setupBats = project.tasks.setupBats
assert project.tasks.setupPackagingTest != null
Task setupPackagingTest = project.tasks.setupPackagingTest

assert project.tasks.packagingTest != null
Task packagingTest = project.tasks.packagingTest
Expand Down Expand Up @@ -308,7 +303,7 @@ class VagrantTestPlugin implements Plugin<Project> {
environmentVars vagrantEnvVars
dependsOn vagrantCheckVersion, virtualboxCheckVersion
}
update.mustRunAfter(setupBats)
update.mustRunAfter(setupPackagingTest)

/*
* Destroying before every execution can be annoying while iterating on tests locally. Therefore, we provide a flag
Expand Down Expand Up @@ -359,32 +354,39 @@ class VagrantTestPlugin implements Plugin<Project> {
}
vagrantSmokeTest.dependsOn(smoke)

Task packaging = project.tasks.create("vagrant${boxTask}#packagingTest", BatsOverVagrantTask) {
Task batsPackagingTest = project.tasks.create("vagrant${boxTask}#batsPackagingTest", BatsOverVagrantTask) {
remoteCommand BATS_TEST_COMMAND
boxName box
environmentVars vagrantEnvVars
dependsOn up, setupBats
dependsOn up, setupPackagingTest
finalizedBy halt
}

TaskExecutionAdapter packagingReproListener = new TaskExecutionAdapter() {
@Override
void afterExecute(Task task, TaskState state) {
final String gradlew = Os.isFamily(Os.FAMILY_WINDOWS) ? "gradlew" : "./gradlew"
if (state.failure != null) {
println "REPRODUCE WITH: ${gradlew} ${packaging.path} " +
"-Dtests.seed=${project.testSeed} "
}
}
TaskExecutionAdapter batsPackagingReproListener = createReproListener(project, batsPackagingTest.path)
batsPackagingTest.doFirst {
project.gradle.addListener(batsPackagingReproListener)
}
batsPackagingTest.doLast {
project.gradle.removeListener(batsPackagingReproListener)
}
packaging.doFirst {
project.gradle.addListener(packagingReproListener)
if (project.extensions.esvagrant.boxes.contains(box)) {
packagingTest.dependsOn(batsPackagingTest)
}

// This task doesn't do anything yet. In the future it will execute a jar containing tests on the vm
Task groovyPackagingTest = project.tasks.create("vagrant${boxTask}#groovyPackagingTest")
groovyPackagingTest.dependsOn(up)
groovyPackagingTest.finalizedBy(halt)

TaskExecutionAdapter groovyPackagingReproListener = createReproListener(project, groovyPackagingTest.path)
groovyPackagingTest.doFirst {
project.gradle.addListener(groovyPackagingReproListener)
}
packaging.doLast {
project.gradle.removeListener(packagingReproListener)
groovyPackagingTest.doLast {
project.gradle.removeListener(groovyPackagingReproListener)
}
if (project.extensions.esvagrant.boxes.contains(box)) {
packagingTest.dependsOn(packaging)
packagingTest.dependsOn(groovyPackagingTest)
}

Task platform = project.tasks.create("vagrant${boxTask}#platformTest", VagrantCommandTask) {
Expand All @@ -395,15 +397,7 @@ class VagrantTestPlugin implements Plugin<Project> {
finalizedBy halt
args '--command', PLATFORM_TEST_COMMAND + " -Dtests.seed=${-> project.testSeed}"
}
TaskExecutionAdapter platformReproListener = new TaskExecutionAdapter() {
@Override
void afterExecute(Task task, TaskState state) {
if (state.failure != null) {
println "REPRODUCE WITH: gradle ${platform.path} " +
"-Dtests.seed=${project.testSeed} "
}
}
}
TaskExecutionAdapter platformReproListener = createReproListener(project, platform.path)
platform.doFirst {
project.gradle.addListener(platformReproListener)
}
Expand All @@ -415,4 +409,16 @@ class VagrantTestPlugin implements Plugin<Project> {
}
}
}

private static TaskExecutionAdapter createReproListener(Project project, String reproTaskPath) {
return new TaskExecutionAdapter() {
@Override
void afterExecute(Task task, TaskState state) {
final String gradlew = Os.isFamily(Os.FAMILY_WINDOWS) ? "gradlew" : "./gradlew"
if (state.failure != null) {
println "REPRODUCE WITH: ${gradlew} ${reproTaskPath} -Dtests.seed=${project.testSeed} "
}
}
}
}
}
4 changes: 2 additions & 2 deletions qa/vagrant/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,14 +25,14 @@ for (Project subproj : project.rootProject.subprojects) {
if (subproj.path.startsWith(':plugins:') || subproj.path.equals(':example-plugins:custom-settings')) {
// add plugin as a dep
dependencies {
bats project(path: "${subproj.path}", configuration: 'zip')
packaging project(path: "${subproj.path}", configuration: 'zip')
}
plugins.add(subproj.name)
}
}
plugins = plugins.toSorted()

setupBats {
setupPackagingTest {
doFirst {
File expectedPlugins = file('build/plugins/expected')
expectedPlugins.parentFile.mkdirs()
Expand Down
Loading

0 comments on commit 17fbfc5

Please sign in to comment.