Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Pass itest jar path to tests through system property #1443

Merged
merged 1 commit into from
Mar 19, 2020
Merged

Pass itest jar path to tests through system property #1443

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,6 +428,9 @@ class BuildPlugin implements Plugin<Project> {

Test integrationTest = project.tasks.create('integrationTest', RestTestRunnerTask.class)
integrationTest.dependsOn(itestJar)
integrationTest.doFirst {
integrationTest.systemProperty("es.hadoop.job.jar", itestJar.getArchiveFile().get().asFile.absolutePath)
}
Comment on lines +431 to +433
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I could not get this to use a closure - It looks like Gradle was trying to serialize it, I'm assuming in order to cache the input values, but the serialization did not resolve the value into a string first.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That is correct. Gradle essentially just calls toString() on these. One thing we've done in our build is just to create an anonymous object whose toString() implementation returns the value we want.


integrationTest.testClassesDirs = project.sourceSets.itest.output.classesDirs
integrationTest.classpath = project.sourceSets.itest.runtimeClasspath
Expand Down
2 changes: 0 additions & 2 deletions mr/src/itest/java/org/elasticsearch/hadoop/ProvisionerTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,6 @@
*/
package org.elasticsearch.hadoop;

import org.elasticsearch.hadoop.HdpBootstrap;
import org.elasticsearch.hadoop.Provisioner;
import org.junit.Test;

public class ProvisionerTest {
Expand Down
31 changes: 2 additions & 29 deletions spark/core/itest/java/org/elasticsearch/spark/integration/SparkUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,44 +18,17 @@
*/
package org.elasticsearch.spark.integration;

import java.io.File;
import java.io.FileFilter;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.util.Arrays;

import org.apache.spark.SparkConf;
import org.elasticsearch.hadoop.util.Assert;
import org.elasticsearch.hadoop.Provisioner;
import org.elasticsearch.hadoop.util.ReflectionUtils;

import com.esotericsoftware.kryo.Kryo;

public abstract class SparkUtils {

public static final String[] ES_SPARK_TESTING_JAR;

static {
// init ES-Hadoop JAR
// expect the jar under build\libs
try {
File folder = new File("build" + File.separator + "libs" + File.separator).getCanonicalFile();
System.out.println(folder.getAbsolutePath());
// find proper jar
File[] files = folder.listFiles(new FileFilter() {

@Override
public boolean accept(File pathname) {
return pathname.getName().contains("-testing.jar");
}
});
Assert.isTrue(files != null && files.length == 1,
String.format("Cannot find elasticsearch spark jar (too many or no file found);%s", Arrays.toString(files)));
ES_SPARK_TESTING_JAR = new String[] { files[0].getAbsoluteFile().toURI().toString() };

} catch (IOException ex) {
throw new RuntimeException("Cannot find required files", ex);
}
}
public static final String[] ES_SPARK_TESTING_JAR = new String[] {Provisioner.ESHADOOP_TESTING_JAR};

public static Kryo sparkSerializer(SparkConf conf) throws Exception {
// reflection galore
Expand Down
21 changes: 9 additions & 12 deletions test/shared/src/main/java/org/elasticsearch/hadoop/Provisioner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,24 +37,21 @@ public abstract class Provisioner {

public static final String ESHADOOP_TESTING_JAR;
public static final String HDFS_ES_HDP_LIB = "/eshdp/libs/es-hadoop.jar";
public static final String SYS_PROP_JOB_JAR = "es.hadoop.job.jar";

static {
// init ES-Hadoop JAR
// expect the jar under build\libs
try {
File folder = new File("build" + File.separator + "libs" + File.separator).getCanonicalFile();
// find proper jar
File[] files = folder.listFiles(new FileFilter() {

@Override
public boolean accept(File pathname) {
return pathname.getName().contains("-testing");
}
});
Assert.isTrue(files != null && files.length == 1,
String.format("Cannot find elasticsearch hadoop jar (too many or no file found);%s", Arrays.toString(files)));
ESHADOOP_TESTING_JAR = files[0].getAbsoluteFile().toURI().toString();
String jobJarLocation = System.getProperty(SYS_PROP_JOB_JAR);
if (jobJarLocation == null) {
throw new RuntimeException("Cannot find elasticsearch hadoop jar. System Property [" + SYS_PROP_JOB_JAR + "] was not set.");
}

File testJar = new File(jobJarLocation).getCanonicalFile();
Assert.isTrue(testJar.exists(),
String.format("Cannot find elasticsearch hadoop jar. File not found [%s]", testJar));
ESHADOOP_TESTING_JAR = testJar.getAbsolutePath();
} catch (IOException ex) {
throw new RuntimeException("Cannot find required files", ex);
}
Expand Down