From 7f36061d51b247e923f4abbea50b1e2e7db67aa1 Mon Sep 17 00:00:00 2001 From: James Baiera Date: Thu, 19 Mar 2020 13:20:57 -0400 Subject: [PATCH] Pass itest jar path to tests through system property (#1443) --- .../hadoop/gradle/BuildPlugin.groovy | 3 ++ .../elasticsearch/hadoop/ProvisionerTest.java | 2 -- .../spark/integration/SparkUtils.java | 31 ++----------------- .../org/elasticsearch/hadoop/Provisioner.java | 21 ++++++------- 4 files changed, 14 insertions(+), 43 deletions(-) diff --git a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy index 094b71d36..4a34dcf2a 100644 --- a/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy +++ b/buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy @@ -428,6 +428,9 @@ class BuildPlugin implements Plugin { Test integrationTest = project.tasks.create('integrationTest', RestTestRunnerTask.class) integrationTest.dependsOn(itestJar) + integrationTest.doFirst { + integrationTest.systemProperty("es.hadoop.job.jar", itestJar.getArchiveFile().get().asFile.absolutePath) + } integrationTest.testClassesDirs = project.sourceSets.itest.output.classesDirs integrationTest.classpath = project.sourceSets.itest.runtimeClasspath diff --git a/mr/src/itest/java/org/elasticsearch/hadoop/ProvisionerTest.java b/mr/src/itest/java/org/elasticsearch/hadoop/ProvisionerTest.java index f0ac7dc1c..7925f884e 100644 --- a/mr/src/itest/java/org/elasticsearch/hadoop/ProvisionerTest.java +++ b/mr/src/itest/java/org/elasticsearch/hadoop/ProvisionerTest.java @@ -18,8 +18,6 @@ */ package org.elasticsearch.hadoop; -import org.elasticsearch.hadoop.HdpBootstrap; -import org.elasticsearch.hadoop.Provisioner; import org.junit.Test; public class ProvisionerTest { diff --git a/spark/core/itest/java/org/elasticsearch/spark/integration/SparkUtils.java b/spark/core/itest/java/org/elasticsearch/spark/integration/SparkUtils.java index b079f0936..37d6a4436 100644 --- a/spark/core/itest/java/org/elasticsearch/spark/integration/SparkUtils.java +++ b/spark/core/itest/java/org/elasticsearch/spark/integration/SparkUtils.java @@ -18,44 +18,17 @@ */ package org.elasticsearch.spark.integration; -import java.io.File; -import java.io.FileFilter; -import java.io.IOException; import java.lang.reflect.Constructor; -import java.util.Arrays; import org.apache.spark.SparkConf; -import org.elasticsearch.hadoop.util.Assert; +import org.elasticsearch.hadoop.Provisioner; import org.elasticsearch.hadoop.util.ReflectionUtils; import com.esotericsoftware.kryo.Kryo; public abstract class SparkUtils { - public static final String[] ES_SPARK_TESTING_JAR; - - static { - // init ES-Hadoop JAR - // expect the jar under build\libs - try { - File folder = new File("build" + File.separator + "libs" + File.separator).getCanonicalFile(); - System.out.println(folder.getAbsolutePath()); - // find proper jar - File[] files = folder.listFiles(new FileFilter() { - - @Override - public boolean accept(File pathname) { - return pathname.getName().contains("-testing.jar"); - } - }); - Assert.isTrue(files != null && files.length == 1, - String.format("Cannot find elasticsearch spark jar (too many or no file found);%s", Arrays.toString(files))); - ES_SPARK_TESTING_JAR = new String[] { files[0].getAbsoluteFile().toURI().toString() }; - - } catch (IOException ex) { - throw new RuntimeException("Cannot find required files", ex); - } - } + public static final String[] ES_SPARK_TESTING_JAR = new String[] {Provisioner.ESHADOOP_TESTING_JAR}; public static Kryo sparkSerializer(SparkConf conf) throws Exception { // reflection galore diff --git a/test/shared/src/main/java/org/elasticsearch/hadoop/Provisioner.java b/test/shared/src/main/java/org/elasticsearch/hadoop/Provisioner.java index c05862de7..f42e58e2c 100644 --- a/test/shared/src/main/java/org/elasticsearch/hadoop/Provisioner.java +++ b/test/shared/src/main/java/org/elasticsearch/hadoop/Provisioner.java @@ -37,24 +37,21 @@ public abstract class Provisioner { public static final String ESHADOOP_TESTING_JAR; public static final String HDFS_ES_HDP_LIB = "/eshdp/libs/es-hadoop.jar"; + public static final String SYS_PROP_JOB_JAR = "es.hadoop.job.jar"; static { // init ES-Hadoop JAR // expect the jar under build\libs try { - File folder = new File("build" + File.separator + "libs" + File.separator).getCanonicalFile(); - // find proper jar - File[] files = folder.listFiles(new FileFilter() { - - @Override - public boolean accept(File pathname) { - return pathname.getName().contains("-testing"); - } - }); - Assert.isTrue(files != null && files.length == 1, - String.format("Cannot find elasticsearch hadoop jar (too many or no file found);%s", Arrays.toString(files))); - ESHADOOP_TESTING_JAR = files[0].getAbsoluteFile().toURI().toString(); + String jobJarLocation = System.getProperty(SYS_PROP_JOB_JAR); + if (jobJarLocation == null) { + throw new RuntimeException("Cannot find elasticsearch hadoop jar. System Property [" + SYS_PROP_JOB_JAR + "] was not set."); + } + File testJar = new File(jobJarLocation).getCanonicalFile(); + Assert.isTrue(testJar.exists(), + String.format("Cannot find elasticsearch hadoop jar. File not found [%s]", testJar)); + ESHADOOP_TESTING_JAR = testJar.getAbsolutePath(); } catch (IOException ex) { throw new RuntimeException("Cannot find required files", ex); }