Skip to content

Commit

Permalink
HBASE-27938 PE load any custom implementation of tests at runtime (#5307
Browse files Browse the repository at this point in the history 
) (#5899)

Signed-off-by: Duo Zhang <[email protected]>
Signed-off-by: Wellington Chevreuil <[email protected]>
Signed-off-by: Viraj Jasani <[email protected]>
  • Loading branch information
@gvprathyusha6 @virajjasani
gvprathyusha6 authored and virajjasani committed May 15, 2024
1 parent ae25102 commit ce361f6
Show file tree
Hide file tree
Showing 2 changed files with 94 additions and 5 deletions.
52 changes: 47 additions & 5 deletions hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import java.util.Locale;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.Queue;
import java.util.Random;
import java.util.TreeMap;
Expand Down Expand Up @@ -360,7 +361,8 @@ static boolean checkTable(Admin admin, TestOptions opts) throws IOException {
// {RegionSplitPolicy,replica count} does not match requested, or when the
// number of column families does not match requested.
if (
(exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions)
(exists && opts.presplitRegions != DEFAULT_OPTS.presplitRegions
&& opts.presplitRegions != admin.getRegions(tableName).size())
|| (!isReadCmd && desc != null
&& !StringUtils.equals(desc.getRegionSplitPolicyClassName(), opts.splitPolicy))
|| (!isReadCmd && desc != null && desc.getRegionReplication() != opts.replicas)
Expand Down Expand Up @@ -721,6 +723,7 @@ static class TestOptions {
boolean cacheBlocks = true;
Scan.ReadType scanReadType = Scan.ReadType.DEFAULT;
long bufferSize = 2l * 1024l * 1024l;
Properties commandProperties;

public TestOptions() {
}
Expand Down Expand Up @@ -777,6 +780,11 @@ public TestOptions(TestOptions that) {
this.cacheBlocks = that.cacheBlocks;
this.scanReadType = that.scanReadType;
this.bufferSize = that.bufferSize;
this.commandProperties = that.commandProperties;
}

public Properties getCommandProperties() {
return commandProperties;
}

public int getCaching() {
Expand Down Expand Up @@ -1142,10 +1150,10 @@ private static long nextRandomSeed() {
protected final Configuration conf;
protected final TestOptions opts;

private final Status status;
protected final Status status;

private String testName;
private Histogram latencyHistogram;
protected Histogram latencyHistogram;
private Histogram replicaLatencyHistogram;
private Histogram valueSizeHistogram;
private Histogram rpcCallsHistogram;
Expand Down Expand Up @@ -2618,7 +2626,7 @@ protected static void printUsage(final String shortName, final String message) {
System.err.println(message);
}
System.err.print("Usage: hbase " + shortName);
System.err.println(" <OPTIONS> [-D<property=value>]* <command> <nclients>");
System.err.println(" <OPTIONS> [-D<property=value>]* <command|class> <nclients>");
System.err.println();
System.err.println("General Options:");
System.err.println(
Expand Down Expand Up @@ -2719,6 +2727,13 @@ protected static void printUsage(final String shortName, final String message) {
System.err.println(String.format(" %-20s %s", command.getName(), command.getDescription()));
}
System.err.println();
System.err.println("Class:");
System.err.println("To run any custom implementation of PerformanceEvaluation.Test, "
+ "provide the classname of the implementaion class in place of "
+ "command name and it will be loaded at runtime from classpath.:");
System.err.println("Please consider to contribute back "
+ "this custom test impl into a builtin PE command for the benefit of the community");
System.err.println();
System.err.println("Args:");
System.err.println(" nclients Integer. Required. Total number of clients "
+ "(and HRegionServers) running. 1 <= value <= 500");
Expand Down Expand Up @@ -3013,6 +3028,20 @@ static TestOptions parseOpts(Queue<String> args) {
continue;
}

final String commandPropertiesFile = "--commandPropertiesFile=";
if (cmd.startsWith(commandPropertiesFile)) {
String fileName = String.valueOf(cmd.substring(commandPropertiesFile.length()));
Properties properties = new Properties();
try {
properties
.load(PerformanceEvaluation.class.getClassLoader().getResourceAsStream(fileName));
opts.commandProperties = properties;
} catch (IOException e) {
LOG.error("Failed to load metricIds from properties file", e);
}
continue;
}

validateParsedOpts(opts);

if (isCommandClass(cmd)) {
Expand Down Expand Up @@ -3126,7 +3155,20 @@ public int run(String[] args) throws Exception {
}

private static boolean isCommandClass(String cmd) {
return COMMANDS.containsKey(cmd);
return COMMANDS.containsKey(cmd) || isCustomTestClass(cmd);
}

private static boolean isCustomTestClass(String cmd) {
Class<? extends Test> cmdClass;
try {
cmdClass =
(Class<? extends Test>) PerformanceEvaluation.class.getClassLoader().loadClass(cmd);
addCommandDescriptor(cmdClass, cmd, "custom command");
return true;
} catch (Throwable th) {
LOG.info("No class found for command: " + cmd, th);
return false;
}
}

private static Class<? extends TestBase> determineCommandClass(String cmd) {
Expand Down
47 changes: 47 additions & 0 deletions hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/TestPerformanceEvaluation.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,21 +28,26 @@
import com.codahale.metrics.UniformReservoir;
import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.nio.charset.StandardCharsets;
import java.util.LinkedList;
import java.util.NoSuchElementException;
import java.util.Properties;
import java.util.Queue;
import java.util.Random;
import java.util.concurrent.ThreadLocalRandom;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.PerformanceEvaluation.RandomReadTest;
import org.apache.hadoop.hbase.PerformanceEvaluation.Status;
import org.apache.hadoop.hbase.PerformanceEvaluation.TestOptions;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.regionserver.CompactingMemStore;
import org.apache.hadoop.hbase.testclassification.MiscTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
Expand Down Expand Up @@ -357,4 +362,46 @@ public void testParseOptsValueRandom() {
assertEquals(true, options.valueRandom);
}

@Test
public void testCustomTestClassOptions() throws IOException {
Queue<String> opts = new LinkedList<>();
// create custom properties that can be used for a custom test class
Properties commandProps = new Properties();
commandProps.put("prop1", "val1");
String cmdPropsFilePath =
this.getClass().getClassLoader().getResource("").getPath() + "cmd_properties.txt";
FileWriter writer = new FileWriter(new File(cmdPropsFilePath));
commandProps.store(writer, null);
// create opts for the custom test class - commandPropertiesFile, testClassName
opts.offer("--commandPropertiesFile=" + "cmd_properties.txt");
String testClassName = "org.apache.hadoop.hbase.TestPerformanceEvaluation$PESampleTestImpl";
opts.offer(testClassName);
opts.offer("1");
PerformanceEvaluation.TestOptions options = PerformanceEvaluation.parseOpts(opts);
assertNotNull(options);
assertNotNull(options.getCmdName());
assertEquals(testClassName, options.getCmdName());
assertNotNull(options.getCommandProperties());
assertEquals("val1", options.getCommandProperties().get("prop1"));
}

class PESampleTestImpl extends PerformanceEvaluation.Test {

PESampleTestImpl(Connection con, TestOptions options, Status status) {
super(con, options, status);
}

@Override
void onStartup() throws IOException {
}

@Override
void onTakedown() throws IOException {
}

@Override
boolean testRow(int i, long startTime) throws IOException, InterruptedException {
return false;
}
}
}

0 comments on commit ce361f6

Please sign in to comment.