Skip to content

Commit

Permalink
ESQL: Reduce number of ignored tests (elastic#108471)
Browse files Browse the repository at this point in the history
This reduces the number of skipped tests for functions, dropping the
number from 130754 to 3226. This doesn't buy us much more coverage, but
it doesn't really take any more time so it's probably ok. It'd be nice
to have some understanding of each of the skipped tests, 130k is way too
many to put in your head.

The actual test change is: when you are need to build an evaluator but
can't because you'll get a type error, just assert that you get a type
error and let the test finish. This is nearly as fast as just bailing,
and it gets us to the point where we can start reasoning about the
skipped tests.
  • Loading branch information
nik9000 authored May 10, 2024
1 parent 11de886 commit cb43573
Showing 1 changed file with 40 additions and 21 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -166,14 +166,17 @@ protected static Iterable<Object[]> parameterSuppliersFromTypedData(List<TestCas
* Build an {@link Attribute} that loads a field.
*/
public static FieldAttribute field(String name, DataType type) {
return new FieldAttribute(Source.EMPTY, name, new EsField(name, type, Map.of(), true));
return new FieldAttribute(Source.synthetic(name), name, new EsField(name, type, Map.of(), true));
}

/**
* Build an {@link Attribute} that loads a field and then creates a deep copy of its data.
*/
public static Expression deepCopyOfField(String name, DataType type) {
return new DeepCopy(Source.EMPTY, new FieldAttribute(Source.EMPTY, name, new EsField(name, type, Map.of(), true)));
return new DeepCopy(
Source.synthetic(name),
new FieldAttribute(Source.synthetic(name), name, new EsField(name, type, Map.of(), true))
);
}

/**
Expand Down Expand Up @@ -257,11 +260,7 @@ public final void testEvaluate() {
boolean readFloating = randomBoolean();
Expression expression = readFloating ? buildDeepCopyOfFieldExpression(testCase) : buildFieldExpression(testCase);
if (testCase.getExpectedTypeError() != null) {
assertTrue("expected unresolved", expression.typeResolved().unresolved());
if (readFloating == false) {
// The hack that creates floating fields changes the error message so don't assert it
assertThat(expression.typeResolved().message(), equalTo(testCase.getExpectedTypeError()));
}
assertTypeResolutionFailure(expression);
return;
}
Expression.TypeResolution resolution = expression.typeResolved();
Expand Down Expand Up @@ -368,10 +367,13 @@ protected Matcher<Object> allNullsMatcher() {
}

private void testEvaluateBlock(BlockFactory inputBlockFactory, DriverContext context, boolean insertNulls) {
Expression expression = randomBoolean() ? buildDeepCopyOfFieldExpression(testCase) : buildFieldExpression(testCase);
if (testCase.getExpectedTypeError() != null) {
assertTypeResolutionFailure(expression);
return;
}
assumeTrue("Can't build evaluator", testCase.canBuildEvaluator());
assumeTrue("Expected type must be representable to build an evaluator", EsqlDataTypes.isRepresentable(testCase.expectedType()));
assumeTrue("Must build evaluator to test sending it blocks", testCase.getExpectedTypeError() == null);
boolean readFloating = randomBoolean();
int positions = between(1, 1024);
List<TestCaseSupplier.TypedData> data = testCase.getData();
Page onePositionPage = row(testCase.getDataValues());
Expand Down Expand Up @@ -401,7 +403,6 @@ private void testEvaluateBlock(BlockFactory inputBlockFactory, DriverContext con
}
b++;
}
Expression expression = readFloating ? buildDeepCopyOfFieldExpression(testCase) : buildFieldExpression(testCase);
try (
ExpressionEvaluator eval = evaluator(expression).get(context);
Block block = eval.eval(new Page(positions, manyPositionsBlocks))
Expand All @@ -427,13 +428,15 @@ private void testEvaluateBlock(BlockFactory inputBlockFactory, DriverContext con
}
}

// TODO cranky time

public void testSimpleWithNulls() { // TODO replace this with nulls inserted into the test case like anyNullIsNull
Expression expression = buildFieldExpression(testCase);
if (testCase.getExpectedTypeError() != null) {
assertTypeResolutionFailure(expression);
return;
}
assumeTrue("Can't build evaluator", testCase.canBuildEvaluator());
assumeTrue("Nothing to do if a type error", testCase.getExpectedTypeError() == null);
List<Object> simpleData = testCase.getDataValues();
try (EvalOperator.ExpressionEvaluator eval = evaluator(buildFieldExpression(testCase)).get(driverContext())) {
try (EvalOperator.ExpressionEvaluator eval = evaluator(expression).get(driverContext())) {
BlockFactory blockFactory = TestBlockFactory.getNonBreakingInstance();
Block[] orig = BlockUtils.fromListRow(blockFactory, simpleData);
for (int i = 0; i < orig.length; i++) {
Expand Down Expand Up @@ -472,12 +475,16 @@ protected void assertSimpleWithNulls(List<Object> data, Block value, int nullBlo
}

public final void testEvaluateInManyThreads() throws ExecutionException, InterruptedException {
Expression expression = buildFieldExpression(testCase);
if (testCase.getExpectedTypeError() != null) {
assertTypeResolutionFailure(expression);
return;
}
assumeTrue("Can't build evaluator", testCase.canBuildEvaluator());
assumeTrue("Expected type must be representable to build an evaluator", EsqlDataTypes.isRepresentable(testCase.expectedType()));
assumeTrue("Nothing to do if a type error", testCase.getExpectedTypeError() == null);
int count = 10_000;
int threads = 5;
var evalSupplier = evaluator(buildFieldExpression(testCase));
var evalSupplier = evaluator(expression);
ExecutorService exec = Executors.newFixedThreadPool(threads);
try {
List<Future<?>> futures = new ArrayList<>();
Expand All @@ -504,26 +511,33 @@ public final void testEvaluateInManyThreads() throws ExecutionException, Interru
}

public final void testEvaluatorToString() {
Expression expression = buildFieldExpression(testCase);
if (testCase.getExpectedTypeError() != null) {
assertTypeResolutionFailure(expression);
return;
}
assumeTrue("Can't build evaluator", testCase.canBuildEvaluator());
assumeTrue("Nothing to do if a type error", testCase.getExpectedTypeError() == null);
var factory = evaluator(buildFieldExpression(testCase));
var factory = evaluator(expression);
try (ExpressionEvaluator ev = factory.get(driverContext())) {
assertThat(ev.toString(), testCase.evaluatorToString());
}
}

public final void testFactoryToString() {
Expression expression = buildFieldExpression(testCase);
if (testCase.getExpectedTypeError() != null) {
assertTypeResolutionFailure(expression);
return;
}
assumeTrue("Can't build evaluator", testCase.canBuildEvaluator());
assumeTrue("Nothing to do if a type error", testCase.getExpectedTypeError() == null);
var factory = evaluator(buildFieldExpression(testCase));
assertThat(factory.toString(), testCase.evaluatorToString());
}

public final void testFold() {
Expression expression = buildLiteralExpression(testCase);
if (testCase.getExpectedTypeError() != null) {
assertTrue(expression.typeResolved().unresolved());
assertThat(expression.typeResolved().message(), equalTo(testCase.getExpectedTypeError()));
assertTypeResolutionFailure(expression);
return;
}
assertFalse(expression.typeResolved().unresolved());
Expand Down Expand Up @@ -1115,6 +1129,11 @@ protected static DataType[] representableNonSpatialTypes() {
return representableNonSpatial().toArray(DataType[]::new);
}

protected final void assertTypeResolutionFailure(Expression expression) {
assertTrue("expected unresolved", expression.typeResolved().unresolved());
assertThat(expression.typeResolved().message(), equalTo(testCase.getExpectedTypeError()));
}

@AfterClass
public static void renderSignature() throws IOException {
if (System.getProperty("generateDocs") == null) {
Expand Down

0 comments on commit cb43573

Please sign in to comment.