Skip to content

Commit

Permalink
GEODE-9632: Allow INDEX_THRESHOLD_SIZE System property to override Co…
Browse files Browse the repository at this point in the history 
…mpiledValue.RESULT_LIMIT (#7010)

* GEODE-9632: fix output for the range query with wildcard character
  • Loading branch information
@mkevo
mkevo authored Nov 15, 2021
1 parent 3eaeed8 commit 67359dc
Show file tree
Hide file tree
Showing 11 changed files with 250 additions and 16 deletions.
20 changes: 10 additions & 10 deletions ...rc/integrationTest/java/org/apache/geode/cache/query/functional/LimitClauseJUnitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -673,7 +673,7 @@ public void testLimitDistinctIterEvaluatedQueryForResultBagWithIndex() {
assertEquals(5, result.size());
SelectResults wrapper = (SelectResults) result;
assertEquals(5, wrapper.asSet().size());
assertTrue(observer.limitAppliedAtIndex);
assertFalse(observer.limitAppliedAtIndex);
} catch (Exception e) {
CacheUtils.getLogger().error(e);
fail(e.toString());
Expand Down Expand Up @@ -710,7 +710,7 @@ public void testLimitDistinctIterEvaluatedQueryForResultBagWithProjectionAttribu
assertEquals(5, result.size());
SelectResults wrapper = (SelectResults) result;
assertEquals(5, wrapper.asSet().size());
assertTrue(observer.limitAppliedAtIndex);
assertFalse(observer.limitAppliedAtIndex);
} catch (Exception e) {
CacheUtils.getLogger().error(e);
fail(e.toString());
Expand Down Expand Up @@ -754,7 +754,7 @@ public void testLimitDistinctIterEvaluatedQueryWithDuplicatesInIterationForResul
assertEquals(5, result.size());
SelectResults wrapper = (SelectResults) result;
assertEquals(5, wrapper.asSet().size());
assertTrue(observer.limitAppliedAtIndex);
assertFalse(observer.limitAppliedAtIndex);
} catch (Exception e) {
CacheUtils.getLogger().error(e);
fail(e.toString());
Expand Down Expand Up @@ -799,7 +799,7 @@ public void testLimitDistinctIterEvaluatedQueryWithDuplicatesInIterationWithProj
assertEquals(5, result.size());
SelectResults wrapper = (SelectResults) result;
assertEquals(5, wrapper.asSet().size());
assertTrue(observer.limitAppliedAtIndex);
assertFalse(observer.limitAppliedAtIndex);
} catch (Exception e) {
CacheUtils.getLogger().error(e);
fail(e.toString());
Expand Down Expand Up @@ -837,7 +837,7 @@ public void testLimitDistinctIterEvaluatedQueryForStructBagWithProjectionAttribu
SelectResults wrapper = (SelectResults) result;
assertEquals(5, wrapper.asSet().size());
assertTrue(wrapper.getCollectionType().getElementType() instanceof StructType);
assertTrue(observer.limitAppliedAtIndex);
assertFalse(observer.limitAppliedAtIndex);
} catch (Exception e) {
CacheUtils.getLogger().error(e);
fail(e.toString());
Expand Down Expand Up @@ -919,7 +919,7 @@ public void testLimitDistinctIterEvaluatedQueryWithDuplicatesInIterationWithProj
assertEquals(5, result.size());
SelectResults wrapper = (SelectResults) result;
assertEquals(5, wrapper.asSet().size());
assertTrue(observer.limitAppliedAtIndex);
assertFalse(observer.limitAppliedAtIndex);
} catch (Exception e) {
CacheUtils.getLogger().error(e);
fail(e.toString());
Expand Down Expand Up @@ -1051,7 +1051,7 @@ public void testLimitDistinctQueryWithTwoCondWithTwoIndex() {
SelectResults wrapper = (SelectResults) result;
assertEquals(10, wrapper.asSet().size());
assertFalse(observer.limitAppliedAtIndex && observer.indexName.equals("idIndex"));
assertTrue(observer.limitAppliedAtIndex && observer.indexName.equals("statusIndex"));
assertFalse(observer.limitAppliedAtIndex && observer.indexName.equals("statusIndex"));
} catch (Exception e) {
CacheUtils.getLogger().error(e);
fail(e.toString());
Expand Down Expand Up @@ -1130,7 +1130,7 @@ public void testLimitNonDistinctQueryWithTwoCondTwoIndex() {
query = qs.newQuery(qstr);
result = (SelectResults) query.execute();
assertEquals(10, result.size());
assertTrue(observer.limitAppliedAtIndex && observer.indexName.equals("statusIndex"));
assertFalse(observer.limitAppliedAtIndex && observer.indexName.equals("statusIndex"));
}
} catch (Exception e) {
CacheUtils.getLogger().error(e);
Expand Down Expand Up @@ -1528,7 +1528,7 @@ public void testLimitOnEqualsCompactRangeIndexedFieldWithAndClauseNonIndexedFiel

assertNotNull(idIndex);
SelectResults resultsWithIndex = (SelectResults) query.execute();
assertTrue(observer.limitAppliedAtIndex);
assertFalse(observer.limitAppliedAtIndex);
assertEquals(5, resultsWithIndex.size());
}

Expand Down Expand Up @@ -1753,7 +1753,7 @@ public void testLimitOnCompactRangeIndexedFieldWithAndClauseJunctionNonIndexedFi

assertNotNull(idIndex);
SelectResults resultsWithIndex = (SelectResults) query.execute();
assertTrue(observer.limitAppliedAtIndex);
assertFalse(observer.limitAppliedAtIndex);
assertEquals(5, resultsWithIndex.size());
}

Expand Down
2 changes: 1 addition & 1 deletion ...t/java/org/apache/geode/cache/query/functional/NonDistinctOrderByReplicatedJUnitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -190,7 +190,7 @@ public void testLimitApplicationOnPrimaryKeyIndex() throws Exception {

Region r1 = this.createRegion("portfolio1", Portfolio.class);

for (int i = 0; i < 50; i++) {
for (int i = 0; i < 200; i++) {
r1.put(i + "", new Portfolio(i));
}

Expand Down
2 changes: 1 addition & 1 deletion ...egrationTest/java/org/apache/geode/cache/query/functional/OrderByReplicatedJUnitTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,7 @@ public void testLimitApplicationOnPrimaryKeyIndex() throws Exception {

Region r1 = createRegion("portfolio1", Portfolio.class);

for (int i = 0; i < 50; i++) {
for (int i = 0; i < 200; i++) {
r1.put(i + "", new Portfolio(i));
}

Expand Down
4 changes: 0 additions & 4 deletions ...ore/src/main/java/org/apache/geode/cache/query/internal/AbstractGroupOrRangeJunction.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,15 +35,11 @@
import org.apache.geode.cache.query.internal.types.StructTypeImpl;
import org.apache.geode.cache.query.types.ObjectType;
import org.apache.geode.internal.Assert;
import org.apache.geode.util.internal.GeodeGlossary;

public abstract class AbstractGroupOrRangeJunction extends AbstractCompiledValue
implements Filter, OQLLexerTokenTypes {
/** left operand */
final CompiledValue[] _operands;
private static final int INDEX_RESULT_THRESHOLD_DEFAULT = 100;
public static final String INDX_THRESHOLD_PROP_STR =
GeodeGlossary.GEMFIRE_PREFIX + "Query.INDEX_THRESHOLD_SIZE";
private static final int indexThresholdSize =
Integer.getInteger(INDX_THRESHOLD_PROP_STR, INDEX_RESULT_THRESHOLD_DEFAULT).intValue();
private int _operator = 0;
Expand Down
5 changes: 5 additions & 0 deletions geode-core/src/main/java/org/apache/geode/cache/query/internal/QueryUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
*/
package org.apache.geode.cache.query.internal;

import static org.apache.geode.cache.query.internal.CompiledValue.indexThresholdSize;

import java.util.ArrayList;
import java.util.Collections;
import java.util.HashSet;
Expand Down Expand Up @@ -669,6 +671,9 @@ private static int getLimitValue(ExecutionContext context) {
limit = context.cacheGet(CompiledValue.RESULT_LIMIT) != null
? (Integer) context.cacheGet(CompiledValue.RESULT_LIMIT) : -1;
}
if (limit != -1 && limit < indexThresholdSize) {
limit = indexThresholdSize;
}
return limit;
}

Expand Down
9 changes: 9 additions & 0 deletions geode-core/src/main/java/org/apache/geode/cache/query/internal/index/CompactRangeIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
*/
package org.apache.geode.cache.query.internal.index;

import static org.apache.geode.cache.query.internal.CompiledValue.indexThresholdSize;

import java.util.ArrayList;
import java.util.Collection;
import java.util.HashSet;
Expand Down Expand Up @@ -95,6 +97,7 @@ public class CompactRangeIndex extends AbstractIndex {

private IndexStore indexStore;


@MutableForTesting
static boolean TEST_ALWAYS_UPDATE_IN_PROGRESS = false;

Expand Down Expand Up @@ -483,6 +486,9 @@ private void lockedQueryPrivate(Object key, int operator, Collection results,
Boolean applyLimit = (Boolean) context.cacheGet(CompiledValue.CAN_APPLY_LIMIT_AT_INDEX);
if (applyLimit != null && applyLimit) {
limit = (Integer) context.cacheGet(CompiledValue.RESULT_LIMIT);
if (limit < indexThresholdSize) {
limit = indexThresholdSize;
}
}

Boolean orderByClause = (Boolean) context.cacheGet(CompiledValue.CAN_APPLY_ORDER_BY_AT_INDEX);
Expand Down Expand Up @@ -517,6 +523,9 @@ void lockedQuery(Object lowerBoundKey, int lowerBoundOperator, Object upperBound
Boolean applyLimit = (Boolean) context.cacheGet(CompiledValue.CAN_APPLY_LIMIT_AT_INDEX);
if (applyLimit != null && applyLimit) {
limit = (Integer) context.cacheGet(CompiledValue.RESULT_LIMIT);
if (limit != -1 && limit < indexThresholdSize) {
limit = indexThresholdSize;
}
}
Boolean orderByClause = (Boolean) context.cacheGet(CompiledValue.CAN_APPLY_ORDER_BY_AT_INDEX);

Expand Down
4 changes: 4 additions & 0 deletions geode-core/src/main/java/org/apache/geode/cache/query/internal/index/HashIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
*/
package org.apache.geode.cache.query.internal.index;

import static org.apache.geode.cache.query.internal.CompiledValue.indexThresholdSize;
import static org.apache.geode.internal.lang.SystemUtils.getLineSeparator;

import java.util.ArrayList;
Expand Down Expand Up @@ -494,6 +495,9 @@ private void lockedQueryPrivate(Object key, int operator, Collection results,
Boolean applyLimit = (Boolean) context.cacheGet(CompiledValue.CAN_APPLY_LIMIT_AT_INDEX);
if (applyLimit != null && applyLimit) {
limit = (Integer) context.cacheGet(CompiledValue.RESULT_LIMIT);
if (limit != -1 && limit < indexThresholdSize) {
limit = indexThresholdSize;
}
}

Boolean orderByClause = (Boolean) context.cacheGet(CompiledValue.CAN_APPLY_ORDER_BY_AT_INDEX);
Expand Down
2 changes: 2 additions & 0 deletions geode-core/src/main/java/org/apache/geode/cache/query/internal/index/IndexManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,8 @@ public class IndexManager {
public static final int RECREATE_INDEX = 4;
private final InternalCache cache;
protected final Region region;
String indexThresholdSize =
System.getProperty(GeodeGlossary.GEMFIRE_PREFIX + "Query.INDEX_THRESHOLD_SIZE");

private final boolean isOverFlowToDisk;
private final boolean offHeap;
Expand Down
8 changes: 8 additions & 0 deletions geode-core/src/main/java/org/apache/geode/cache/query/internal/index/PrimaryKeyIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
*/
package org.apache.geode.cache.query.internal.index;

import static org.apache.geode.cache.query.internal.CompiledValue.indexThresholdSize;

import java.util.Collection;
import java.util.Iterator;
import java.util.List;
Expand Down Expand Up @@ -105,6 +107,9 @@ void lockedQuery(Object key, int operator, Collection results, Set keysToRemove,
Boolean applyLimit = (Boolean) context.cacheGet(CompiledValue.CAN_APPLY_LIMIT_AT_INDEX);
if (applyLimit != null && applyLimit) {
limit = (Integer) context.cacheGet(CompiledValue.RESULT_LIMIT);
if (limit != -1 && limit < indexThresholdSize) {
limit = indexThresholdSize;
}
}
QueryObserver observer = QueryObserverHolder.getInstance();
if (limit != -1 && results.size() == limit) {
Expand Down Expand Up @@ -184,6 +189,9 @@ void lockedQuery(Object key, int operator, Collection results, CompiledValue ite

if (applyLimit != null && applyLimit.booleanValue()) {
limit = ((Integer) context.cacheGet(CompiledValue.RESULT_LIMIT)).intValue();
if (limit != -1 && limit < indexThresholdSize) {
limit = indexThresholdSize;
}
}
if (limit != -1 && results.size() == limit) {
observer.limitAppliedAtIndexLevel(this, limit, results);
Expand Down
4 changes: 4 additions & 0 deletions geode-core/src/main/java/org/apache/geode/cache/query/internal/index/RangeIndex.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

import static java.lang.Integer.MAX_VALUE;
import static java.lang.Integer.valueOf;
import static org.apache.geode.cache.query.internal.CompiledValue.indexThresholdSize;

import java.util.ArrayList;
import java.util.Collection;
Expand Down Expand Up @@ -1195,6 +1196,9 @@ void lockedQuery(Object key, int operator, Collection results, CompiledValue ite
Boolean applyLimit = (Boolean) context.cacheGet(CompiledValue.CAN_APPLY_LIMIT_AT_INDEX);
if (applyLimit != null && applyLimit) {
limit = (Integer) context.cacheGet(CompiledValue.RESULT_LIMIT);
if (limit != -1 && limit < indexThresholdSize) {
limit = indexThresholdSize;
}
}

Boolean orderByClause = (Boolean) context.cacheGet(CompiledValue.CAN_APPLY_ORDER_BY_AT_INDEX);
Expand Down
Loading

0 comments on commit 67359dc

Please sign in to comment.