diff --git a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java index 4fa1a4802489..a54f4addabb7 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java @@ -149,8 +149,6 @@ private BitmapColumnIndex wrapRangeIndexWithNullValueIndex( BitmapColumnIndex rangeIndex ) { - - final BitmapColumnIndex nullBitmap; final NullValueIndex nulls = indexSupplier.as(NullValueIndex.class); if (nulls == null) { @@ -166,6 +164,12 @@ public ColumnIndexCapabilities getIndexCapabilities() return rangeIndex.getIndexCapabilities().merge(nullBitmap.getIndexCapabilities()); } + @Override + public int estimatedComputeCost() + { + return rangeIndex.estimatedComputeCost(); + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java index 75eaebb27c81..d39afdecb7f0 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/IsBooleanFilter.java @@ -78,6 +78,12 @@ public ColumnIndexCapabilities getIndexCapabilities() return baseIndex.getIndexCapabilities(); } + @Override + public int estimatedComputeCost() + { + return baseIndex.estimatedComputeCost(); + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java index 58dd90c8af18..3e89148e2ccc 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/NotFilter.java @@ -80,6 +80,12 @@ public ColumnIndexCapabilities getIndexCapabilities() return baseIndex.getIndexCapabilities(); } + @Override + public int estimatedComputeCost() + { + return baseIndex.estimatedComputeCost(); + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { diff --git a/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java index a79d4d2ac222..311f32667dd3 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/SpatialFilter.java @@ -94,6 +94,12 @@ public ColumnIndexCapabilities getIndexCapabilities() return new SimpleColumnIndexCapabilities(true, true); } + @Override + public int estimatedComputeCost() + { + return Integer.MAX_VALUE; + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { diff --git a/processing/src/main/java/org/apache/druid/segment/index/AllFalseBitmapColumnIndex.java b/processing/src/main/java/org/apache/druid/segment/index/AllFalseBitmapColumnIndex.java index d0469dffe560..907ce55cb2a9 100644 --- a/processing/src/main/java/org/apache/druid/segment/index/AllFalseBitmapColumnIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/index/AllFalseBitmapColumnIndex.java @@ -50,6 +50,12 @@ public ColumnIndexCapabilities getIndexCapabilities() return SimpleColumnIndexCapabilities.getConstant(); } + @Override + public int estimatedComputeCost() + { + return 0; + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { diff --git a/processing/src/main/java/org/apache/druid/segment/index/AllTrueBitmapColumnIndex.java b/processing/src/main/java/org/apache/druid/segment/index/AllTrueBitmapColumnIndex.java index 02ff2c5bccaa..5bd48d31802b 100644 --- a/processing/src/main/java/org/apache/druid/segment/index/AllTrueBitmapColumnIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/index/AllTrueBitmapColumnIndex.java @@ -39,6 +39,12 @@ public ColumnIndexCapabilities getIndexCapabilities() return SimpleColumnIndexCapabilities.getConstant(); } + @Override + public int estimatedComputeCost() + { + return 0; + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { diff --git a/processing/src/main/java/org/apache/druid/segment/index/AllUnknownBitmapColumnIndex.java b/processing/src/main/java/org/apache/druid/segment/index/AllUnknownBitmapColumnIndex.java index a33ae663d53d..793773ede11e 100644 --- a/processing/src/main/java/org/apache/druid/segment/index/AllUnknownBitmapColumnIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/index/AllUnknownBitmapColumnIndex.java @@ -44,6 +44,12 @@ public ColumnIndexCapabilities getIndexCapabilities() return SimpleColumnIndexCapabilities.getConstant(); } + @Override + public int estimatedComputeCost() + { + return 0; + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { diff --git a/processing/src/main/java/org/apache/druid/segment/index/BitmapColumnIndex.java b/processing/src/main/java/org/apache/druid/segment/index/BitmapColumnIndex.java index f28429969d33..11845da189a6 100644 --- a/processing/src/main/java/org/apache/druid/segment/index/BitmapColumnIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/index/BitmapColumnIndex.java @@ -38,10 +38,7 @@ public interface BitmapColumnIndex /** * Returns an estimated cost for computing the bitmap result. */ - default int estimatedComputeCost() - { - return Integer.MAX_VALUE; - } + int estimatedComputeCost(); /** * Compute a bitmap result wrapped with the {@link BitmapResultFactory} representing the rows matched by this index. diff --git a/processing/src/main/java/org/apache/druid/segment/index/DictionaryRangeScanningBitmapIndex.java b/processing/src/main/java/org/apache/druid/segment/index/DictionaryRangeScanningBitmapIndex.java index 37e84bb8b5d0..ee0a2dc67adf 100644 --- a/processing/src/main/java/org/apache/druid/segment/index/DictionaryRangeScanningBitmapIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/index/DictionaryRangeScanningBitmapIndex.java @@ -46,6 +46,12 @@ public DictionaryRangeScanningBitmapIndex(double sizeScale, int rangeSize) this.rangeSize = rangeSize; } + @Override + public int estimatedComputeCost() + { + return this.rangeSize; + } + @Nullable @Override public final T computeBitmapResult( diff --git a/processing/src/main/java/org/apache/druid/segment/index/DictionaryScanningBitmapIndex.java b/processing/src/main/java/org/apache/druid/segment/index/DictionaryScanningBitmapIndex.java index 1ae00a846baa..118d310d6566 100644 --- a/processing/src/main/java/org/apache/druid/segment/index/DictionaryScanningBitmapIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/index/DictionaryScanningBitmapIndex.java @@ -49,6 +49,12 @@ public DictionaryScanningBitmapIndex(int dictionarySize, double scaleThreshold) this.scaleThreshold = scaleThreshold; } + @Override + public int estimatedComputeCost() + { + return this.dictionarySize; + } + @Nullable @Override public final T computeBitmapResult( diff --git a/processing/src/main/java/org/apache/druid/segment/index/IndexedUtf8ValueIndexes.java b/processing/src/main/java/org/apache/druid/segment/index/IndexedUtf8ValueIndexes.java index 6015088d558d..1d769b3438cb 100644 --- a/processing/src/main/java/org/apache/druid/segment/index/IndexedUtf8ValueIndexes.java +++ b/processing/src/main/java/org/apache/druid/segment/index/IndexedUtf8ValueIndexes.java @@ -22,7 +22,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import org.apache.druid.annotations.SuppressFBWarnings; import org.apache.druid.collections.bitmap.BitmapFactory; import org.apache.druid.collections.bitmap.ImmutableBitmap; @@ -48,6 +48,7 @@ import java.util.Comparator; import java.util.List; import java.util.SortedSet; +import java.util.stream.Collectors; public final class IndexedUtf8ValueIndexes> implements StringValueSetIndexes, Utf8ValueSetIndexes, ValueIndexes, ValueSetIndexes @@ -83,6 +84,11 @@ public BitmapColumnIndex forValue(@Nullable String value) final ByteBuffer utf8 = StringUtils.toUtf8ByteBuffer(value); return new SimpleBitmapColumnIndex() { + @Override + public int estimatedComputeCost() + { + return 1; + } @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) @@ -122,10 +128,7 @@ public BitmapColumnIndex forValue(@Nonnull Object value, TypeSignature values) { return getBitmapColumnIndexForSortedIterableUtf8( - Iterables.transform( - values, - StringUtils::toUtf8ByteBuffer - ), + values.stream().map(StringUtils::toUtf8ByteBuffer).collect(Collectors.toList()), values.size(), values.contains(null) ); @@ -170,7 +173,7 @@ private ImmutableBitmap getBitmap(int idx) * Helper used by {@link #forSortedValues} and {@link #forSortedValuesUtf8}. */ private BitmapColumnIndex getBitmapColumnIndexForSortedIterableUtf8( - Iterable valuesUtf8, + List valuesUtf8, int size, boolean valuesContainsNull ) @@ -241,7 +244,7 @@ public BitmapColumnIndex forSortedValues(@Nonnull List sortedValues, TypeSign return ValueSetIndexes.buildBitmapColumnIndexFromSortedIteratorScan( bitmapFactory, ByteBufferUtils.utf8Comparator(), - Iterables.transform(tailSet, StringUtils::toUtf8ByteBuffer), + Lists.transform(tailSet, StringUtils::toUtf8ByteBuffer), dictionary, bitmaps, unknownsIndex @@ -250,7 +253,7 @@ public BitmapColumnIndex forSortedValues(@Nonnull List sortedValues, TypeSign // fall through to value iteration return ValueSetIndexes.buildBitmapColumnIndexFromSortedIteratorBinarySearch( bitmapFactory, - Iterables.transform(tailSet, StringUtils::toUtf8ByteBuffer), + Lists.transform(tailSet, StringUtils::toUtf8ByteBuffer), dictionary, bitmaps, unknownsIndex @@ -258,7 +261,7 @@ public BitmapColumnIndex forSortedValues(@Nonnull List sortedValues, TypeSign } else { return ValueSetIndexes.buildBitmapColumnIndexFromIteratorBinarySearch( bitmapFactory, - Iterables.transform( + Lists.transform( sortedValues, x -> StringUtils.toUtf8ByteBuffer(DimensionHandlerUtils.convertObjectToString(x)) ), diff --git a/processing/src/main/java/org/apache/druid/segment/index/SimpleImmutableBitmapIndex.java b/processing/src/main/java/org/apache/druid/segment/index/SimpleImmutableBitmapIndex.java index 7c6f6023ed0a..6c4177dc53a4 100644 --- a/processing/src/main/java/org/apache/druid/segment/index/SimpleImmutableBitmapIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/index/SimpleImmutableBitmapIndex.java @@ -35,6 +35,12 @@ public SimpleImmutableBitmapIndex(ImmutableBitmap bitmap) this.bitmap = bitmap; } + @Override + public int estimatedComputeCost() + { + return 0; + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { diff --git a/processing/src/main/java/org/apache/druid/segment/index/semantic/ValueSetIndexes.java b/processing/src/main/java/org/apache/druid/segment/index/semantic/ValueSetIndexes.java index 56d6da9a19af..9c004d354855 100644 --- a/processing/src/main/java/org/apache/druid/segment/index/semantic/ValueSetIndexes.java +++ b/processing/src/main/java/org/apache/druid/segment/index/semantic/ValueSetIndexes.java @@ -63,8 +63,8 @@ public interface ValueSetIndexes * @param sortedValues values to match, sorted in matchValueType order * @param matchValueType type of the value to match, used to assist conversion from the match value type to the column * value type - * @return {@link ImmutableBitmap} corresponding to the rows which match the values, or null if an index - * connot be computed for the supplied value type + * @return {@link ImmutableBitmap} corresponding to the rows which match the values, or null if an index + * connot be computed for the supplied value type */ @Nullable BitmapColumnIndex forSortedValues(@Nonnull List sortedValues, TypeSignature matchValueType); @@ -78,17 +78,17 @@ public interface ValueSetIndexes * {@link Indexed} value dictionary. Uses a strategy that does zipping similar to the merge step of a sort-merge, * where we step forward on both the iterator and the dictionary to find matches to build a * {@link Iterable}. - *

+ *

* If sorted match value iterator size is greater than (dictionary size * {@link #SORTED_SCAN_RATIO_THRESHOLD}), * consider using this method instead of {@link #buildBitmapColumnIndexFromSortedIteratorBinarySearch}. - *

+ *

* If the values in the iterator are NOT sorted the same as the dictionary, do NOT use this method, use * {@link #buildBitmapColumnIndexFromIteratorBinarySearch} instead. */ static BitmapColumnIndex buildBitmapColumnIndexFromSortedIteratorScan( BitmapFactory bitmapFactory, Comparator comparator, - Iterable values, + List values, Indexed dictionary, Indexed bitmaps, Supplier unknownsBitmap @@ -96,6 +96,13 @@ static BitmapColumnIndex buildBitmapColumnIndexFromSortedIteratorScan( { return new BaseValueSetIndexesFromIterable(bitmapFactory, bitmaps, unknownsBitmap) { + + @Override + public int estimatedComputeCost() + { + return Integer.max(values.size(), dictionary.size()); + } + @Override public Iterable getBitmapIterable() { @@ -163,13 +170,13 @@ private void findNext() *

* If sorted match value iterator size is less than (dictionary size * {@link #SORTED_SCAN_RATIO_THRESHOLD}), * consider using this method instead of {@link #buildBitmapColumnIndexFromSortedIteratorScan}. - *

+ *

* If the values in the iterator are not sorted the same as the dictionary, do not use this method, use * {@link #buildBitmapColumnIndexFromIteratorBinarySearch} instead. */ static BitmapColumnIndex buildBitmapColumnIndexFromSortedIteratorBinarySearch( BitmapFactory bitmapFactory, - Iterable values, + List values, Indexed dictionary, Indexed bitmaps, Supplier getUnknownsIndex @@ -177,6 +184,13 @@ static BitmapColumnIndex buildBitmapColumnIndexFromSortedIteratorBinarySearc { return new BaseValueSetIndexesFromIterable(bitmapFactory, bitmaps, getUnknownsIndex) { + + @Override + public int estimatedComputeCost() + { + return values.size(); + } + @Override public Iterable getBitmapIterable() { @@ -236,14 +250,14 @@ private void findNext() * {@link Indexed} value dictionary. This algorithm iterates the values to match and does a binary search for * matching values using {@link Indexed#indexOf(Object)} to build a {@link Iterable} until the match * values iterator is exhausted. - *

+ *

* If values of the iterator are sorted the same as the dictionary, use * {@link #buildBitmapColumnIndexFromSortedIteratorScan} or * {@link #buildBitmapColumnIndexFromSortedIteratorBinarySearch} instead. */ static BitmapColumnIndex buildBitmapColumnIndexFromIteratorBinarySearch( BitmapFactory bitmapFactory, - Iterable values, + List values, Indexed dictionary, Indexed bitmaps, Supplier getUnknownsIndex @@ -251,12 +265,18 @@ static BitmapColumnIndex buildBitmapColumnIndexFromIteratorBinarySearch( { return new BaseValueSetIndexesFromIterable(bitmapFactory, bitmaps, getUnknownsIndex) { + @Override + public int estimatedComputeCost() + { + return values.size(); + } + @Override public Iterable getBitmapIterable() { return () -> new Iterator() { - final Iterator iterator = values.iterator(); + final Iterator iterator = values.iterator(); int next = -1; @Override diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldColumnIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldColumnIndexSupplier.java index c784eaa8d13f..d53596964f8b 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldColumnIndexSupplier.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldColumnIndexSupplier.java @@ -158,7 +158,8 @@ public T as(Class clazz) nullIndex = new SimpleImmutableBitmapIndex(bitmapFactory.makeEmptyImmutableBitmap()); } return (T) (NullValueIndex) () -> nullIndex; - } else if (clazz.equals(DictionaryEncodedStringValueIndex.class) || clazz.equals(DictionaryEncodedValueIndex.class)) { + } else if (clazz.equals(DictionaryEncodedStringValueIndex.class) + || clazz.equals(DictionaryEncodedValueIndex.class)) { return (T) new NestedFieldDictionaryEncodedStringValueIndex(); } @@ -402,6 +403,12 @@ public BitmapColumnIndex forValue(@Nullable String value) final FixedIndexed localDictionary = localDictionarySupplier.get(); final Indexed stringDictionary = globalStringDictionarySupplier.get(); + @Override + public int estimatedComputeCost() + { + return 1; + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { @@ -430,6 +437,12 @@ public BitmapColumnIndex forSortedValues(SortedSet values) { return new SimpleImmutableBitmapDelegatingIterableIndex() { + @Override + public int estimatedComputeCost() + { + return values.size(); + } + @Override public Iterable getBitmapIterable() { @@ -548,7 +561,9 @@ public Iterable getBitmapIterable() private int findNext() { - while (currIndex < end && !matcher.apply(StringUtils.fromUtf8Nullable(stringDictionary.get(localDictionary.get(currIndex)))).matches(false)) { + while (currIndex < end + && !matcher.apply(StringUtils.fromUtf8Nullable(stringDictionary.get(localDictionary.get(currIndex)))) + .matches(false)) { currIndex++; } @@ -674,6 +689,12 @@ public BitmapColumnIndex forValue(@Nonnull Object value, TypeSignature localDictionary = localDictionarySupplier.get(); final FixedIndexed globalDictionary = globalLongDictionarySupplier.get(); + @Override + public int estimatedComputeCost() + { + return 1; + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { @@ -715,6 +736,12 @@ public BitmapColumnIndex forValue(@Nullable String value) final FixedIndexed localDictionary = localDictionarySupplier.get(); final FixedIndexed longDictionary = globalLongDictionarySupplier.get(); + @Override + public int estimatedComputeCost() + { + return 1; + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { @@ -750,6 +777,12 @@ public BitmapColumnIndex forSortedValues(SortedSet values) { return new SimpleImmutableBitmapDelegatingIterableIndex() { + @Override + public int estimatedComputeCost() + { + return values.size(); + } + @Override public Iterable getBitmapIterable() { @@ -922,7 +955,8 @@ private void findNext() if (nextValue == 0) { nextSet = longPredicate.applyNull().matches(includeUnknown); } else { - nextSet = longPredicate.applyLong(longDictionary.get(nextValue - adjustLongId)).matches(includeUnknown); + nextSet = longPredicate.applyLong(longDictionary.get(nextValue - adjustLongId)) + .matches(includeUnknown); } } } @@ -953,6 +987,12 @@ public BitmapColumnIndex forValue(@Nonnull Object value, TypeSignature localDictionary = localDictionarySupplier.get(); final FixedIndexed globalDictionary = globalDoubleDictionarySupplier.get(); + @Override + public int estimatedComputeCost() + { + return 1; + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { @@ -993,6 +1033,12 @@ public BitmapColumnIndex forValue(@Nullable String value) final FixedIndexed localDictionary = localDictionarySupplier.get(); final FixedIndexed doubleDictionary = globalDoubleDictionarySupplier.get(); + @Override + public int estimatedComputeCost() + { + return 1; + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { @@ -1028,6 +1074,12 @@ public BitmapColumnIndex forSortedValues(SortedSet values) { return new SimpleImmutableBitmapDelegatingIterableIndex() { + @Override + public int estimatedComputeCost() + { + return values.size(); + } + @Override public Iterable getBitmapIterable() { @@ -1204,6 +1256,9 @@ private abstract class NestedVariantIndexes final FrontCodedIntArrayIndexed arrayDictionary = globalArrayDictionarySupplier == null ? null : globalArrayDictionarySupplier.get(); + // For every single String value, we need to look up indexes from stringDictionary, longDictionary and + // doubleDictionary. Hence, the compute cost for one value is 3. + static final int INDEX_COMPUTE_SCALE = 3; IntList getIndexes(@Nullable String value) { @@ -1256,6 +1311,12 @@ public BitmapColumnIndex forValue(@Nullable String value) { return new SimpleImmutableBitmapDelegatingIterableIndex() { + @Override + public int estimatedComputeCost() + { + return NestedVariantIndexes.INDEX_COMPUTE_SCALE; + } + @Override protected Iterable getBitmapIterable() { @@ -1293,6 +1354,15 @@ public BitmapColumnIndex forSortedValues(SortedSet values) { return new SimpleImmutableBitmapDelegatingIterableIndex() { + @Override + public int estimatedComputeCost() + { + if (values.size() >= Integer.MAX_VALUE / NestedVariantIndexes.INDEX_COMPUTE_SCALE) { + return Integer.MAX_VALUE; + } + return values.size() * NestedVariantIndexes.INDEX_COMPUTE_SCALE; + } + @Override public Iterable getBitmapIterable() { @@ -1504,6 +1574,12 @@ public BitmapColumnIndex forValue(@Nonnull Object value, TypeSignature T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { @@ -1574,6 +1650,12 @@ public BitmapColumnIndex containsValue(@Nullable Object value, TypeSignature T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnAndIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnAndIndexSupplier.java index e535d9cb54b6..59f08e75ea43 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnAndIndexSupplier.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnAndIndexSupplier.java @@ -22,7 +22,7 @@ import com.google.common.base.Strings; import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import com.google.common.primitives.Doubles; import it.unimi.dsi.fastutil.doubles.DoubleArraySet; import it.unimi.dsi.fastutil.doubles.DoubleIterator; @@ -241,10 +241,17 @@ public BitmapColumnIndex forValue(@Nonnull Object value, TypeSignature dictionary = doubleDictionarySupplier.get(); + @Override + public int estimatedComputeCost() + { + return 1; + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { @@ -324,7 +331,7 @@ public BitmapColumnIndex forSortedValues(@Nonnull List sortedValues, TypeSign // values in set are not sorted in double order, transform them on the fly and iterate them all return ValueSetIndexes.buildBitmapColumnIndexFromIteratorBinarySearch( bitmapFactory, - Iterables.transform(sortedValues, DimensionHandlerUtils::convertObjectToDouble), + Lists.transform(sortedValues, DimensionHandlerUtils::convertObjectToDouble), dictionary, valueIndexes, unknownsIndex @@ -345,6 +352,11 @@ public BitmapColumnIndex forValue(@Nullable String value) return new SimpleBitmapColumnIndex() { + @Override + public int estimatedComputeCost() + { + return 1; + } @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) @@ -390,6 +402,12 @@ public BitmapColumnIndex forSortedValues(SortedSet values) { return new SimpleImmutableBitmapDelegatingIterableIndex() { + @Override + public int estimatedComputeCost() + { + return values.size(); + } + @Override public Iterable getBitmapIterable() { diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnAndIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnAndIndexSupplier.java index a9932f7ae3cd..7716f9696e74 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnAndIndexSupplier.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnAndIndexSupplier.java @@ -21,7 +21,7 @@ import com.google.common.base.Supplier; import com.google.common.collect.ImmutableList; -import com.google.common.collect.Iterables; +import com.google.common.collect.Lists; import it.unimi.dsi.fastutil.ints.IntIntPair; import it.unimi.dsi.fastutil.ints.IntIterator; import it.unimi.dsi.fastutil.longs.LongArraySet; @@ -246,6 +246,12 @@ public BitmapColumnIndex forValue(@Nonnull Object value, TypeSignature dictionary = longDictionarySupplier.get(); + @Override + public int estimatedComputeCost() + { + return 1; + } + @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) { @@ -322,7 +328,7 @@ public BitmapColumnIndex forSortedValues(@Nonnull List sortedValues, TypeSign // values in set are not sorted in double order, transform them on the fly and iterate them all return ValueSetIndexes.buildBitmapColumnIndexFromIteratorBinarySearch( bitmapFactory, - Iterables.transform( + Lists.transform( sortedValues, DimensionHandlerUtils::convertObjectToLong ), @@ -346,6 +352,11 @@ public BitmapColumnIndex forValue(@Nullable String value) final Long longValue = GuavaUtils.tryParseLong(value); return new SimpleBitmapColumnIndex() { + @Override + public int estimatedComputeCost() + { + return 1; + } @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) @@ -390,6 +401,12 @@ public BitmapColumnIndex forSortedValues(SortedSet values) { return new SimpleImmutableBitmapDelegatingIterableIndex() { + @Override + public int estimatedComputeCost() + { + return values.size(); + } + @Override public Iterable getBitmapIterable() { diff --git a/processing/src/main/java/org/apache/druid/segment/nested/VariantColumnAndIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/VariantColumnAndIndexSupplier.java index aa1dd143e4e7..3cd9e4f7308a 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/VariantColumnAndIndexSupplier.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/VariantColumnAndIndexSupplier.java @@ -363,6 +363,11 @@ public BitmapColumnIndex forValue(@Nonnull Object value, TypeSignature T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) @@ -428,6 +433,11 @@ public BitmapColumnIndex containsValue(@Nullable Object value, TypeSignature T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java index 9262e8a17303..f82949512345 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java @@ -26,6 +26,8 @@ import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.Iterables; +import it.unimi.dsi.fastutil.ints.IntIntImmutablePair; +import it.unimi.dsi.fastutil.ints.IntIntPair; import org.apache.druid.collections.bitmap.BitmapFactory; import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.common.config.NullHandling; @@ -49,6 +51,8 @@ import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.index.BitmapColumnIndex; +import org.apache.druid.segment.index.DictionaryRangeScanningBitmapIndex; +import org.apache.druid.segment.index.DictionaryScanningBitmapIndex; import org.apache.druid.segment.index.SimpleBitmapColumnIndex; import org.apache.druid.segment.index.SimpleImmutableBitmapDelegatingIterableIndex; import org.apache.druid.segment.index.semantic.DictionaryEncodedStringValueIndex; @@ -139,9 +143,17 @@ public DimensionSelector makeDimensionSelector( ) { if (allowList) { - return ListFilteredDimensionSpec.filterAllowList(values, factory.makeDimensionSelector(delegate), delegate.getExtractionFn() != null); + return ListFilteredDimensionSpec.filterAllowList( + values, + factory.makeDimensionSelector(delegate), + delegate.getExtractionFn() != null + ); } else { - return ListFilteredDimensionSpec.filterDenyList(values, factory.makeDimensionSelector(delegate), delegate.getExtractionFn() != null); + return ListFilteredDimensionSpec.filterDenyList( + values, + factory.makeDimensionSelector(delegate), + delegate.getExtractionFn() != null + ); } } @@ -243,7 +255,8 @@ public T as(Class clazz) return (T) new ListFilteredDruidPredicateIndexes(underlyingIndex, idMapping, nullValueBitmapSupplier); } else if (clazz.equals(LexicographicalRangeIndexes.class)) { return (T) new ListFilteredLexicographicalRangeIndexes(underlyingIndex, idMapping, nullValueBitmapSupplier); - } else if (clazz.equals(DictionaryEncodedStringValueIndex.class) || clazz.equals(DictionaryEncodedValueIndex.class)) { + } else if (clazz.equals(DictionaryEncodedStringValueIndex.class) + || clazz.equals(DictionaryEncodedValueIndex.class)) { return (T) new ListFilteredDictionaryEncodedStringValueIndex(underlyingIndex, idMapping); } return null; @@ -320,7 +333,8 @@ private static Iterable getBitmapsInRange( private int findNext() { - while (currIndex < end && !matcher.apply(delegate.getValue(idMapping.getReverseId(currIndex))).matches(includeUnknown)) { + while (currIndex < end && !matcher.apply(delegate.getValue(idMapping.getReverseId(currIndex))) + .matches(includeUnknown)) { currIndex++; } @@ -400,7 +414,12 @@ int getReverseIndex(@Nullable String value) return -(minIndex + 1); } - Iterable getBitmapsInRange(DruidObjectPredicate matcher, int start, int end, boolean includeUnknown) + Iterable getBitmapsInRange( + DruidObjectPredicate matcher, + int start, + int end, + boolean includeUnknown + ) { return ListFilteredVirtualColumn.getBitmapsInRange(delegate, idMapping, matcher, start, end, includeUnknown); } @@ -420,6 +439,11 @@ public BitmapColumnIndex get() { return new SimpleBitmapColumnIndex() { + @Override + public int estimatedComputeCost() + { + return 0; + } @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknowns) @@ -450,6 +474,11 @@ public BitmapColumnIndex forValue(@Nullable String value) { return new SimpleBitmapColumnIndex() { + @Override + public int estimatedComputeCost() + { + return 1; + } @Override public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) @@ -478,6 +507,12 @@ public BitmapColumnIndex forSortedValues(SortedSet values) { return new SimpleImmutableBitmapDelegatingIterableIndex() { + @Override + public int estimatedComputeCost() + { + return values.size(); + } + @Override public Iterable getBitmapIterable() { @@ -548,22 +583,17 @@ private ListFilteredDruidPredicateIndexes( @Nullable public BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory) { - return new SimpleBitmapColumnIndex() + return new DictionaryScanningBitmapIndex(getCardinality()) { @Override - public T computeBitmapResult(BitmapResultFactory bitmapResultFactory, boolean includeUnknown) + protected Iterable getBitmapIterable(boolean includeUnknown) { - final int start = 0, end = getCardinality(); - Iterable bitmaps; - if (includeUnknown) { - bitmaps = Iterables.concat( - getBitmapsInRange(matcherFactory.makeStringPredicate(), start, end, includeUnknown), - Collections.singletonList(nullValueBitmapSupplier.get()) - ); - } else { - bitmaps = getBitmapsInRange(matcherFactory.makeStringPredicate(), start, end, includeUnknown); - } - return bitmapResultFactory.unionDimensionValueBitmaps(bitmaps); + return Iterables.concat(getBitmapsInRange( + matcherFactory.makeStringPredicate(), + 0, + getCardinality(), + includeUnknown + ), includeUnknown ? Collections.singletonList(nullValueBitmapSupplier.get()) : Collections.emptyList()); } }; } @@ -606,37 +636,13 @@ public BitmapColumnIndex forRange( DruidObjectPredicate matcher ) { - return new SimpleImmutableBitmapDelegatingIterableIndex() + final IntIntPair range = getRange(startValue, startStrict, endValue, endStrict); + final int start = range.leftInt(), end = range.rightInt(); + return new DictionaryRangeScanningBitmapIndex(1.0, end - start) { @Override - public Iterable getBitmapIterable() + protected Iterable getBitmapIterable() { - int startIndex, endIndex; - final int firstValue = NullHandling.isNullOrEquivalent(delegate.getValue(idMapping.getReverseId(0))) ? 1 : 0; - if (startValue == null) { - startIndex = firstValue; - } else { - final int found = getReverseIndex(NullHandling.emptyToNullIfNeeded(startValue)); - if (found >= 0) { - startIndex = startStrict ? found + 1 : found; - } else { - startIndex = -(found + 1); - } - } - - if (endValue == null) { - endIndex = idMapping.getValueCardinality(); - } else { - final int found = getReverseIndex(NullHandling.emptyToNullIfNeeded(endValue)); - if (found >= 0) { - endIndex = endStrict ? found : found + 1; - } else { - endIndex = -(found + 1); - } - } - - endIndex = Math.max(startIndex, endIndex); - final int start = startIndex, end = endIndex; return getBitmapsInRange(matcher, start, end, false); } @@ -648,6 +654,41 @@ protected ImmutableBitmap getUnknownsBitmap() } }; } + + private IntIntPair getRange( + @Nullable String startValue, + boolean startStrict, + @Nullable String endValue, + boolean endStrict + ) + { + int startIndex, endIndex; + final int firstValue = NullHandling.isNullOrEquivalent(delegate.getValue(idMapping.getReverseId(0))) ? 1 : 0; + if (startValue == null) { + startIndex = firstValue; + } else { + final int found = getReverseIndex(NullHandling.emptyToNullIfNeeded(startValue)); + if (found >= 0) { + startIndex = startStrict ? found + 1 : found; + } else { + startIndex = -(found + 1); + } + } + + if (endValue == null) { + endIndex = idMapping.getValueCardinality(); + } else { + final int found = getReverseIndex(NullHandling.emptyToNullIfNeeded(endValue)); + if (found >= 0) { + endIndex = endStrict ? found : found + 1; + } else { + endIndex = -(found + 1); + } + } + + endIndex = Math.max(startIndex, endIndex); + return new IntIntImmutablePair(startIndex, endIndex); + } } private static class ListFilteredDictionaryEncodedStringValueIndex extends BaseListFilteredColumnIndex