Skip to content

Commit

Permalink
Add estimatedComputeCost for all BitmapColumnIndex classes.
Browse files Browse the repository at this point in the history
  • Loading branch information
cecemei committed Sep 20, 2024
1 parent 635e418 commit e2ffa6a
Show file tree
Hide file tree
Showing 18 changed files with 324 additions and 78 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -149,8 +149,6 @@ private BitmapColumnIndex wrapRangeIndexWithNullValueIndex(
BitmapColumnIndex rangeIndex
)
{


final BitmapColumnIndex nullBitmap;
final NullValueIndex nulls = indexSupplier.as(NullValueIndex.class);
if (nulls == null) {
Expand All @@ -166,6 +164,12 @@ public ColumnIndexCapabilities getIndexCapabilities()
return rangeIndex.getIndexCapabilities().merge(nullBitmap.getIndexCapabilities());
}

@Override
public int estimatedComputeCost()
{
return rangeIndex.estimatedComputeCost();
}

@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,12 @@ public ColumnIndexCapabilities getIndexCapabilities()
return baseIndex.getIndexCapabilities();
}

@Override
public int estimatedComputeCost()
{
return baseIndex.estimatedComputeCost();
}

@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,12 @@ public ColumnIndexCapabilities getIndexCapabilities()
return baseIndex.getIndexCapabilities();
}

@Override
public int estimatedComputeCost()
{
return baseIndex.estimatedComputeCost();
}

@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -94,6 +94,12 @@ public ColumnIndexCapabilities getIndexCapabilities()
return new SimpleColumnIndexCapabilities(true, true);
}

@Override
public int estimatedComputeCost()
{
return Integer.MAX_VALUE;
}

@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,12 @@ public ColumnIndexCapabilities getIndexCapabilities()
return SimpleColumnIndexCapabilities.getConstant();
}

@Override
public int estimatedComputeCost()
{
return 0;
}

@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,12 @@ public ColumnIndexCapabilities getIndexCapabilities()
return SimpleColumnIndexCapabilities.getConstant();
}

@Override
public int estimatedComputeCost()
{
return 0;
}

@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,12 @@ public ColumnIndexCapabilities getIndexCapabilities()
return SimpleColumnIndexCapabilities.getConstant();
}

@Override
public int estimatedComputeCost()
{
return 0;
}

@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,10 +38,7 @@ public interface BitmapColumnIndex
/**
* Returns an estimated cost for computing the bitmap result.
*/
default int estimatedComputeCost()
{
return Integer.MAX_VALUE;
}
int estimatedComputeCost();

/**
* Compute a bitmap result wrapped with the {@link BitmapResultFactory} representing the rows matched by this index.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,12 @@ public DictionaryRangeScanningBitmapIndex(double sizeScale, int rangeSize)
this.rangeSize = rangeSize;
}

@Override
public int estimatedComputeCost()
{
return this.rangeSize;
}

@Nullable
@Override
public final <T> T computeBitmapResult(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,12 @@ public DictionaryScanningBitmapIndex(int dictionarySize, double scaleThreshold)
this.scaleThreshold = scaleThreshold;
}

@Override
public int estimatedComputeCost()
{
return this.dictionarySize;
}

@Nullable
@Override
public final <T> T computeBitmapResult(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
import com.google.common.base.Preconditions;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
import com.google.common.collect.Lists;
import org.apache.druid.annotations.SuppressFBWarnings;
import org.apache.druid.collections.bitmap.BitmapFactory;
import org.apache.druid.collections.bitmap.ImmutableBitmap;
Expand All @@ -48,6 +48,7 @@
import java.util.Comparator;
import java.util.List;
import java.util.SortedSet;
import java.util.stream.Collectors;

public final class IndexedUtf8ValueIndexes<TDictionary extends Indexed<ByteBuffer>>
implements StringValueSetIndexes, Utf8ValueSetIndexes, ValueIndexes, ValueSetIndexes
Expand Down Expand Up @@ -83,6 +84,11 @@ public BitmapColumnIndex forValue(@Nullable String value)
final ByteBuffer utf8 = StringUtils.toUtf8ByteBuffer(value);
return new SimpleBitmapColumnIndex()
{
@Override
public int estimatedComputeCost()
{
return 1;
}

@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
Expand Down Expand Up @@ -122,10 +128,7 @@ public BitmapColumnIndex forValue(@Nonnull Object value, TypeSignature<ValueType
public BitmapColumnIndex forSortedValues(SortedSet<String> values)
{
return getBitmapColumnIndexForSortedIterableUtf8(
Iterables.transform(
values,
StringUtils::toUtf8ByteBuffer
),
values.stream().map(StringUtils::toUtf8ByteBuffer).collect(Collectors.toList()),
values.size(),
values.contains(null)
);
Expand Down Expand Up @@ -170,7 +173,7 @@ private ImmutableBitmap getBitmap(int idx)
* Helper used by {@link #forSortedValues} and {@link #forSortedValuesUtf8}.
*/
private BitmapColumnIndex getBitmapColumnIndexForSortedIterableUtf8(
Iterable<ByteBuffer> valuesUtf8,
List<ByteBuffer> valuesUtf8,
int size,
boolean valuesContainsNull
)
Expand Down Expand Up @@ -241,7 +244,7 @@ public BitmapColumnIndex forSortedValues(@Nonnull List<?> sortedValues, TypeSign
return ValueSetIndexes.buildBitmapColumnIndexFromSortedIteratorScan(
bitmapFactory,
ByteBufferUtils.utf8Comparator(),
Iterables.transform(tailSet, StringUtils::toUtf8ByteBuffer),
Lists.transform(tailSet, StringUtils::toUtf8ByteBuffer),
dictionary,
bitmaps,
unknownsIndex
Expand All @@ -250,15 +253,15 @@ public BitmapColumnIndex forSortedValues(@Nonnull List<?> sortedValues, TypeSign
// fall through to value iteration
return ValueSetIndexes.buildBitmapColumnIndexFromSortedIteratorBinarySearch(
bitmapFactory,
Iterables.transform(tailSet, StringUtils::toUtf8ByteBuffer),
Lists.transform(tailSet, StringUtils::toUtf8ByteBuffer),
dictionary,
bitmaps,
unknownsIndex
);
} else {
return ValueSetIndexes.buildBitmapColumnIndexFromIteratorBinarySearch(
bitmapFactory,
Iterables.transform(
Lists.transform(
sortedValues,
x -> StringUtils.toUtf8ByteBuffer(DimensionHandlerUtils.convertObjectToString(x))
),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,12 @@ public SimpleImmutableBitmapIndex(ImmutableBitmap bitmap)
this.bitmap = bitmap;
}

@Override
public int estimatedComputeCost()
{
return 0;
}

@Override
public <T> T computeBitmapResult(BitmapResultFactory<T> bitmapResultFactory, boolean includeUnknown)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -63,8 +63,8 @@ public interface ValueSetIndexes
* @param sortedValues values to match, sorted in matchValueType order
* @param matchValueType type of the value to match, used to assist conversion from the match value type to the column
* value type
* @return {@link ImmutableBitmap} corresponding to the rows which match the values, or null if an index
* connot be computed for the supplied value type
* @return {@link ImmutableBitmap} corresponding to the rows which match the values, or null if an index
* connot be computed for the supplied value type
*/
@Nullable
BitmapColumnIndex forSortedValues(@Nonnull List<?> sortedValues, TypeSignature<ValueType> matchValueType);
Expand All @@ -78,24 +78,31 @@ public interface ValueSetIndexes
* {@link Indexed<T>} value dictionary. Uses a strategy that does zipping similar to the merge step of a sort-merge,
* where we step forward on both the iterator and the dictionary to find matches to build a
* {@link Iterable<ImmutableBitmap>}.
* <p>
* <p>
* If sorted match value iterator size is greater than (dictionary size * {@link #SORTED_SCAN_RATIO_THRESHOLD}),
* consider using this method instead of {@link #buildBitmapColumnIndexFromSortedIteratorBinarySearch}.
* <p>
* <p>
* If the values in the iterator are NOT sorted the same as the dictionary, do NOT use this method, use
* {@link #buildBitmapColumnIndexFromIteratorBinarySearch} instead.
*/
static <T> BitmapColumnIndex buildBitmapColumnIndexFromSortedIteratorScan(
BitmapFactory bitmapFactory,
Comparator<T> comparator,
Iterable<T> values,
List<? extends T> values,
Indexed<T> dictionary,
Indexed<ImmutableBitmap> bitmaps,
Supplier<ImmutableBitmap> unknownsBitmap
)
{
return new BaseValueSetIndexesFromIterable(bitmapFactory, bitmaps, unknownsBitmap)
{

@Override
public int estimatedComputeCost()
{
return Integer.max(values.size(), dictionary.size());
}

@Override
public Iterable<ImmutableBitmap> getBitmapIterable()
{
Expand Down Expand Up @@ -163,20 +170,27 @@ private void findNext()
* <p>
* If sorted match value iterator size is less than (dictionary size * {@link #SORTED_SCAN_RATIO_THRESHOLD}),
* consider using this method instead of {@link #buildBitmapColumnIndexFromSortedIteratorScan}.
* <p>
* <p>
* If the values in the iterator are not sorted the same as the dictionary, do not use this method, use
* {@link #buildBitmapColumnIndexFromIteratorBinarySearch} instead.
*/
static <T> BitmapColumnIndex buildBitmapColumnIndexFromSortedIteratorBinarySearch(
BitmapFactory bitmapFactory,
Iterable<T> values,
List<T> values,
Indexed<T> dictionary,
Indexed<ImmutableBitmap> bitmaps,
Supplier<ImmutableBitmap> getUnknownsIndex
)
{
return new BaseValueSetIndexesFromIterable(bitmapFactory, bitmaps, getUnknownsIndex)
{

@Override
public int estimatedComputeCost()
{
return values.size();
}

@Override
public Iterable<ImmutableBitmap> getBitmapIterable()
{
Expand Down Expand Up @@ -236,27 +250,33 @@ private void findNext()
* {@link Indexed<T>} value dictionary. This algorithm iterates the values to match and does a binary search for
* matching values using {@link Indexed#indexOf(Object)} to build a {@link Iterable<ImmutableBitmap>} until the match
* values iterator is exhausted.
* <p>
* <p>
* If values of the iterator are sorted the same as the dictionary, use
* {@link #buildBitmapColumnIndexFromSortedIteratorScan} or
* {@link #buildBitmapColumnIndexFromSortedIteratorBinarySearch} instead.
*/
static <T> BitmapColumnIndex buildBitmapColumnIndexFromIteratorBinarySearch(
BitmapFactory bitmapFactory,
Iterable<T> values,
List<? extends T> values,
Indexed<T> dictionary,
Indexed<ImmutableBitmap> bitmaps,
Supplier<ImmutableBitmap> getUnknownsIndex
)
{
return new BaseValueSetIndexesFromIterable(bitmapFactory, bitmaps, getUnknownsIndex)
{
@Override
public int estimatedComputeCost()
{
return values.size();
}

@Override
public Iterable<ImmutableBitmap> getBitmapIterable()
{
return () -> new Iterator<ImmutableBitmap>()
{
final Iterator<T> iterator = values.iterator();
final Iterator<? extends T> iterator = values.iterator();
int next = -1;

@Override
Expand Down
Loading

0 comments on commit e2ffa6a

Please sign in to comment.