Skip to content

Commit

Permalink
ESQL: Add boolean support to Max and Min aggs (elastic#110527)
Browse files Browse the repository at this point in the history
- Added support for Booleans on Max and Min
- Added some helper methods to BitArray (`set(index, value)` and `fill(from, to, value)`). This way, the container is more similar to other BigArrays, and it's easier to work with

Part of elastic#110346, as Max
and Min are dependencies of Top.
  • Loading branch information
ivancea authored Jul 10, 2024
1 parent 688870c commit 2901711
Show file tree
Hide file tree
Showing 43 changed files with 1,537 additions and 88 deletions.
5 changes: 5 additions & 0 deletions docs/changelog/110527.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
pr: 110527
summary: "ESQL: Add boolean support to Max and Min aggs"
area: ES|QL
type: feature
issues: []
2 changes: 1 addition & 1 deletion docs/reference/esql/functions/description/max.asciidoc

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion docs/reference/esql/functions/description/min.asciidoc

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 17 additions & 5 deletions docs/reference/esql/functions/kibana/definition/max.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

22 changes: 17 additions & 5 deletions docs/reference/esql/functions/kibana/definition/min.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion docs/reference/esql/functions/kibana/docs/max.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion docs/reference/esql/functions/kibana/docs/min.md

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion docs/reference/esql/functions/parameters/max.asciidoc

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion docs/reference/esql/functions/parameters/min.asciidoc

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion docs/reference/esql/functions/signature/max.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
2 changes: 1 addition & 1 deletion docs/reference/esql/functions/signature/min.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
3 changes: 2 additions & 1 deletion docs/reference/esql/functions/types/max.asciidoc

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 2 additions & 1 deletion docs/reference/esql/functions/types/min.asciidoc

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

73 changes: 73 additions & 0 deletions server/src/main/java/org/elasticsearch/common/util/BitArray.java
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,17 @@ public void writeTo(StreamOutput out) throws IOException {
bits.writeTo(out);
}

/**
* Set or clear the {@code index}th bit based on the specified value.
*/
public void set(long index, boolean value) {
if (value) {
set(index);
} else {
clear(index);
}
}

/**
* Set the {@code index}th bit.
*/
Expand Down Expand Up @@ -158,6 +169,68 @@ public boolean get(long index) {
return (bits.get(wordNum) & bitmask) != 0;
}

/**
* Set or clear slots between {@code fromIndex} inclusive to {@code toIndex} based on {@code value}.
*/
public void fill(long fromIndex, long toIndex, boolean value) {
if (fromIndex > toIndex) {
throw new IllegalArgumentException("From should be less than or equal to toIndex");
}
long currentSize = size();
if (value == false) {
// There's no need to grow the array just to clear bits.
toIndex = Math.min(toIndex, currentSize);
}
if (fromIndex == toIndex) {
return; // Empty range
}

if (toIndex > currentSize) {
bits = bigArrays.grow(bits, wordNum(toIndex) + 1);
}

int wordLength = Long.BYTES * Byte.SIZE;
long fullWord = 0xFFFFFFFFFFFFFFFFL;

long firstWordIndex = fromIndex % wordLength;
long lastWordIndex = toIndex % wordLength;

long firstWordNum = wordNum(fromIndex);
long lastWordNum = wordNum(toIndex - 1);

// Mask first word
if (firstWordIndex > 0) {
long mask = fullWord << firstWordIndex;

if (firstWordNum == lastWordNum) {
mask &= fullWord >>> (wordLength - lastWordIndex);
}

if (value) {
bits.set(firstWordNum, bits.get(firstWordNum) | mask);
} else {
bits.set(firstWordNum, bits.get(firstWordNum) & ~mask);
}

firstWordNum++;
}

// Mask last word
if (firstWordNum <= lastWordNum) {
long mask = fullWord >>> (wordLength - lastWordIndex);

if (value) {
bits.set(lastWordNum, bits.get(lastWordNum) | mask);
} else {
bits.set(lastWordNum, bits.get(lastWordNum) & ~mask);
}
}

if (firstWordNum < lastWordNum) {
bits.fill(firstWordNum, lastWordNum, value ? fullWord : 0L);
}
}

public long size() {
return bits.size() * (long) Long.BYTES * Byte.SIZE;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,27 @@ public void testRandom() {
}
}

public void testRandomSetValue() {
try (BitArray bitArray = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE)) {
int numBits = randomIntBetween(1000, 10000);
for (int step = 0; step < 3; step++) {
boolean[] bits = new boolean[numBits];
List<Integer> slots = new ArrayList<>();
for (int i = 0; i < numBits; i++) {
bits[i] = randomBoolean();
slots.add(i);
}
Collections.shuffle(slots, random());
for (int i : slots) {
bitArray.set(i, bits[i]);
}
for (int i = 0; i < numBits; i++) {
assertEquals(bitArray.get(i), bits[i]);
}
}
}
}

public void testVeryLarge() {
assumeThat(Runtime.getRuntime().maxMemory(), greaterThanOrEqualTo(ByteSizeUnit.MB.toBytes(512)));
try (BitArray bitArray = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE)) {
Expand Down Expand Up @@ -183,6 +204,78 @@ public void testGetAndSet() {
}
}

public void testFillTrueRandom() {
try (BitArray bitArray = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE)) {
int from = randomIntBetween(0, 1000);
int to = randomIntBetween(from, 1000);

bitArray.fill(0, 1000, false);
bitArray.fill(from, to, true);

for (int i = 0; i < 1000; i++) {
if (i < from || i >= to) {
assertFalse(bitArray.get(i));
} else {
assertTrue(bitArray.get(i));
}
}
}
}

public void testFillFalseRandom() {
try (BitArray bitArray = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE)) {
int from = randomIntBetween(0, 1000);
int to = randomIntBetween(from, 1000);

bitArray.fill(0, 1000, true);
bitArray.fill(from, to, false);

for (int i = 0; i < 1000; i++) {
if (i < from || i >= to) {
assertTrue(bitArray.get(i));
} else {
assertFalse(bitArray.get(i));
}
}
}
}

public void testFillTrueSingleWord() {
try (BitArray bitArray = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE)) {
int from = 8;
int to = 56;

bitArray.fill(0, 64, false);
bitArray.fill(from, to, true);

for (int i = 0; i < 64; i++) {
if (i < from || i >= to) {
assertFalse(bitArray.get(i));
} else {
assertTrue(bitArray.get(i));
}
}
}
}

public void testFillFalseSingleWord() {
try (BitArray bitArray = new BitArray(1, BigArrays.NON_RECYCLING_INSTANCE)) {
int from = 8;
int to = 56;

bitArray.fill(0, 64, true);
bitArray.fill(from, to, false);

for (int i = 0; i < 64; i++) {
if (i < from || i >= to) {
assertTrue(bitArray.get(i));
} else {
assertFalse(bitArray.get(i));
}
}
}
}

public void testSerialize() throws Exception {
int initial = randomIntBetween(1, 100_000);
BitArray bits1 = new BitArray(initial, BigArrays.NON_RECYCLING_INSTANCE);
Expand Down
Loading

0 comments on commit 2901711

Please sign in to comment.