Skip to content

Commit

Permalink
Expose multi-valued dates to scripts and document painless's date fun…
Browse files Browse the repository at this point in the history 
…ctions (#22875)

Implemented by wrapping an array of reused `ModuleDateTime`s that
we grow when needed. The `ModuleDateTime`s are reused when we
move to the next document.

Also improves the error message returned when attempting to modify
the `ScriptdocValues`, removes a couple of allocations, and documents
that the date functions are available in Painless.

Relates to #22162
  • Loading branch information
@nik9000
nik9000 authored Feb 2, 2017
1 parent 7f59bed commit dacc150
Show file tree
Hide file tree
Showing 9 changed files with 404 additions and 65 deletions.
176 changes: 133 additions & 43 deletions core/src/main/java/org/elasticsearch/index/fielddata/ScriptDocValues.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -25,32 +25,62 @@
import org.elasticsearch.common.geo.GeoHashUtils;
import org.elasticsearch.common.geo.GeoPoint;
import org.elasticsearch.common.geo.GeoUtils;
import org.joda.time.DateTime;
import org.joda.time.DateTimeZone;
import org.joda.time.MutableDateTime;
import org.joda.time.ReadableDateTime;

import java.util.AbstractList;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.function.UnaryOperator;


/**
* Script level doc values, the assumption is that any implementation will implement a <code>getValue</code>
* and a <code>getValues</code> that return the relevant type that then can be used in scripts.
*/
public interface ScriptDocValues<T> extends List<T> {
public abstract class ScriptDocValues<T> extends AbstractList<T> {

/**
* Set the current doc ID.
*/
void setNextDocId(int docId);
public abstract void setNextDocId(int docId);

/**
* Return a copy of the list of the values for the current document.
*/
List<T> getValues();
public final List<T> getValues() {
return this;
}

// Throw meaningful exceptions if someone tries to modify the ScriptDocValues.
@Override
public final void add(int index, T element) {
throw new UnsupportedOperationException("doc values are unmodifiable");
}

@Override
public final boolean remove(Object o) {
throw new UnsupportedOperationException("doc values are unmodifiable");
}

@Override
public final void replaceAll(UnaryOperator<T> operator) {
throw new UnsupportedOperationException("doc values are unmodifiable");
}

@Override
public final T set(int index, T element) {
throw new UnsupportedOperationException("doc values are unmodifiable");
}

@Override
public final void sort(Comparator<? super T> c) {
throw new UnsupportedOperationException("doc values are unmodifiable");
}

public static final class Strings extends AbstractList<String> implements ScriptDocValues<String> {
public static final class Strings extends ScriptDocValues<String> {

private final SortedBinaryDocValues values;

Expand Down Expand Up @@ -84,11 +114,6 @@ public String getValue() {
}
}

@Override
public List<String> getValues() {
return Collections.unmodifiableList(this);
}

@Override
public String get(int index) {
return values.valueAt(index).utf8ToString();
Expand All @@ -101,10 +126,10 @@ public int size() {

}

public static class Longs extends AbstractList<Long> implements ScriptDocValues<Long> {
public static final class Longs extends ScriptDocValues<Long> {

private final SortedNumericDocValues values;
private final MutableDateTime date = new MutableDateTime(0, DateTimeZone.UTC);
private Dates dates;

public Longs(SortedNumericDocValues values) {
this.values = values;
Expand All @@ -113,6 +138,9 @@ public Longs(SortedNumericDocValues values) {
@Override
public void setNextDocId(int docId) {
values.setDocument(docId);
if (dates != null) {
dates.refreshArray();
}
}

public SortedNumericDocValues getInternalValues() {
Expand All @@ -127,14 +155,20 @@ public long getValue() {
return values.valueAt(0);
}

@Override
public List<Long> getValues() {
return Collections.unmodifiableList(this);
public ReadableDateTime getDate() {
if (dates == null) {
dates = new Dates(values);
dates.refreshArray();
}
return dates.getValue();
}

public ReadableDateTime getDate() {
date.setMillis(getValue());
return date;
public List<ReadableDateTime> getDates() {
if (dates == null) {
dates = new Dates(values);
dates.refreshArray();
}
return dates;
}

@Override
Expand All @@ -146,10 +180,87 @@ public Long get(int index) {
public int size() {
return values.count();
}
}

public static final class Dates extends ScriptDocValues<ReadableDateTime> {
private static final ReadableDateTime EPOCH = new DateTime(0, DateTimeZone.UTC);

private final SortedNumericDocValues values;
/**
* Values wrapped in {@link MutableDateTime}. Null by default an allocated on first usage so we allocate a reasonably size. We keep
* this array so we don't have allocate new {@link MutableDateTime}s on every usage. Instead we reuse them for every document.
*/
private MutableDateTime[] dates;

public Dates(SortedNumericDocValues values) {
this.values = values;
}

/**
* Fetch the first field value or 0 millis after epoch if there are no values.
*/
public ReadableDateTime getValue() {
if (values.count() == 0) {
return EPOCH;
}
return get(0);
}

@Override
public ReadableDateTime get(int index) {
if (index >= values.count()) {
throw new IndexOutOfBoundsException(
"attempted to fetch the [" + index + "] date when there are only [" + values.count() + "] dates.");
}
return dates[index];
}

@Override
public int size() {
return values.count();
}

@Override
public void setNextDocId(int docId) {
values.setDocument(docId);
refreshArray();
}

/**
* Refresh the backing array. Package private so it can be called when {@link Longs} loads dates.
*/
void refreshArray() {
if (values.count() == 0) {
return;
}
if (dates == null) {
// Happens for the document. We delay allocating dates so we can allocate it with a reasonable size.
dates = new MutableDateTime[values.count()];
for (int i = 0; i < dates.length; i++) {
dates[i] = new MutableDateTime(values.valueAt(i), DateTimeZone.UTC);
}
return;
}
if (values.count() > dates.length) {
// Happens when we move to a new document and it has more dates than any documents before it.
MutableDateTime[] backup = dates;
dates = new MutableDateTime[values.count()];
System.arraycopy(backup, 0, dates, 0, backup.length);
for (int i = 0; i < backup.length; i++) {
dates[i].setMillis(values.valueAt(i));
}
for (int i = backup.length; i < dates.length; i++) {
dates[i] = new MutableDateTime(values.valueAt(i), DateTimeZone.UTC);
}
return;
}
for (int i = 0; i < values.count(); i++) {
dates[i].setMillis(values.valueAt(i));
}
}
}

public static class Doubles extends AbstractList<Double> implements ScriptDocValues<Double> {
public static final class Doubles extends ScriptDocValues<Double> {

private final SortedNumericDoubleValues values;

Expand All @@ -174,11 +285,6 @@ public double getValue() {
return values.valueAt(0);
}

@Override
public List<Double> getValues() {
return Collections.unmodifiableList(this);
}

@Override
public Double get(int index) {
return values.valueAt(index);
Expand All @@ -190,7 +296,7 @@ public int size() {
}
}

class GeoPoints extends AbstractList<GeoPoint> implements ScriptDocValues<GeoPoint> {
public static final class GeoPoints extends ScriptDocValues<GeoPoint> {

private final MultiGeoPointValues values;

Expand Down Expand Up @@ -237,11 +343,6 @@ public double getLon() {
return getValue().lon();
}

@Override
public List<GeoPoint> getValues() {
return Collections.unmodifiableList(this);
}

@Override
public GeoPoint get(int index) {
final GeoPoint point = values.valueAt(index);
Expand Down Expand Up @@ -291,7 +392,7 @@ public double geohashDistanceWithDefault(String geohash, double defaultValue) {
}
}

final class Booleans extends AbstractList<Boolean> implements ScriptDocValues<Boolean> {
public static final class Booleans extends ScriptDocValues<Boolean> {

private final SortedNumericDocValues values;

Expand All @@ -304,11 +405,6 @@ public void setNextDocId(int docId) {
values.setDocument(docId);
}

@Override
public List<Boolean> getValues() {
return this;
}

public boolean getValue() {
return values.count() != 0 && values.valueAt(0) == 1;
}
Expand All @@ -325,7 +421,7 @@ public int size() {

}

public static class BytesRefs extends AbstractList<BytesRef> implements ScriptDocValues<BytesRef> {
public static final class BytesRefs extends ScriptDocValues<BytesRef> {

private final SortedBinaryDocValues values;

Expand All @@ -350,11 +446,6 @@ public BytesRef getValue() {
return values.valueAt(0);
}

@Override
public List<BytesRef> getValues() {
return Collections.unmodifiableList(this);
}

@Override
public BytesRef get(int index) {
return values.valueAt(index);
Expand All @@ -365,5 +456,4 @@ public int size() {
return values.count();
}
}

}
11 changes: 2 additions & 9 deletions core/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,9 +46,7 @@

import java.io.IOException;
import java.net.InetAddress;
import java.util.AbstractList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -233,11 +231,11 @@ public FieldStats.Ip stats(IndexReader reader) throws IOException {
InetAddressPoint.decode(min), InetAddressPoint.decode(max));
}

public static final class IpScriptDocValues extends AbstractList<String> implements ScriptDocValues<String> {
public static final class IpScriptDocValues extends ScriptDocValues<String> {

private final RandomAccessOrds values;

IpScriptDocValues(RandomAccessOrds values) {
public IpScriptDocValues(RandomAccessOrds values) {
this.values = values;
}

Expand All @@ -254,11 +252,6 @@ public String getValue() {
}
}

@Override
public List<String> getValues() {
return Collections.unmodifiableList(this);
}

@Override
public String get(int index) {
BytesRef encoded = values.lookupOrd(values.ordAt(0));
Expand Down
Loading

0 comments on commit dacc150

Please sign in to comment.