Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Convert double script to return array #61504

Merged
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -166,6 +166,9 @@ private static String painlessToLoadFromSource(String name, String type) {
return null;
}
StringBuilder b = new StringBuilder();
if ("double".equals(type)) {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The if is temporary and it'll become all the time once it is done.

b.append("List result = new ArrayList();");
}
b.append("def v = source['").append(name).append("'];\n");
b.append("if (v instanceof Iterable) {\n");
b.append(" for (def vv : ((Iterable) v)) {\n");
Expand All @@ -180,6 +183,9 @@ private static String painlessToLoadFromSource(String name, String type) {
b.append(" ").append(emit).append("\n");
b.append(" }\n");
b.append("}\n");
if ("double".equals(type)) {
b.append("return result;");
}
return b.toString();
}

Expand All @@ -188,7 +194,7 @@ private static String painlessToLoadFromSource(String name, String type) {
Map.entry(DateFieldMapper.CONTENT_TYPE, "millis(parse(value.toString()));"),
Map.entry(
NumberType.DOUBLE.typeName(),
"value(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));"
"result.add(value instanceof Number ? ((Number) value).doubleValue() : Double.parseDouble(value.toString()));"
),
Map.entry(KeywordFieldMapper.CONTENT_TYPE, "value(value.toString());"),
Map.entry(IpFieldMapper.CONTENT_TYPE, "stringValue(value.toString());"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ public AbstractLongScriptFieldScript(Map<String, Object> params, SearchLookup se
super(params, searchLookup, ctx);
}

public abstract void execute();
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Because I'm only doing one of them I have to move the old execute method decalarations into the subclasses.


/**
* Execute the script for the provided {@code docId}.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,4 @@ public final Map<String, Object> getSource() {
public final Map<String, ScriptDocValues<?>> getDoc() {
return leafSearchLookup.doc();
}

public abstract void execute();
}
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ public BooleanScriptFieldScript(Map<String, Object> params, SearchLookup searchL
super(params, searchLookup, ctx);
}

public abstract void execute();

/**
* Execute the script for the provided {@code docId}.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,14 +7,14 @@
package org.elasticsearch.xpack.runtimefields;

import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.util.ArrayUtil;
import org.elasticsearch.painless.spi.Whitelist;
import org.elasticsearch.painless.spi.WhitelistLoader;
import org.elasticsearch.script.ScriptContext;
import org.elasticsearch.script.ScriptFactory;
import org.elasticsearch.search.lookup.SearchLookup;

import java.io.IOException;
import java.util.Collection;
import java.util.List;
import java.util.Map;

Expand All @@ -35,55 +35,40 @@ public interface LeafFactory {
DoubleScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException;
}

private double[] values = new double[1];
private int count;

public DoubleScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup, LeafReaderContext ctx) {
super(params, searchLookup, ctx);
}

public abstract double[] execute();

/**
* Execute the script for the provided {@code docId}.
*/
public final void runForDoc(int docId) {
count = 0;
public final double[] runForDoc(int docId) {
setDocument(docId);
execute();
return execute();
}

/**
* Values from the last time {@link #runForDoc(int)} was called. This array
* is mutable and will change with the next call of {@link #runForDoc(int)}.
* It is also oversized and will contain garbage at all indices at and
* above {@link #count()}.
*/
public final double[] values() {
return values;
public static double[] convertFromDouble(double v) {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@stu-elastic and @jdconrad do these look right? I borrowed them from FactoryTests.

I see right now you force the converters to be static - would it be possible to make them non-static on the script?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not easily because the only "this" pointer we have is through class bindings.

return new double[] { v };
}

/**
* The number of results produced the last time {@link #runForDoc(int)} was called.
*/
public final int count() {
return count;
}

private void collectValue(double v) {
if (values.length < count + 1) {
values = ArrayUtil.grow(values, count + 1);
public static double[] convertFromCollection(Collection<?> v) {
double[] result = new double[v.size()];
int i = 0;
for (Object o : v) {
result[i++] = ((Number) o).doubleValue();
}
values[count++] = v;
return result;
}

public static class Value {
private final DoubleScriptFieldScript script;

public Value(DoubleScriptFieldScript script) {
this.script = script;
}

public void value(double v) {
script.collectValue(v);
public static double[] convertFromDef(Object o) {
if (o instanceof Number) {
return convertFromDouble(((Number) o).doubleValue());
} else if (o instanceof Collection) {
return convertFromCollection((Collection<?>) o);
} else {
return (double[]) o;
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

oh boy I was hoping we would not need this sort of stuff, but I guess we do? I mean the instanceof as well as the cast to double array

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

😢 Me too! I imagine the painless folk are thinking about it, but I'm not sure. This seems like a perfect spot for invokedynamic, but I'm not an expert at all.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what happens if we do without this conversion? Really bad I guess?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@javanna without it you can't really return def. You get a lot of funny class cast exceptions.

@stu-elastic or @jdconrad do we have plans indy-ify this or something? So it'd call the converter based on the def type. Is that what #61389 is all about?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We needed to get this in your hands ASAP. We'll be improving it: #61389

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Indy for this is really challenging. We are going to look into it, but allowing you to do def conversions this way ensures we have something for runtime fields now that doesn't use reflection invocation. I would recommend that this cover all numeric cases including byte through long as well. Check out something like DefMath.plus.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Gotcha. So if I implement convertFromDef returning a def type is all on me at the moment. I have to do what DefMath does, basically.

Copy link
Contributor

@jdconrad jdconrad Aug 25, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@nik9000 Yes, that's correct. Again we are going to rectify this, but wanted to make sure we had something that was usable now.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🤘

}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,8 @@ public IpScriptFieldScript(Map<String, Object> params, SearchLookup searchLookup
super(params, searchLookup, ctx);
}

public abstract void execute();

/**
* Execute the script for the provided {@code docId}.
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ public StringScriptFieldScript(Map<String, Object> params, SearchLookup searchLo
super(params, searchLookup, ctx);
}

public abstract void execute();

/**
* Execute the script for the provided {@code docId}.
* <p>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@

public final class ScriptDoubleDocValues extends SortedNumericDoubleValues {
private final DoubleScriptFieldScript script;
private double[] values;
private int cursor;

ScriptDoubleDocValues(DoubleScriptFieldScript script) {
Expand All @@ -22,22 +23,22 @@ public final class ScriptDoubleDocValues extends SortedNumericDoubleValues {

@Override
public boolean advanceExact(int docId) {
script.runForDoc(docId);
if (script.count() == 0) {
values = script.runForDoc(docId);
if (values.length == 0) {
return false;
}
Arrays.sort(script.values(), 0, script.count());
Arrays.sort(values);
cursor = 0;
return true;
}

@Override
public double nextValue() throws IOException {
return script.values()[cursor++];
return values[cursor++];
}

@Override
public int docValueCount() {
return script.count();
return values.length;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ abstract class AbstractDoubleScriptFieldQuery extends AbstractScriptFieldQuery {
/**
* Does the value match this query?
*/
protected abstract boolean matches(double[] values, int count);
protected abstract boolean matches(double[] values);
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We don't need the count any more because we use the whole array.


@Override
public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
Expand All @@ -53,8 +53,7 @@ public Scorer scorer(LeafReaderContext ctx) throws IOException {
TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) {
@Override
public boolean matches() throws IOException {
script.runForDoc(approximation().docID());
return AbstractDoubleScriptFieldQuery.this.matches(script.values(), script.count());
return AbstractDoubleScriptFieldQuery.this.matches(script.runForDoc(approximation().docID()));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ public DoubleScriptFieldExistsQuery(Script script, DoubleScriptFieldScript.LeafF
}

@Override
protected boolean matches(double[] values, int count) {
return count > 0;
protected boolean matches(double[] values) {
return values.length > 0;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ public DoubleScriptFieldRangeQuery(
}

@Override
protected boolean matches(double[] values, int count) {
for (int i = 0; i < count; i++) {
if (lowerValue <= values[i] && values[i] <= upperValue) {
protected boolean matches(double[] values) {
for (double value : values) {
if (lowerValue <= value && value <= upperValue) {
return true;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,9 @@ public DoubleScriptFieldTermQuery(Script script, DoubleScriptFieldScript.LeafFac
}

@Override
protected boolean matches(double[] values, int count) {
for (int i = 0; i < count; i++) {
if (term == values[i]) {
protected boolean matches(double[] values) {
for (double value : values) {
if (term == value) {
return true;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@ public DoubleScriptFieldTermsQuery(Script script, DoubleScriptFieldScript.LeafFa
}

@Override
protected boolean matches(double[] values, int count) {
for (int i = 0; i < count; i++) {
if (terms.contains(Double.doubleToLongBits(values[i]))) {
protected boolean matches(double[] values) {
for (double value : values) {
if (terms.contains(Double.doubleToLongBits(value))) {
return true;
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,6 @@
class org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript @no_import {
}

static_import {
void value(org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript, double) bound_to org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript$Value
}

# This import is required to make painless happy and it isn't 100% clear why
# This whitelist is required to allow painless to build the factory
class org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript$Factory @no_import {
}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was hoping that this file would go away completely. can you remind what the remaining lines are for?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll try one more time to drop it. I tried and it blew up. But I'll try without it entirely.

I think these are required so painless can use the classes that we defined. But these might should all be "implied" because they are part of the script context.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I checked - it is still required.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@stu-elastic and @jdconrad - I think we still need this file because without it painless can't find the factory class. I think that is because is part of a plugin and not in painless's classloader. Or something like that. Is that how the world is supposed to be? Could these classes be implied by default because the context mentions them directly?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Painless has no way to load these classes w/o explicitly stating them as part of SPI.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍.

Could you dig them out of the ScriptContext, I wonder?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That's something worth looking into. Agreed.

Original file line number Diff line number Diff line change
Expand Up @@ -8,15 +8,19 @@

import org.elasticsearch.script.ScriptContext;

import java.util.List;

import static org.hamcrest.Matchers.equalTo;

public class DoubleScriptFieldScriptTests extends ScriptFieldScriptTestCase<DoubleScriptFieldScript.Factory> {
public static final DoubleScriptFieldScript.Factory DUMMY = (params, lookup) -> ctx -> new DoubleScriptFieldScript(
params,
lookup,
ctx
) {
@Override
public void execute() {
new DoubleScriptFieldScript.Value(this).value(1.0);
public double[] execute() {
return new double[] { 1.0 };
}
};

Expand All @@ -29,4 +33,29 @@ protected ScriptContext<DoubleScriptFieldScript.Factory> context() {
protected DoubleScriptFieldScript.Factory dummyScript() {
return DUMMY;
}

public void testConvertDouble() {
double v = randomDouble();
assertThat(DoubleScriptFieldScript.convertFromDouble(v), equalTo(new double[] { v }));
assertThat(DoubleScriptFieldScript.convertFromDef(v), equalTo(new double[] { v }));
}

public void testConvertFromCollection() {
double d = randomDouble();
long l = randomLong();
int i = randomInt();
assertThat(DoubleScriptFieldScript.convertFromCollection(List.of(d, l, i)), equalTo(new double[] { d, l, i }));
assertThat(DoubleScriptFieldScript.convertFromDef(List.of(d, l, i)), equalTo(new double[] { d, l, i }));
}

public void testConvertDoubleArrayFromDef() {
double[] a = new double[] { 1, 2, 3 };
assertThat(DoubleScriptFieldScript.convertFromDef(a), equalTo(a));
}

public void testConvertNumberFromDef() {
for (Number n : new Number[] { randomByte(), randomShort(), randomInt(), randomLong(), randomFloat() }) {
assertThat(DoubleScriptFieldScript.convertFromDef(n), equalTo(new double[] { n.doubleValue() }));
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -270,21 +270,27 @@ private DoubleScriptFieldScript.Factory factory(String code) {
case "read_foo":
return (params, lookup) -> (ctx) -> new DoubleScriptFieldScript(params, lookup, ctx) {
@Override
public void execute() {
for (Object foo : (List<?>) getSource().get("foo")) {
new DoubleScriptFieldScript.Value(this).value(((Number) foo).doubleValue());
public double[] execute() {
List<?> foos = (List<?>) getSource().get("foo");
double[] results = new double[foos.size()];
int i = 0;
for (Object foo : foos) {
results[i++] = ((Number) foo).doubleValue();
}
return results;
}
};
case "add_param":
return (params, lookup) -> (ctx) -> new DoubleScriptFieldScript(params, lookup, ctx) {
@Override
public void execute() {
for (Object foo : (List<?>) getSource().get("foo")) {
new DoubleScriptFieldScript.Value(this).value(
((Number) foo).doubleValue() + ((Number) getParams().get("param")).doubleValue()
);
public double[] execute() {
List<?> foos = (List<?>) getSource().get("foo");
double[] results = new double[foos.size()];
int i = 0;
for (Object foo : foos) {
results[i++] = ((Number) getParams().get("param")).doubleValue() + ((Number) foo).doubleValue();
}
return results;
}
};
default:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,13 @@ protected DoubleScriptFieldExistsQuery mutate(DoubleScriptFieldExistsQuery orig)

@Override
public void testMatches() {
assertTrue(createTestInstance().matches(new double[0], randomIntBetween(1, Integer.MAX_VALUE)));
assertFalse(createTestInstance().matches(new double[0], 0));
assertFalse(createTestInstance().matches(new double[] { 1, 2, 3 }, 0));
assertTrue(createTestInstance().matches(new double[] { 1 }));
assertFalse(createTestInstance().matches(new double[0]));
double[] big = new double[between(1, 10000)];
for (int i = 0; i < big.length; i++) {
big[i] = 1.0;
}
assertTrue(createTestInstance().matches(big));
}

@Override
Expand Down
Loading