diff --git a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java index 9f42420fe1aa9..ef96bbefb08f4 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java @@ -54,6 +54,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.function.BiFunction; import java.util.function.Supplier; /** A {@link FieldMapper} for ip addresses. */ @@ -155,7 +156,7 @@ public String typeName() { return CONTENT_TYPE; } - private InetAddress parse(Object value) { + private static InetAddress parse(Object value) { if (value instanceof InetAddress) { return (InetAddress) value; } else { @@ -221,6 +222,26 @@ public Query termsQuery(List values, QueryShardContext context) { @Override public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper, QueryShardContext context) { failIfNotIndexed(); + return rangeQuery( + lowerTerm, + upperTerm, + includeLower, + includeUpper, + (lower, upper) -> InetAddressPoint.newRangeQuery(name(), lower, upper) + ); + } + + /** + * Processes query bounds into {@code long}s and delegates the + * provided {@code builder} to build a range query. + */ + public static Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + BiFunction builder + ) { InetAddress lower; if (lowerTerm == null) { lower = InetAddressPoint.MIN_VALUE; @@ -247,7 +268,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower } } - return InetAddressPoint.newRangeQuery(name(), lower, upper); + return builder.apply(lower, upper); } public static final class IpScriptDocValues extends ScriptDocValues { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/IpScriptFieldScript.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/IpScriptFieldScript.java new file mode 100644 index 0000000000000..32ac41e66853d --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/IpScriptFieldScript.java @@ -0,0 +1,113 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.util.ArrayUtil; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.painless.spi.Whitelist; +import org.elasticsearch.painless.spi.WhitelistLoader; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptFactory; +import org.elasticsearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.net.Inet4Address; +import java.net.Inet6Address; +import java.net.InetAddress; +import java.util.List; +import java.util.Map; + +/** + * Script producing IP addresses. Unlike the other {@linkplain AbstractScriptFieldScript}s + * which deal with their native java objects this converts its values to the same format + * that Lucene uses to store its fields, {@link InetAddressPoint}. There are a few compelling + * reasons to do this: + * + */ +public abstract class IpScriptFieldScript extends AbstractScriptFieldScript { + public static final ScriptContext CONTEXT = new ScriptContext<>("ip_script_field", Factory.class); + + static List whitelist() { + return List.of(WhitelistLoader.loadFromResourceFiles(RuntimeFieldsPainlessExtension.class, "ip_whitelist.txt")); + } + + public static final String[] PARAMETERS = {}; + + public interface Factory extends ScriptFactory { + LeafFactory newFactory(Map params, SearchLookup searchLookup); + } + + public interface LeafFactory { + IpScriptFieldScript newInstance(LeafReaderContext ctx) throws IOException; + } + + private BytesRef[] values = new BytesRef[1]; + private int count; + + public IpScriptFieldScript(Map params, SearchLookup searchLookup, LeafReaderContext ctx) { + super(params, searchLookup, ctx); + } + + /** + * Execute the script for the provided {@code docId}. + */ + public final void runForDoc(int docId) { + count = 0; + setDocument(docId); + execute(); + } + + /** + * Values from the last time {@link #runForDoc(int)} was called. This array + * is mutable and will change with the next call of {@link #runForDoc(int)}. + * It is also oversized and will contain garbage at all indices at and + * above {@link #count()}. + *

+ * All values are IPv6 addresses so they are 16 bytes. IPv4 addresses are + * encoded by rfc4291. + */ + public final BytesRef[] values() { + return values; + } + + /** + * The number of results produced the last time {@link #runForDoc(int)} was called. + */ + public final int count() { + return count; + } + + private void collectValue(String v) { + if (values.length < count + 1) { + values = ArrayUtil.grow(values, count + 1); + } + values[count++] = new BytesRef(InetAddressPoint.encode(InetAddresses.forString(v))); + } + + public static class StringValue { + private final IpScriptFieldScript script; + + public StringValue(IpScriptFieldScript script) { + this.script = script; + } + + public void stringValue(String v) { + script.collectValue(v); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java index b00168f500a10..4aa72b4fcd815 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFields.java @@ -29,6 +29,7 @@ public List> getContexts() { return List.of( DateScriptFieldScript.CONTEXT, DoubleScriptFieldScript.CONTEXT, + IpScriptFieldScript.CONTEXT, LongScriptFieldScript.CONTEXT, StringScriptFieldScript.CONTEXT ); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java index 92508058750f8..3bab6de8c60a1 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/RuntimeFieldsPainlessExtension.java @@ -19,6 +19,7 @@ public Map, List> getContextWhitelists() { return Map.ofEntries( Map.entry(DateScriptFieldScript.CONTEXT, DateScriptFieldScript.whitelist()), Map.entry(DoubleScriptFieldScript.CONTEXT, DoubleScriptFieldScript.whitelist()), + Map.entry(IpScriptFieldScript.CONTEXT, IpScriptFieldScript.whitelist()), Map.entry(LongScriptFieldScript.CONTEXT, LongScriptFieldScript.whitelist()), Map.entry(StringScriptFieldScript.CONTEXT, StringScriptFieldScript.whitelist()) ); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java index b115742917dcb..7db0d6bd79b4d 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryFieldData.java @@ -11,46 +11,18 @@ import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.common.util.BigArrays; import org.elasticsearch.index.fielddata.IndexFieldData; -import org.elasticsearch.index.fielddata.IndexFieldDataCache; import org.elasticsearch.index.fielddata.LeafFieldData; -import org.elasticsearch.index.fielddata.ScriptDocValues; -import org.elasticsearch.index.fielddata.SortedBinaryDocValues; import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource; -import org.elasticsearch.index.mapper.MapperService; -import org.elasticsearch.indices.breaker.CircuitBreakerService; import org.elasticsearch.search.DocValueFormat; import org.elasticsearch.search.MultiValueMode; -import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; import org.elasticsearch.search.sort.BucketedSort; import org.elasticsearch.search.sort.SortOrder; -import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; - -import java.io.IOException; - -public final class ScriptBinaryFieldData implements IndexFieldData { - - public static class Builder implements IndexFieldData.Builder { - private final String name; - private final StringScriptFieldScript.LeafFactory leafFactory; - - public Builder(String name, StringScriptFieldScript.LeafFactory leafFactory) { - this.name = name; - this.leafFactory = leafFactory; - } - - @Override - public ScriptBinaryFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { - return new ScriptBinaryFieldData(name, leafFactory); - } - } +public abstract class ScriptBinaryFieldData implements IndexFieldData { private final String fieldName; - private final StringScriptFieldScript.LeafFactory leafFactory; - private ScriptBinaryFieldData(String fieldName, StringScriptFieldScript.LeafFactory leafFactory) { + protected ScriptBinaryFieldData(String fieldName) { this.fieldName = fieldName; - this.leafFactory = leafFactory; } @Override @@ -58,11 +30,6 @@ public String getFieldName() { return fieldName; } - @Override - public ValuesSourceType getValuesSourceType() { - return CoreValuesSourceType.BYTES; - } - @Override public ScriptBinaryLeafFieldData load(LeafReaderContext context) { try { @@ -72,11 +39,6 @@ public ScriptBinaryLeafFieldData load(LeafReaderContext context) { } } - @Override - public ScriptBinaryLeafFieldData loadDirect(LeafReaderContext context) throws IOException { - return new ScriptBinaryLeafFieldData(new ScriptBinaryDocValues(leafFactory.newInstance(context))); - } - @Override public SortField sortField(Object missingValue, MultiValueMode sortMode, XFieldComparatorSource.Nested nested, boolean reverse) { final XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue, sortMode, nested); @@ -97,23 +59,7 @@ public BucketedSort newBucketedSort( throw new IllegalArgumentException("only supported on numeric fields"); } - public static class ScriptBinaryLeafFieldData implements LeafFieldData { - private final ScriptBinaryDocValues scriptBinaryDocValues; - - ScriptBinaryLeafFieldData(ScriptBinaryDocValues scriptBinaryDocValues) { - this.scriptBinaryDocValues = scriptBinaryDocValues; - } - - @Override - public ScriptDocValues getScriptValues() { - return new ScriptDocValues.Strings(getBytesValues()); - } - - @Override - public SortedBinaryDocValues getBytesValues() { - return scriptBinaryDocValues; - } - + public abstract class ScriptBinaryLeafFieldData implements LeafFieldData { @Override public long ramBytesUsed() { return 0; diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpDocValues.java new file mode 100644 index 0000000000000..0cecde2302d41 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpDocValues.java @@ -0,0 +1,44 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; + +import java.io.IOException; +import java.util.Arrays; + +public final class ScriptIpDocValues extends SortedBinaryDocValues { + private final IpScriptFieldScript script; + private int cursor; + + ScriptIpDocValues(IpScriptFieldScript script) { + this.script = script; + } + + @Override + public boolean advanceExact(int docId) { + script.runForDoc(docId); + if (script.count() == 0) { + return false; + } + Arrays.sort(script.values(), 0, script.count()); + cursor = 0; + return true; + } + + @Override + public BytesRef nextValue() throws IOException { + return script.values()[cursor++]; + } + + @Override + public int docValueCount() { + return script.count(); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpFieldData.java new file mode 100644 index 0000000000000..52b6b9d05ca77 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptIpFieldData.java @@ -0,0 +1,115 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; + +import java.io.IOException; +import java.net.InetAddress; + +public class ScriptIpFieldData extends ScriptBinaryFieldData { + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final IpScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, IpScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptIpFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptIpFieldData(name, leafFactory); + } + } + + private final IpScriptFieldScript.LeafFactory leafFactory; + + private ScriptIpFieldData(String fieldName, IpScriptFieldScript.LeafFactory leafFactory) { + super(fieldName); + this.leafFactory = leafFactory; + } + + @Override + public ScriptBinaryLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + IpScriptFieldScript script = leafFactory.newInstance(context); + return new ScriptBinaryLeafFieldData() { + @Override + public ScriptDocValues getScriptValues() { + return new IpScriptDocValues(script); + } + + @Override + public SortedBinaryDocValues getBytesValues() { + return new ScriptIpDocValues(script); + } + }; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.IP; + } + + /** + * We can't share {@link IpFieldMapper.IpFieldType.IpScriptDocValues} because it + * is based on global ordinals and we don't have those. + */ + public static class IpScriptDocValues extends ScriptDocValues { + private final IpScriptFieldScript script; + + public IpScriptDocValues(IpScriptFieldScript script) { + this.script = script; + } + + @Override + public void setNextDocId(int docId) throws IOException { + script.runForDoc(docId); + } + + public String getValue() { + if (size() == 0) { + return null; + } + return get(0); + } + + @Override + public String get(int index) { + if (index >= size()) { + if (size() == 0) { + throw new IllegalStateException( + "A document doesn't have a value for a field! " + + "Use doc[].size()==0 to check if a document is missing a field!" + ); + } + throw new ArrayIndexOutOfBoundsException("There are only [" + size() + "] values."); + } + InetAddress addr = InetAddressPoint.decode(BytesReference.toBytes(new BytesArray(script.values()[index]))); + return InetAddresses.toAddrString(addr); + } + + @Override + public int size() { + return script.count(); + } + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringDocValues.java similarity index 88% rename from x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java rename to x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringDocValues.java index f43ed950b4dd7..ada5dcff13e29 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptBinaryDocValues.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringDocValues.java @@ -11,10 +11,10 @@ import java.util.List; -public final class ScriptBinaryDocValues extends SortingBinaryDocValues { +public final class ScriptStringDocValues extends SortingBinaryDocValues { private final StringScriptFieldScript script; - ScriptBinaryDocValues(StringScriptFieldScript script) { + ScriptStringDocValues(StringScriptFieldScript script) { this.script = script; } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringFieldData.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringFieldData.java new file mode 100644 index 0000000000000..6c851a6be852a --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/fielddata/ScriptStringFieldData.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.fielddata; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.index.fielddata.IndexFieldData; +import org.elasticsearch.index.fielddata.IndexFieldDataCache; +import org.elasticsearch.index.fielddata.ScriptDocValues; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.mapper.MapperService; +import org.elasticsearch.indices.breaker.CircuitBreakerService; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; + +public class ScriptStringFieldData extends ScriptBinaryFieldData { + public static class Builder implements IndexFieldData.Builder { + private final String name; + private final StringScriptFieldScript.LeafFactory leafFactory; + + public Builder(String name, StringScriptFieldScript.LeafFactory leafFactory) { + this.name = name; + this.leafFactory = leafFactory; + } + + @Override + public ScriptStringFieldData build(IndexFieldDataCache cache, CircuitBreakerService breakerService, MapperService mapperService) { + return new ScriptStringFieldData(name, leafFactory); + } + } + + private final StringScriptFieldScript.LeafFactory leafFactory; + + private ScriptStringFieldData(String fieldName, StringScriptFieldScript.LeafFactory leafFactory) { + super(fieldName); + this.leafFactory = leafFactory; + } + + @Override + public ScriptBinaryLeafFieldData loadDirect(LeafReaderContext context) throws Exception { + StringScriptFieldScript script = leafFactory.newInstance(context); + return new ScriptBinaryLeafFieldData() { + @Override + public ScriptDocValues getScriptValues() { + return new ScriptDocValues.Strings(getBytesValues()); + } + + @Override + public SortedBinaryDocValues getBytesValues() { + return new ScriptStringDocValues(script); + } + }; + } + + @Override + public ValuesSourceType getValuesSourceType() { + return CoreValuesSourceType.BYTES; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java index 472b59ac00823..83ef48cdb45df 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapper.java @@ -10,6 +10,7 @@ import org.elasticsearch.common.util.LocaleUtils; import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.FieldMapper; +import org.elasticsearch.index.mapper.IpFieldMapper; import org.elasticsearch.index.mapper.KeywordFieldMapper; import org.elasticsearch.index.mapper.Mapper; import org.elasticsearch.index.mapper.NumberFieldMapper.NumberType; @@ -20,6 +21,7 @@ import org.elasticsearch.script.ScriptType; import org.elasticsearch.xpack.runtimefields.DateScriptFieldScript; import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; @@ -118,6 +120,20 @@ public static class Builder extends ParametrizedFieldMapper.Builder { builder.meta.getValue() ); }, + IpFieldMapper.CONTENT_TYPE, + (builder, context) -> { + builder.formatAndLocaleNotSupported(); + IpScriptFieldScript.Factory factory = builder.scriptCompiler.compile( + builder.script.getValue(), + IpScriptFieldScript.CONTEXT + ); + return new ScriptIpMappedFieldType( + builder.buildFullName(context), + builder.script.getValue(), + factory, + builder.meta.getValue() + ); + }, KeywordFieldMapper.CONTENT_TYPE, (builder, context) -> { builder.formatAndLocaleNotSupported(); diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptIpMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptIpMappedFieldType.java new file mode 100644 index 0000000000000..762c134a0e09e --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptIpMappedFieldType.java @@ -0,0 +1,181 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.search.BooleanClause.Occur; +import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.collect.Tuple; +import org.elasticsearch.common.lucene.BytesRefs; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.time.DateMathParser; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BytesRefHash; +import org.elasticsearch.index.mapper.IpFieldMapper; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.script.Script; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.fielddata.ScriptIpFieldData; +import org.elasticsearch.xpack.runtimefields.query.IpScriptFieldExistsQuery; +import org.elasticsearch.xpack.runtimefields.query.IpScriptFieldRangeQuery; +import org.elasticsearch.xpack.runtimefields.query.IpScriptFieldTermQuery; +import org.elasticsearch.xpack.runtimefields.query.IpScriptFieldTermsQuery; + +import java.net.InetAddress; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.function.Supplier; + +public final class ScriptIpMappedFieldType extends AbstractScriptMappedFieldType { + + private final Script script; + private final IpScriptFieldScript.Factory scriptFactory; + + ScriptIpMappedFieldType(String name, Script script, IpScriptFieldScript.Factory scriptFactory, Map meta) { + super(name, script, meta); + this.script = script; + this.scriptFactory = scriptFactory; + } + + @Override + protected String runtimeType() { + return IpFieldMapper.CONTENT_TYPE; + } + + @Override + public Object valueForDisplay(Object value) { + if (value == null) { + return null; + } + return DocValueFormat.IP.format((BytesRef) value); + } + + @Override + public DocValueFormat docValueFormat(String format, ZoneId timeZone) { + if (format != null) { + String message = "Field [%s] of type [%s] with runtime type [%s] does not support custom formats"; + throw new IllegalArgumentException(String.format(Locale.ROOT, message, name(), typeName(), runtimeType())); + } + if (timeZone != null) { + String message = "Field [%s] of type [%s] with runtime type [%s] does not support custom time zones"; + throw new IllegalArgumentException(String.format(Locale.ROOT, message, name(), typeName(), runtimeType())); + } + return DocValueFormat.IP; + } + + @Override + public ScriptIpFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + return new ScriptIpFieldData.Builder(name(), leafFactory(searchLookup.get())); + } + + private IpScriptFieldScript.LeafFactory leafFactory(SearchLookup searchLookup) { + return scriptFactory.newFactory(script.getParams(), searchLookup); + } + + @Override + public Query existsQuery(QueryShardContext context) { + checkAllowExpensiveQueries(context); + return new IpScriptFieldExistsQuery(script, leafFactory(context.lookup()), name()); + } + + @Override + public Query rangeQuery( + Object lowerTerm, + Object upperTerm, + boolean includeLower, + boolean includeUpper, + ZoneId timeZone, + DateMathParser parser, + QueryShardContext context + ) { + checkAllowExpensiveQueries(context); + return IpFieldMapper.IpFieldType.rangeQuery( + lowerTerm, + upperTerm, + includeLower, + includeUpper, + (lower, upper) -> new IpScriptFieldRangeQuery( + script, + leafFactory(context.lookup()), + name(), + new BytesRef(InetAddressPoint.encode(lower)), + new BytesRef(InetAddressPoint.encode(upper)) + ) + ); + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + checkAllowExpensiveQueries(context); + if (value instanceof InetAddress) { + return InetAddressPoint.newExactQuery(name(), (InetAddress) value); + } + String term = BytesRefs.toString(value); + if (term.contains("/")) { + return cidrQuery(term, context); + } + InetAddress address = InetAddresses.forString(term); + return new IpScriptFieldTermQuery(script, leafFactory(context.lookup()), name(), new BytesRef(InetAddressPoint.encode(address))); + } + + @Override + public Query termsQuery(List values, QueryShardContext context) { + checkAllowExpensiveQueries(context); + BytesRefHash terms = new BytesRefHash(values.size(), BigArrays.NON_RECYCLING_INSTANCE); + List cidrQueries = null; + for (Object value : values) { + if (value instanceof InetAddress) { + terms.add(new BytesRef(InetAddressPoint.encode((InetAddress) value))); + continue; + } + String term = BytesRefs.toString(value); + if (false == term.contains("/")) { + terms.add(new BytesRef(InetAddressPoint.encode(InetAddresses.forString(term)))); + continue; + } + if (cidrQueries == null) { + cidrQueries = new ArrayList<>(); + } + cidrQueries.add(cidrQuery(term, context)); + } + Query termsQuery = new IpScriptFieldTermsQuery(script, leafFactory(context.lookup()), name(), terms); + if (cidrQueries == null) { + return termsQuery; + } + BooleanQuery.Builder bool = new BooleanQuery.Builder(); + bool.add(termsQuery, Occur.SHOULD); + for (Query cidrQuery : cidrQueries) { + bool.add(cidrQuery, Occur.SHOULD); + } + return bool.build(); + } + + private Query cidrQuery(String term, QueryShardContext context) { + Tuple cidr = InetAddresses.parseCidr(term); + InetAddress addr = cidr.v1(); + int prefixLength = cidr.v2(); + // create the lower value by zeroing out the host portion, upper value by filling it with all ones. + byte lower[] = addr.getAddress(); + byte upper[] = addr.getAddress(); + for (int i = prefixLength; i < 8 * lower.length; i++) { + int m = 1 << (7 - (i & 7)); + lower[i >> 3] &= ~m; + upper[i >> 3] |= m; + } + // Force the terms into IPv6 + BytesRef lowerBytes = new BytesRef(InetAddressPoint.encode(InetAddressPoint.decode(lower))); + BytesRef upperBytes = new BytesRef(InetAddressPoint.encode(InetAddressPoint.decode(upper))); + return new IpScriptFieldRangeQuery(script, leafFactory(context.lookup()), name(), lowerBytes, upperBytes); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldType.java index 34671ac8cb51b..f41c053f7d580 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldType.java @@ -17,7 +17,7 @@ import org.elasticsearch.script.Script; import org.elasticsearch.search.lookup.SearchLookup; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; -import org.elasticsearch.xpack.runtimefields.fielddata.ScriptBinaryFieldData; +import org.elasticsearch.xpack.runtimefields.fielddata.ScriptStringFieldData; import org.elasticsearch.xpack.runtimefields.query.StringScriptFieldExistsQuery; import org.elasticsearch.xpack.runtimefields.query.StringScriptFieldFuzzyQuery; import org.elasticsearch.xpack.runtimefields.query.StringScriptFieldPrefixQuery; @@ -63,8 +63,8 @@ public Object valueForDisplay(Object value) { } @Override - public ScriptBinaryFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { - return new ScriptBinaryFieldData.Builder(name(), leafFactory(searchLookup.get())); + public ScriptStringFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, Supplier searchLookup) { + return new ScriptStringFieldData.Builder(name(), leafFactory(searchLookup.get())); } private StringScriptFieldScript.LeafFactory leafFactory(SearchLookup searchLookup) { diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractIpScriptFieldQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractIpScriptFieldQuery.java new file mode 100644 index 0000000000000..c68982bd944f5 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/AbstractIpScriptFieldQuery.java @@ -0,0 +1,78 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.search.ConstantScoreScorer; +import org.apache.lucene.search.ConstantScoreWeight; +import org.apache.lucene.search.DocIdSetIterator; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Scorer; +import org.apache.lucene.search.TwoPhaseIterator; +import org.apache.lucene.search.Weight; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; + +import java.io.IOException; +import java.net.InetAddress; +import java.util.Objects; + +/** + * Abstract base class for building queries based on {@link StringScriptFieldScript}. + */ +abstract class AbstractIpScriptFieldQuery extends AbstractScriptFieldQuery { + private final IpScriptFieldScript.LeafFactory leafFactory; + + AbstractIpScriptFieldQuery(Script script, IpScriptFieldScript.LeafFactory leafFactory, String fieldName) { + super(script, fieldName); + this.leafFactory = Objects.requireNonNull(leafFactory); + } + + /** + * Does the value match this query? + */ + protected abstract boolean matches(BytesRef[] values, int conut); + + @Override + public final Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException { + return new ConstantScoreWeight(this, boost) { + @Override + public boolean isCacheable(LeafReaderContext ctx) { + return false; // scripts aren't really cacheable at this point + } + + @Override + public Scorer scorer(LeafReaderContext ctx) throws IOException { + IpScriptFieldScript script = leafFactory.newInstance(ctx); + DocIdSetIterator approximation = DocIdSetIterator.all(ctx.reader().maxDoc()); + TwoPhaseIterator twoPhase = new TwoPhaseIterator(approximation) { + @Override + public boolean matches() throws IOException { + script.runForDoc(approximation().docID()); + return AbstractIpScriptFieldQuery.this.matches(script.values(), script.count()); + } + + @Override + public float matchCost() { + return MATCH_COST; + } + }; + return new ConstantScoreScorer(this, score(), scoreMode, twoPhase); + } + }; + } + + protected static InetAddress decode(BytesRef ref) { + return InetAddressPoint.decode(BytesReference.toBytes(new BytesArray(ref))); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldExistsQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldExistsQuery.java new file mode 100644 index 0000000000000..423c557ccb7fc --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldExistsQuery.java @@ -0,0 +1,32 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; + +public class IpScriptFieldExistsQuery extends AbstractIpScriptFieldQuery { + public IpScriptFieldExistsQuery(Script script, IpScriptFieldScript.LeafFactory leafFactory, String fieldName) { + super(script, leafFactory, fieldName); + } + + @Override + protected boolean matches(BytesRef[] values, int count) { + return count > 0; + } + + @Override + public final String toString(String field) { + if (fieldName().contentEquals(field)) { + return getClass().getSimpleName(); + } + return fieldName() + ":" + getClass().getSimpleName(); + } + + // Superclass's equals and hashCode are great for this class +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldRangeQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldRangeQuery.java new file mode 100644 index 0000000000000..8257401d655a8 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldRangeQuery.java @@ -0,0 +1,79 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; + +import java.net.InetAddress; +import java.util.Objects; + +public class IpScriptFieldRangeQuery extends AbstractIpScriptFieldQuery { + private final BytesRef lower; + private final BytesRef upper; + + public IpScriptFieldRangeQuery( + Script script, + IpScriptFieldScript.LeafFactory leafFactory, + String fieldName, + BytesRef lower, + BytesRef upper + ) { + super(script, leafFactory, fieldName); + this.lower = lower; + this.upper = upper; + assert this.lower.compareTo(this.upper) <= 0; + } + + @Override + protected boolean matches(BytesRef[] values, int count) { + for (int i = 0; i < count; i++) { + if (lower.compareTo(values[i]) <= 0 && upper.compareTo(values[i]) >= 0) { + return true; + } + } + return false; + } + + @Override + public final String toString(String field) { + StringBuilder b = new StringBuilder(); + if (false == fieldName().contentEquals(field)) { + b.append(fieldName()).append(':'); + } + b.append('[') + .append(InetAddresses.toAddrString(lowerAddress())) + .append(" TO ") + .append(InetAddresses.toAddrString(upperAddress())) + .append(']'); + return b.toString(); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), lower, upper); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + IpScriptFieldRangeQuery other = (IpScriptFieldRangeQuery) obj; + return lower.bytesEquals(other.lower) && upper.bytesEquals(other.upper); + } + + InetAddress lowerAddress() { + return decode(lower); + } + + InetAddress upperAddress() { + return decode(upper); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermQuery.java new file mode 100644 index 0000000000000..6ddc264a62452 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermQuery.java @@ -0,0 +1,60 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; + +import java.net.InetAddress; +import java.util.Objects; + +public class IpScriptFieldTermQuery extends AbstractIpScriptFieldQuery { + private final BytesRef term; + + public IpScriptFieldTermQuery(Script script, IpScriptFieldScript.LeafFactory leafFactory, String fieldName, BytesRef term) { + super(script, leafFactory, fieldName); + this.term = term; + } + + @Override + protected boolean matches(BytesRef[] values, int count) { + for (int i = 0; i < count; i++) { + if (term.bytesEquals(values[i])) { + return true; + } + } + return false; + } + + @Override + public final String toString(String field) { + if (fieldName().contentEquals(field)) { + return InetAddresses.toAddrString(address()); + } + return fieldName() + ":" + InetAddresses.toAddrString(address()); + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), term); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + IpScriptFieldTermQuery other = (IpScriptFieldTermQuery) obj; + return term.bytesEquals(other.term); + } + + InetAddress address() { + return decode(term); + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermsQuery.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermsQuery.java new file mode 100644 index 0000000000000..a808fb6d8ef60 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermsQuery.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.util.BytesRefHash; +import org.elasticsearch.script.Script; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; + +import java.util.Objects; + +public class IpScriptFieldTermsQuery extends AbstractIpScriptFieldQuery { + private final BytesRefHash terms; + + public IpScriptFieldTermsQuery(Script script, IpScriptFieldScript.LeafFactory leafFactory, String fieldName, BytesRefHash terms) { + super(script, leafFactory, fieldName); + this.terms = terms; + } + + @Override + protected boolean matches(BytesRef[] values, int count) { + for (int i = 0; i < count; i++) { + if (terms.find(values[i]) >= 0) { + return true; + } + } + return false; + } + + @Override + public final String toString(String field) { + StringBuilder b = new StringBuilder(); + if (false == fieldName().contentEquals(field)) { + b.append(fieldName()).append(":"); + } + b.append("["); + BytesRef spare = new BytesRef(); + long i = 0; + while (i < terms.size() && b.length() < 5000) { + if (i != 0) { + b.append(", "); + } + b.append(InetAddresses.toAddrString(decode(terms.get(i++, spare)))); + } + if (i < terms.size()) { + b.append("..."); + } + return b.append("]").toString(); + } + + @Override + public int hashCode() { + long hash = 0; + BytesRef spare = new BytesRef(); + for (long i = 0; i < terms.size(); i++) { + hash = 31 * hash + terms.get(i, spare).hashCode(); + } + return Objects.hash(super.hashCode(), hash); + } + + @Override + public boolean equals(Object obj) { + if (false == super.equals(obj)) { + return false; + } + IpScriptFieldTermsQuery other = (IpScriptFieldTermsQuery) obj; + if (terms.size() != other.terms.size()) { + return false; + } + BytesRef mySpare = new BytesRef(); + BytesRef otherSpare = new BytesRef(); + for (long i = 0; i < terms.size(); i++) { + terms.get(i, mySpare); + other.terms.get(i, otherSpare); + if (false == mySpare.bytesEquals(otherSpare)) { + return false; + } + } + return true; + } + + BytesRefHash terms() { + return terms; + } +} diff --git a/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/ip_whitelist.txt b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/ip_whitelist.txt new file mode 100644 index 0000000000000..e0829526bef89 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/main/resources/org/elasticsearch/xpack/runtimefields/ip_whitelist.txt @@ -0,0 +1,18 @@ +# +# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +# or more contributor license agreements. Licensed under the Elastic License; +# you may not use this file except in compliance with the Elastic License. +# + +# The whitelist for ip-valued runtime fields + +class org.elasticsearch.xpack.runtimefields.IpScriptFieldScript @no_import { +} + +static_import { + void stringValue(org.elasticsearch.xpack.runtimefields.IpScriptFieldScript, String) bound_to org.elasticsearch.xpack.runtimefields.IpScriptFieldScript$StringValue +} + +# This import is required to make painless happy and it isn't 100% clear why +class org.elasticsearch.xpack.runtimefields.IpScriptFieldScript$Factory @no_import { +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java index 3c251330f3902..43c8319c73a3d 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/RuntimeScriptFieldMapperTests.java @@ -25,6 +25,7 @@ import org.elasticsearch.test.InternalSettingsPlugin; import org.elasticsearch.xpack.runtimefields.DateScriptFieldScript; import org.elasticsearch.xpack.runtimefields.DoubleScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; import org.elasticsearch.xpack.runtimefields.LongScriptFieldScript; import org.elasticsearch.xpack.runtimefields.RuntimeFields; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; @@ -167,6 +168,13 @@ public void testDouble() throws IOException { assertEquals(Strings.toString(mapping("double")), Strings.toString(mapperService.documentMapper())); } + public void testIp() throws IOException { + MapperService mapperService = createIndex("test", Settings.EMPTY, mapping("ip")).mapperService(); + FieldMapper mapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); + assertThat(mapper, instanceOf(RuntimeScriptFieldMapper.class)); + assertEquals(Strings.toString(mapping("ip")), Strings.toString(mapperService.documentMapper())); + } + public void testKeyword() throws IOException { MapperService mapperService = createIndex("test", Settings.EMPTY, mapping("keyword")).mapperService(); FieldMapper mapper = (FieldMapper) mapperService.documentMapper().mappers().getMapper("field"); @@ -340,6 +348,14 @@ public void execute() { } }; } + if (context == IpScriptFieldScript.CONTEXT) { + return (IpScriptFieldScript.Factory) (params, lookup) -> ctx -> new IpScriptFieldScript(params, lookup, ctx) { + @Override + public void execute() { + new IpScriptFieldScript.StringValue(this).stringValue("192.168.0.1"); + } + }; + } if (context == StringScriptFieldScript.CONTEXT) { return (StringScriptFieldScript.Factory) (params, lookup) -> ctx -> new StringScriptFieldScript( params, diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptIpMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptIpMappedFieldTypeTests.java new file mode 100644 index 0000000000000..aebe5fe955b88 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptIpMappedFieldTypeTests.java @@ -0,0 +1,341 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.mapper; + +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.Collector; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.LeafCollector; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Scorable; +import org.apache.lucene.search.ScoreMode; +import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; +import org.apache.lucene.search.TopFieldDocs; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.Version; +import org.elasticsearch.common.lucene.search.function.ScriptScoreQuery; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.index.fielddata.SortedBinaryDocValues; +import org.elasticsearch.index.query.QueryShardContext; +import org.elasticsearch.plugins.ScriptPlugin; +import org.elasticsearch.script.ScoreScript; +import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptContext; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.MultiValueMode; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.RuntimeFields; +import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; +import org.elasticsearch.xpack.runtimefields.fielddata.ScriptBinaryFieldData; +import org.elasticsearch.xpack.runtimefields.fielddata.ScriptIpFieldData; + +import java.io.IOException; +import java.time.ZoneId; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.BiConsumer; + +import static java.util.Collections.emptyMap; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.sameInstance; + +public class ScriptIpMappedFieldTypeTests extends AbstractScriptMappedFieldTypeTestCase { + public void testFormat() throws IOException { + assertThat(simpleMappedFieldType().docValueFormat(null, null), sameInstance(DocValueFormat.IP)); + Exception e = expectThrows(IllegalArgumentException.class, () -> simpleMappedFieldType().docValueFormat("ASDFA", null)); + assertThat(e.getMessage(), equalTo("Field [test] of type [runtime_script] with runtime type [ip] does not support custom formats")); + e = expectThrows(IllegalArgumentException.class, () -> simpleMappedFieldType().docValueFormat(null, ZoneId.of("America/New_York"))); + assertThat( + e.getMessage(), + equalTo("Field [test] of type [runtime_script] with runtime type [ip] does not support custom time zones") + ); + } + + @Override + public void testDocValues() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.2\", \"192.168.1\"]}")))); + List results = new ArrayList<>(); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + ScriptIpMappedFieldType ft = build("append_param", Map.of("param", ".1")); + ScriptBinaryFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup).build(null, null, null); + DocValueFormat format = ft.docValueFormat(null, null); + searcher.search(new MatchAllDocsQuery(), new Collector() { + @Override + public ScoreMode scoreMode() { + return ScoreMode.COMPLETE_NO_SCORES; + } + + @Override + public LeafCollector getLeafCollector(LeafReaderContext context) { + SortedBinaryDocValues dv = ifd.load(context).getBytesValues(); + return new LeafCollector() { + @Override + public void setScorer(Scorable scorer) {} + + @Override + public void collect(int doc) throws IOException { + if (dv.advanceExact(doc)) { + for (int i = 0; i < dv.docValueCount(); i++) { + results.add(format.format(dv.nextValue())); + } + } + } + }; + } + }); + assertThat(results, equalTo(List.of("192.168.0.1", "192.168.1.1", "192.168.2.1"))); + } + } + } + + @Override + public void testSort() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0.1\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0.4\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0.2\"]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + ScriptBinaryFieldData ifd = simpleMappedFieldType().fielddataBuilder("test", mockContext()::lookup).build(null, null, null); + SortField sf = ifd.sortField(null, MultiValueMode.MIN, null, false); + TopFieldDocs docs = searcher.search(new MatchAllDocsQuery(), 3, new Sort(sf)); + assertThat( + reader.document(docs.scoreDocs[0].doc).getBinaryValue("_source").utf8ToString(), + equalTo("{\"foo\": [\"192.168.0.1\"]}") + ); + assertThat( + reader.document(docs.scoreDocs[1].doc).getBinaryValue("_source").utf8ToString(), + equalTo("{\"foo\": [\"192.168.0.2\"]}") + ); + assertThat( + reader.document(docs.scoreDocs[2].doc).getBinaryValue("_source").utf8ToString(), + equalTo("{\"foo\": [\"192.168.0.4\"]}") + ); + } + } + } + + @Override + public void testUsedInScript() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0.1\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0.4\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0.2\"]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + QueryShardContext qsc = mockContext(true, simpleMappedFieldType()); + assertThat(searcher.count(new ScriptScoreQuery(new MatchAllDocsQuery(), new Script("test"), new ScoreScript.LeafFactory() { + @Override + public boolean needs_score() { + return false; + } + + @Override + public ScoreScript newInstance(LeafReaderContext ctx) { + return new ScoreScript(Map.of(), qsc.lookup(), ctx) { + @Override + public double execute(ExplanationHolder explanation) { + ScriptIpFieldData.IpScriptDocValues bytes = (ScriptIpFieldData.IpScriptDocValues) getDoc().get("test"); + return Integer.parseInt(bytes.getValue().substring(bytes.getValue().lastIndexOf(".") + 1)); + } + }; + } + }, 2.5f, "test", 0, Version.CURRENT)), equalTo(1)); + } + } + } + + @Override + public void testExistsQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0.1\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": []}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + assertThat(searcher.count(simpleMappedFieldType().existsQuery(mockContext())), equalTo(1)); + } + } + } + + @Override + public void testExistsQueryIsExpensive() throws IOException { + checkExpensiveQuery(ScriptIpMappedFieldType::existsQuery); + } + + @Override + public void testRangeQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0.1\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"200.0.0.1\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"1.1.1.1\"]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + assertThat( + searcher.count( + simpleMappedFieldType().rangeQuery("192.0.0.0", "200.0.0.0", false, false, null, null, null, mockContext()) + ), + equalTo(1) + ); + } + } + } + + @Override + public void testRangeQueryIsExpensive() throws IOException { + checkExpensiveQuery((ft, ctx) -> ft.rangeQuery("192.0.0.0", "200.0.0.0", randomBoolean(), randomBoolean(), null, null, null, ctx)); + } + + @Override + public void testTermQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.1\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"200.0.0\"]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + ScriptIpMappedFieldType fieldType = build("append_param", Map.of("param", ".1")); + assertThat(searcher.count(fieldType.termQuery("192.168.0.1", mockContext())), equalTo(1)); + assertThat(searcher.count(fieldType.termQuery("192.168.0.7", mockContext())), equalTo(0)); + assertThat(searcher.count(fieldType.termQuery("192.168.0.0/16", mockContext())), equalTo(2)); + assertThat(searcher.count(fieldType.termQuery("10.168.0.0/16", mockContext())), equalTo(0)); + } + } + } + + @Override + public void testTermQueryIsExpensive() throws IOException { + checkExpensiveQuery((ft, ctx) -> ft.termQuery(randomIp(randomBoolean()), ctx)); + } + + @Override + public void testTermsQuery() throws IOException { + try (Directory directory = newDirectory(); RandomIndexWriter iw = new RandomIndexWriter(random(), directory)) { + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.0.1\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"192.168.1.1\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"200.0.0.1\"]}")))); + iw.addDocument(List.of(new StoredField("_source", new BytesRef("{\"foo\": [\"1.1.1.1\"]}")))); + try (DirectoryReader reader = iw.getReader()) { + IndexSearcher searcher = newSearcher(reader); + assertThat( + searcher.count(simpleMappedFieldType().termsQuery(List.of("192.168.0.1", "1.1.1.1"), mockContext())), + equalTo(2) + ); + assertThat( + searcher.count(simpleMappedFieldType().termsQuery(List.of("192.168.0.0/16", "1.1.1.1"), mockContext())), + equalTo(3) + ); + } + } + } + + @Override + public void testTermsQueryIsExpensive() throws IOException { + checkExpensiveQuery((ft, ctx) -> ft.termsQuery(randomList(100, () -> randomAlphaOfLengthBetween(1, 1000)), ctx)); + } + + @Override + protected ScriptIpMappedFieldType simpleMappedFieldType() throws IOException { + return build("read_foo", Map.of()); + } + + @Override + protected String runtimeType() { + return "ip"; + } + + private static ScriptIpMappedFieldType build(String code, Map params) throws IOException { + return build(new Script(ScriptType.INLINE, "test", code, params)); + } + + private static ScriptIpMappedFieldType build(Script script) throws IOException { + ScriptPlugin scriptPlugin = new ScriptPlugin() { + @Override + public ScriptEngine getScriptEngine(Settings settings, Collection> contexts) { + return new ScriptEngine() { + @Override + public String getType() { + return "test"; + } + + @Override + public Set> getSupportedContexts() { + return Set.of(StringScriptFieldScript.CONTEXT); + } + + @Override + public FactoryType compile( + String name, + String code, + ScriptContext context, + Map params + ) { + @SuppressWarnings("unchecked") + FactoryType factory = (FactoryType) factory(code); + return factory; + } + + private IpScriptFieldScript.Factory factory(String code) { + switch (code) { + case "read_foo": + return (params, lookup) -> (ctx) -> new IpScriptFieldScript(params, lookup, ctx) { + @Override + public void execute() { + for (Object foo : (List) getSource().get("foo")) { + new IpScriptFieldScript.StringValue(this).stringValue(foo.toString()); + } + } + }; + case "append_param": + return (params, lookup) -> (ctx) -> new IpScriptFieldScript(params, lookup, ctx) { + @Override + public void execute() { + for (Object foo : (List) getSource().get("foo")) { + new IpScriptFieldScript.StringValue(this).stringValue( + foo.toString() + getParams().get("param") + ); + } + } + }; + default: + throw new IllegalArgumentException("unsupported script [" + code + "]"); + } + } + }; + } + }; + ScriptModule scriptModule = new ScriptModule(Settings.EMPTY, List.of(scriptPlugin, new RuntimeFields())); + try (ScriptService scriptService = new ScriptService(Settings.EMPTY, scriptModule.engines, scriptModule.contexts)) { + IpScriptFieldScript.Factory factory = scriptService.compile(script, IpScriptFieldScript.CONTEXT); + return new ScriptIpMappedFieldType("test", script, factory, emptyMap()); + } + } + + private void checkExpensiveQuery(BiConsumer queryBuilder) throws IOException { + ScriptIpMappedFieldType ft = simpleMappedFieldType(); + Exception e = expectThrows(ElasticsearchException.class, () -> queryBuilder.accept(ft, mockContext(false))); + assertThat( + e.getMessage(), + equalTo("queries cannot be executed against [runtime_script] fields while [search.allow_expensive_queries] is set to [false].") + ); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java index bbfb9a94bb49d..493c6cf62c902 100644 --- a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/mapper/ScriptKeywordMappedFieldTypeTests.java @@ -44,6 +44,7 @@ import org.elasticsearch.xpack.runtimefields.RuntimeFields; import org.elasticsearch.xpack.runtimefields.StringScriptFieldScript; import org.elasticsearch.xpack.runtimefields.fielddata.ScriptBinaryFieldData; +import org.elasticsearch.xpack.runtimefields.fielddata.ScriptStringFieldData; import java.io.IOException; import java.util.ArrayList; @@ -66,7 +67,7 @@ public void testDocValues() throws IOException { try (DirectoryReader reader = iw.getReader()) { IndexSearcher searcher = newSearcher(reader); ScriptKeywordMappedFieldType ft = build("append_param", Map.of("param", "-suffix")); - ScriptBinaryFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup).build(null, null, null); + ScriptStringFieldData ifd = ft.fielddataBuilder("test", mockContext()::lookup).build(null, null, null); searcher.search(new MatchAllDocsQuery(), new Collector() { @Override public ScoreMode scoreMode() { diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractIpScriptFieldQueryTestCase.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractIpScriptFieldQueryTestCase.java new file mode 100644 index 0000000000000..1f2ec87024896 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/AbstractIpScriptFieldQueryTestCase.java @@ -0,0 +1,55 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.index.Term; +import org.apache.lucene.search.Query; +import org.apache.lucene.search.QueryVisitor; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.ByteRunAutomaton; +import org.elasticsearch.xpack.runtimefields.IpScriptFieldScript; + +import java.net.InetAddress; +import java.util.ArrayList; +import java.util.List; +import java.util.function.Supplier; + +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; + +public abstract class AbstractIpScriptFieldQueryTestCase extends AbstractScriptFieldQueryTestCase { + + protected final IpScriptFieldScript.LeafFactory leafFactory = mock(IpScriptFieldScript.LeafFactory.class); + + @Override + public final void testVisit() { + T query = createTestInstance(); + List leavesVisited = new ArrayList<>(); + query.visit(new QueryVisitor() { + @Override + public void consumeTerms(Query query, Term... terms) { + fail(); + } + + @Override + public void consumeTermsMatching(Query query, String field, Supplier automaton) { + fail(); + } + + @Override + public void visitLeaf(Query query) { + leavesVisited.add(query); + } + }); + assertThat(leavesVisited, equalTo(List.of(query))); + } + + protected static BytesRef encode(InetAddress addr) { + return new BytesRef(InetAddressPoint.encode(addr)); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldRangeQueryTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldRangeQueryTests.java new file mode 100644 index 0000000000000..e4ff8ee7c347f --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/DoubleScriptFieldRangeQueryTests.java @@ -0,0 +1,74 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.elasticsearch.script.Script; +import org.elasticsearch.test.ESTestCase; + +import static org.hamcrest.Matchers.equalTo; + +public class DoubleScriptFieldRangeQueryTests extends AbstractDoubleScriptFieldQueryTestCase { + @Override + protected DoubleScriptFieldRangeQuery createTestInstance() { + double lower = randomDouble(); + double upper = randomValueOtherThan(lower, ESTestCase::randomDouble); + if (lower > upper) { + double tmp = lower; + lower = upper; + upper = tmp; + } + return new DoubleScriptFieldRangeQuery(randomScript(), leafFactory, randomAlphaOfLength(5), lower, upper); + } + + @Override + protected DoubleScriptFieldRangeQuery copy(DoubleScriptFieldRangeQuery orig) { + return new DoubleScriptFieldRangeQuery(orig.script(), leafFactory, orig.fieldName(), orig.lowerValue(), orig.upperValue()); + } + + @Override + protected DoubleScriptFieldRangeQuery mutate(DoubleScriptFieldRangeQuery orig) { + Script script = orig.script(); + String fieldName = orig.fieldName(); + double lower = orig.lowerValue(); + double upper = orig.upperValue(); + switch (randomInt(3)) { + case 0: + script = randomValueOtherThan(script, this::randomScript); + break; + case 1: + fieldName += "modified"; + break; + case 2: + lower -= 1; + break; + case 3: + upper += 1; + break; + default: + fail(); + } + return new DoubleScriptFieldRangeQuery(script, leafFactory, fieldName, lower, upper); + } + + @Override + public void testMatches() { + DoubleScriptFieldRangeQuery query = new DoubleScriptFieldRangeQuery(randomScript(), leafFactory, "test", 1.2, 3.14); + assertTrue(query.matches(new double[] { 1.2 }, 1)); + assertTrue(query.matches(new double[] { 3.14 }, 1)); + assertTrue(query.matches(new double[] { 2 }, 1)); + assertFalse(query.matches(new double[] { 0 }, 0)); + assertFalse(query.matches(new double[] { 5 }, 1)); + assertTrue(query.matches(new double[] { 2, 5 }, 2)); + assertTrue(query.matches(new double[] { 5, 2 }, 2)); + assertFalse(query.matches(new double[] { 5, 2 }, 1)); + } + + @Override + protected void assertToString(DoubleScriptFieldRangeQuery query) { + assertThat(query.toString(query.fieldName()), equalTo("[" + query.lowerValue() + " TO " + query.upperValue() + "]")); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldExistsQueryTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldExistsQueryTests.java new file mode 100644 index 0000000000000..d33cbfeb44832 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldExistsQueryTests.java @@ -0,0 +1,43 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.util.BytesRef; + +import static org.hamcrest.Matchers.equalTo; + +public class IpScriptFieldExistsQueryTests extends AbstractIpScriptFieldQueryTestCase { + @Override + protected IpScriptFieldExistsQuery createTestInstance() { + return new IpScriptFieldExistsQuery(randomScript(), leafFactory, randomAlphaOfLength(5)); + } + + @Override + protected IpScriptFieldExistsQuery copy(IpScriptFieldExistsQuery orig) { + return new IpScriptFieldExistsQuery(orig.script(), leafFactory, orig.fieldName()); + } + + @Override + protected IpScriptFieldExistsQuery mutate(IpScriptFieldExistsQuery orig) { + if (randomBoolean()) { + new IpScriptFieldExistsQuery(randomValueOtherThan(orig.script(), this::randomScript), leafFactory, orig.fieldName()); + } + return new IpScriptFieldExistsQuery(orig.script(), leafFactory, orig.fieldName() + "modified"); + } + + @Override + public void testMatches() { + assertTrue(createTestInstance().matches(new BytesRef[0], randomIntBetween(1, Integer.MAX_VALUE))); + assertFalse(createTestInstance().matches(new BytesRef[0], 0)); + assertFalse(createTestInstance().matches(new BytesRef[] { new BytesRef("not even an IP") }, 0)); + } + + @Override + protected void assertToString(IpScriptFieldExistsQuery query) { + assertThat(query.toString(query.fieldName()), equalTo("IpScriptFieldExistsQuery")); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldRangeQueryTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldRangeQueryTests.java new file mode 100644 index 0000000000000..cb6bb2953da2a --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldRangeQueryTests.java @@ -0,0 +1,125 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.script.Script; + +import java.net.InetAddress; + +import static org.hamcrest.Matchers.equalTo; + +public class IpScriptFieldRangeQueryTests extends AbstractIpScriptFieldQueryTestCase { + @Override + protected IpScriptFieldRangeQuery createTestInstance() { + InetAddress lower = randomIp(randomBoolean()); + InetAddress upper = randomIp(randomBoolean()); + if (mustFlip(lower, upper)) { + InetAddress tmp = lower; + lower = upper; + upper = tmp; + } + return new IpScriptFieldRangeQuery(randomScript(), leafFactory, randomAlphaOfLength(5), encode(lower), encode(upper)); + } + + @Override + protected IpScriptFieldRangeQuery copy(IpScriptFieldRangeQuery orig) { + return new IpScriptFieldRangeQuery( + orig.script(), + leafFactory, + orig.fieldName(), + encode(orig.lowerAddress()), + encode(orig.upperAddress()) + ); + } + + @Override + protected IpScriptFieldRangeQuery mutate(IpScriptFieldRangeQuery orig) { + Script script = orig.script(); + String fieldName = orig.fieldName(); + InetAddress lower = orig.lowerAddress(); + InetAddress upper = orig.upperAddress(); + switch (randomInt(3)) { + case 0: + script = randomValueOtherThan(script, this::randomScript); + break; + case 1: + fieldName += "modified"; + break; + case 2: + lower = randomIp(randomBoolean()); + break; + case 3: + upper = randomIp(randomBoolean()); + break; + default: + fail(); + } + if (mustFlip(lower, upper)) { + InetAddress tmp = lower; + lower = upper; + upper = tmp; + } + return new IpScriptFieldRangeQuery(script, leafFactory, fieldName, encode(lower), encode(upper)); + } + + @Override + public void testMatches() { + // Try with ipv4 + BytesRef min = encode(InetAddresses.forString("192.168.0.1")); + BytesRef max = encode(InetAddresses.forString("200.255.255.255")); + IpScriptFieldRangeQuery query = new IpScriptFieldRangeQuery(randomScript(), leafFactory, "test", min, max); + assertTrue(query.matches(new BytesRef[] { min }, 1)); + assertTrue(query.matches(new BytesRef[] { max }, 1)); + assertTrue(query.matches(new BytesRef[] { encode(InetAddresses.forString("200.255.255.0")) }, 1)); + assertFalse(query.matches(new BytesRef[] { encode(InetAddresses.forString("127.0.0.1")) }, 0)); + assertFalse(query.matches(new BytesRef[] { encode(InetAddresses.forString("127.0.0.1")) }, 1)); + assertFalse(query.matches(new BytesRef[] { encode(InetAddresses.forString("241.0.0.0")) }, 1)); + assertTrue(query.matches(new BytesRef[] { min, encode(InetAddresses.forString("241.0.0.0")) }, 2)); + assertTrue(query.matches(new BytesRef[] { encode(InetAddresses.forString("241.0.0.0")), min }, 2)); + assertFalse(query.matches(new BytesRef[] { encode(InetAddresses.forString("241.0.0.0")), min }, 1)); + + // Now ipv6 + min = encode(InetAddresses.forString("ff00::")); + max = encode(InetAddresses.forString("fff0:ffff:ffff:ffff:ffff:ffff:ffff:ffff")); + query = new IpScriptFieldRangeQuery(randomScript(), leafFactory, "test", min, max); + assertTrue(query.matches(new BytesRef[] { min }, 1)); + assertTrue(query.matches(new BytesRef[] { max }, 1)); + assertTrue(query.matches(new BytesRef[] { encode(InetAddresses.forString("ff00::1")) }, 1)); + assertFalse(query.matches(new BytesRef[] { encode(InetAddresses.forString("fa00::")) }, 1)); + assertFalse(query.matches(new BytesRef[] { encode(InetAddresses.forString("127.0.0.1")) }, 1)); + assertFalse(query.matches(new BytesRef[] { encode(InetAddresses.forString("127.0.0.1")) }, 0)); + assertFalse(query.matches(new BytesRef[] { encode(InetAddresses.forString("241.0.0.0")) }, 1)); + assertTrue(query.matches(new BytesRef[] { min, encode(InetAddresses.forString("fa00::")) }, 2)); + assertTrue(query.matches(new BytesRef[] { encode(InetAddresses.forString("fa00::")), min }, 2)); + assertFalse(query.matches(new BytesRef[] { encode(InetAddresses.forString("fa00::")), min }, 1)); + + // Finally, try with an ipv6 range that contains ipv4 + min = encode(InetAddresses.forString("::fff:0:0")); + max = encode(InetAddresses.forString("::ffff:ffff:ffff")); + query = new IpScriptFieldRangeQuery(randomScript(), leafFactory, "test", min, max); + assertTrue(query.matches(new BytesRef[] { min }, 1)); + assertTrue(query.matches(new BytesRef[] { max }, 1)); + assertTrue(query.matches(new BytesRef[] { encode(InetAddresses.forString("127.0.0.1")) }, 1)); + assertTrue(query.matches(new BytesRef[] { encode(InetAddresses.forString("241.0.0.0")) }, 1)); + } + + @Override + protected void assertToString(IpScriptFieldRangeQuery query) { + assertThat( + query.toString(query.fieldName()), + equalTo( + "[" + InetAddresses.toAddrString(query.lowerAddress()) + " TO " + InetAddresses.toAddrString(query.upperAddress()) + "]" + ) + ); + } + + private boolean mustFlip(InetAddress minCandidate, InetAddress maxCandidate) { + return encode(minCandidate).compareTo(encode(maxCandidate)) > 0; + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermQueryTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermQueryTests.java new file mode 100644 index 0000000000000..e141215598fb3 --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermQueryTests.java @@ -0,0 +1,64 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.script.Script; + +import java.net.InetAddress; + +import static org.hamcrest.Matchers.equalTo; + +public class IpScriptFieldTermQueryTests extends AbstractIpScriptFieldQueryTestCase { + @Override + protected IpScriptFieldTermQuery createTestInstance() { + return new IpScriptFieldTermQuery(randomScript(), leafFactory, randomAlphaOfLength(5), encode(randomIp(randomBoolean()))); + } + + @Override + protected IpScriptFieldTermQuery copy(IpScriptFieldTermQuery orig) { + return new IpScriptFieldTermQuery(orig.script(), leafFactory, orig.fieldName(), encode(orig.address())); + } + + @Override + protected IpScriptFieldTermQuery mutate(IpScriptFieldTermQuery orig) { + Script script = orig.script(); + String fieldName = orig.fieldName(); + InetAddress term = orig.address(); + switch (randomInt(2)) { + case 0: + script = randomValueOtherThan(script, this::randomScript); + break; + case 1: + fieldName += "modified"; + break; + case 2: + term = randomValueOtherThan(term, () -> randomIp(randomBoolean())); + break; + default: + fail(); + } + return new IpScriptFieldTermQuery(script, leafFactory, fieldName, encode(term)); + } + + @Override + public void testMatches() { + BytesRef ip = encode(randomIp(randomBoolean())); + BytesRef notIpRef = randomValueOtherThan(ip, () -> encode(randomIp(randomBoolean()))); + IpScriptFieldTermQuery query = new IpScriptFieldTermQuery(randomScript(), leafFactory, "test", ip); + assertTrue(query.matches(new BytesRef[] { ip }, 1)); // Match because value matches + assertFalse(query.matches(new BytesRef[] { notIpRef }, 1)); // No match because wrong value + assertFalse(query.matches(new BytesRef[] { notIpRef, ip }, 1)); // No match because value after count of values + assertTrue(query.matches(new BytesRef[] { notIpRef, ip }, 2)); // Match because one value matches + } + + @Override + protected void assertToString(IpScriptFieldTermQuery query) { + assertThat(query.toString(query.fieldName()), equalTo(InetAddresses.toAddrString(query.address()))); + } +} diff --git a/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermsQueryTests.java b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermsQueryTests.java new file mode 100644 index 0000000000000..70f68140edb1b --- /dev/null +++ b/x-pack/plugin/runtime-fields/src/test/java/org/elasticsearch/xpack/runtimefields/query/IpScriptFieldTermsQueryTests.java @@ -0,0 +1,141 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License; + * you may not use this file except in compliance with the Elastic License. + */ + +package org.elasticsearch.xpack.runtimefields.query; + +import org.apache.lucene.document.InetAddressPoint; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.network.InetAddresses; +import org.elasticsearch.common.util.BigArrays; +import org.elasticsearch.common.util.BytesRefHash; +import org.elasticsearch.script.Script; + +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.startsWith; + +public class IpScriptFieldTermsQueryTests extends AbstractIpScriptFieldQueryTestCase { + @Override + protected IpScriptFieldTermsQuery createTestInstance() { + return createTestInstance(between(1, 100)); + } + + private IpScriptFieldTermsQuery createTestInstance(int size) { + BytesRefHash terms = new BytesRefHash(size, BigArrays.NON_RECYCLING_INSTANCE); + while (terms.size() < size) { + terms.add(new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean())))); + } + return new IpScriptFieldTermsQuery(randomScript(), leafFactory, randomAlphaOfLength(5), terms); + } + + @Override + protected IpScriptFieldTermsQuery copy(IpScriptFieldTermsQuery orig) { + return new IpScriptFieldTermsQuery(orig.script(), leafFactory, orig.fieldName(), copyTerms(orig.terms())); + } + + private BytesRefHash copyTerms(BytesRefHash terms) { + BytesRefHash copy = new BytesRefHash(terms.size(), BigArrays.NON_RECYCLING_INSTANCE); + BytesRef spare = new BytesRef(); + for (long i = 0; i < terms.size(); i++) { + terms.get(i, spare); + assertEquals(i, copy.add(spare)); + } + return copy; + } + + @Override + protected IpScriptFieldTermsQuery mutate(IpScriptFieldTermsQuery orig) { + Script script = orig.script(); + String fieldName = orig.fieldName(); + BytesRefHash terms = copyTerms(orig.terms()); + switch (randomInt(2)) { + case 0: + script = randomValueOtherThan(script, this::randomScript); + break; + case 1: + fieldName += "modified"; + break; + case 2: + long size = terms.size() + 1; + while (terms.size() < size) { + terms.add(new BytesRef(InetAddressPoint.encode(randomIp(randomBoolean())))); + } + break; + default: + fail(); + } + return new IpScriptFieldTermsQuery(script, leafFactory, fieldName, terms); + } + + @Override + public void testMatches() { + BytesRef ip1 = encode(InetAddresses.forString("192.168.0.1")); + BytesRef ip2 = encode(InetAddresses.forString("192.168.0.2")); + BytesRef notIp = encode(InetAddresses.forString("192.168.0.3")); + + BytesRefHash terms = new BytesRefHash(2, BigArrays.NON_RECYCLING_INSTANCE); + terms.add(ip1); + terms.add(ip2); + IpScriptFieldTermsQuery query = new IpScriptFieldTermsQuery(randomScript(), leafFactory, "test", terms); + assertTrue(query.matches(new BytesRef[] { ip1 }, 1)); + assertTrue(query.matches(new BytesRef[] { ip2 }, 1)); + assertTrue(query.matches(new BytesRef[] { ip1, notIp }, 2)); + assertTrue(query.matches(new BytesRef[] { notIp, ip1 }, 2)); + assertFalse(query.matches(new BytesRef[] { notIp }, 1)); + assertFalse(query.matches(new BytesRef[] { notIp, ip1 }, 1)); + } + + @Override + protected void assertToString(IpScriptFieldTermsQuery query) { + if (query.toString(query.fieldName()).contains("...")) { + assertBigToString(query); + } else { + assertLittleToString(query); + } + } + + private void assertBigToString(IpScriptFieldTermsQuery query) { + String toString = query.toString(query.fieldName()); + BytesRef spare = new BytesRef(); + assertThat(toString, startsWith("[")); + query.terms().get(0, spare); + assertThat( + toString, + containsString(InetAddresses.toAddrString(InetAddressPoint.decode(BytesReference.toBytes(new BytesArray(spare))))) + ); + query.terms().get(query.terms().size() - 1, spare); + assertThat( + toString, + not(containsString(InetAddresses.toAddrString(InetAddressPoint.decode(BytesReference.toBytes(new BytesArray(spare)))))) + ); + assertThat(toString, endsWith("...]")); + } + + private void assertLittleToString(IpScriptFieldTermsQuery query) { + String toString = query.toString(query.fieldName()); + BytesRef spare = new BytesRef(); + assertThat(toString, startsWith("[")); + for (long i = 0; i < query.terms().size(); i++) { + query.terms().get(i, spare); + assertThat( + toString, + containsString(InetAddresses.toAddrString(InetAddressPoint.decode(BytesReference.toBytes(new BytesArray(spare))))) + ); + } + assertThat(toString, endsWith("]")); + } + + public void testBigToString() { + assertBigToString(createTestInstance(1000)); + } + + public void testLittleToString() { + assertLittleToString(createTestInstance(5)); + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml new file mode 100644 index 0000000000000..e572ce7462f30 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/runtime_fields/50_ip.yml @@ -0,0 +1,126 @@ +--- +setup: + - do: + indices.create: + index: http_logs + body: + settings: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + timestamp: + type: date + message: + type: keyword + ip: + type: runtime_script + runtime_type: ip + script: + source: | + String m = doc["message"].value; + int end = m.indexOf(" "); + stringValue(m.substring(0, end)); + # Test fetching from _source + ip_from_source: + type: runtime_script + runtime_type: ip + script: + source: | + String m = source["message"]; + int end = m.indexOf(" "); + stringValue(m.substring(0, end)); + # Test emitting many values + ip_many: + type: runtime_script + runtime_type: ip + script: + source: | + String m = doc["message"].value; + int end = m.indexOf(" "); + end = m.lastIndexOf(".", end); + String stem = m.substring(0, end + 1); + for (int i = 0; i < 5; i++) { + stringValue(stem + i); + } + - do: + bulk: + index: http_logs + refresh: true + body: | + {"index":{}} + {"timestamp": "1998-04-30T14:30:17-05:00", "message" : "40.135.0.0 - - [1998-04-30T14:30:17-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:30:53-05:00", "message" : "232.0.0.0 - - [1998-04-30T14:30:53-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:12-05:00", "message" : "26.1.0.0 - - [1998-04-30T14:31:12-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:19-05:00", "message" : "247.37.0.0 - - [1998-04-30T14:31:19-05:00] \"GET /french/splash_inet.html HTTP/1.0\" 200 3781"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:22-05:00", "message" : "247.37.0.0 - - [1998-04-30T14:31:22-05:00] \"GET /images/hm_nbg.jpg HTTP/1.0\" 304 0"} + {"index":{}} + {"timestamp": "1998-04-30T14:31:27-05:00", "message" : "252.0.0.0 - - [1998-04-30T14:31:27-05:00] \"GET /images/hm_bg.jpg HTTP/1.0\" 200 24736"} + + +--- +"get mapping": + - do: + indices.get_mapping: + index: http_logs + - match: {http_logs.mappings.properties.ip.type: runtime_script } + - match: {http_logs.mappings.properties.ip.runtime_type: ip } + - match: + http_logs.mappings.properties.ip.script.source: | + String m = doc["message"].value; + int end = m.indexOf(" "); + stringValue(m.substring(0, end)); + - match: {http_logs.mappings.properties.ip.script.lang: painless } + +--- +"docvalue_fields": + - do: + search: + index: http_logs + body: + sort: timestamp + docvalue_fields: [ip, ip_from_source, ip_many] + - match: {hits.total.value: 6} + - match: {hits.hits.0.fields.ip: ["40.135.0.0"] } + - match: {hits.hits.0.fields.ip_from_source: ["40.135.0.0"] } + - match: + hits.hits.0.fields.ip_many: + - 40.135.0.0 + - 40.135.0.1 + - 40.135.0.2 + - 40.135.0.3 + - 40.135.0.4 + +--- +"terms agg": + - do: + search: + index: http_logs + body: + aggs: + ip: + terms: + field: ip + - match: {hits.total.value: 6} + - match: {aggregations.ip.buckets.0.key: 247.37.0.0} + - match: {aggregations.ip.buckets.0.doc_count: 2} + - match: {aggregations.ip.buckets.1.key: 26.1.0.0} + - match: {aggregations.ip.buckets.1.doc_count: 1} + +--- +"term query": + - do: + search: + index: http_logs + body: + query: + term: + ip: 252.0.0.0 + - match: {hits.total.value: 1} + - match: {hits.hits.0._source.timestamp: 1998-04-30T14:31:27-05:00} + +# TODO tests for using the ip in a script. there is almost certainly whitelist "fun" here.