Fix for wildcard fields that returned ByteRefs not Strings to scripts.

This need some reorg of BinaryDV field data classes to allow specialisation of scripted doc values.
Moved common logic to a new abstract base class and added a new subclass to return string-based representations to scripts.

Closes elastic#58044

server/src/main/java/org/elasticsearch/index/fielddata/plain/AbstractBinaryDVLeafFieldData.java

@@ -0,0 +1,94 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.apache.lucene.store.ByteArrayDataInput;
+import org.apache.lucene.util.Accountable;
+import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.index.fielddata.LeafFieldData;
+import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
+
+abstract class  AbstractBinaryDVLeafFieldData implements LeafFieldData {
+    private final BinaryDocValues values;
+
+    AbstractBinaryDVLeafFieldData(BinaryDocValues values) {
+        super();
+        this.values = values;
+    }
+
+    @Override
+    public long ramBytesUsed() {
+        return 0; // not exposed by Lucene
+    }
+
+    @Override
+    public Collection<Accountable> getChildResources() {
+        return Collections.emptyList();
+    }
+
+    @Override
+    public SortedBinaryDocValues getBytesValues() {
+        return new SortedBinaryDocValues() {
+
+            int count;
+            final ByteArrayDataInput in = new ByteArrayDataInput();
+            final BytesRef scratch = new BytesRef();
+
+            @Override
+            public boolean advanceExact(int doc) throws IOException {
+                if (values.advanceExact(doc)) {
+                    final BytesRef bytes = values.binaryValue();
+                    assert bytes.length > 0;
+                    in.reset(bytes.bytes, bytes.offset, bytes.length);
+                    count = in.readVInt();
+                    scratch.bytes = bytes.bytes;
+                    return true;
+                } else {
+                    return false;
+                }
+            }
+
+            @Override
+            public int docValueCount() {
+                return count;
+            }
+
+            @Override
+            public BytesRef nextValue() throws IOException {
+                scratch.length = in.readVInt();
+                scratch.offset = in.getPosition();
+                in.setPosition(scratch.offset + scratch.length);
+                return scratch;
+            }
+
+        };
+    }
+
+
+    @Override
+    public void close() {
+        // no-op
+    }
+}

server/src/main/java/org/elasticsearch/index/fielddata/plain/BytesBinaryDVLeafFieldData.java

@@ -20,83 +20,18 @@
 package org.elasticsearch.index.fielddata.plain;
 
 import org.apache.lucene.index.BinaryDocValues;
-import org.apache.lucene.store.ByteArrayDataInput;
-import org.apache.lucene.util.Accountable;
 import org.apache.lucene.util.BytesRef;
-import org.elasticsearch.index.fielddata.LeafFieldData;
 import org.elasticsearch.index.fielddata.ScriptDocValues;
-import org.elasticsearch.index.fielddata.SortedBinaryDocValues;
 
-import java.io.IOException;
-import java.util.Collection;
-import java.util.Collections;
-
-final class BytesBinaryDVLeafFieldData implements LeafFieldData {
-
-    private final BinaryDocValues values;
 
+final class BytesBinaryDVLeafFieldData extends AbstractBinaryDVLeafFieldData {
     BytesBinaryDVLeafFieldData(BinaryDocValues values) {
-        super();
-        this.values = values;
-    }
-
-    @Override
-    public long ramBytesUsed() {
-        return 0; // not exposed by Lucene
-    }
-
-    @Override
-    public Collection<Accountable> getChildResources() {
-        return Collections.emptyList();
-    }
-
-    @Override
-    public SortedBinaryDocValues getBytesValues() {
-        return new SortedBinaryDocValues() {
-
-            int count;
-            final ByteArrayDataInput in = new ByteArrayDataInput();
-            final BytesRef scratch = new BytesRef();
-
-            @Override
-            public boolean advanceExact(int doc) throws IOException {
-                if (values.advanceExact(doc)) {
-                    final BytesRef bytes = values.binaryValue();
-                    assert bytes.length > 0;
-                    in.reset(bytes.bytes, bytes.offset, bytes.length);
-                    count = in.readVInt();
-                    scratch.bytes = bytes.bytes;
-                    return true;
-                } else {
-                    return false;
-                }
-            }
-
-            @Override
-            public int docValueCount() {
-                return count;
-            }
-
-            @Override
-            public BytesRef nextValue() throws IOException {
-                scratch.length = in.readVInt();
-                scratch.offset = in.getPosition();
-                in.setPosition(scratch.offset + scratch.length);
-                return scratch;
-            }
-
-        };
+        super(values);
     }
 
     @Override
     public ScriptDocValues<BytesRef> getScriptValues() {
         return new ScriptDocValues.BytesRefs(getBytesValues());
     }
-
-    @Override
-    public void close() {
-        // no-op
-    }
-
-
 }
+

server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryDVLeafFieldData.java

@@ -0,0 +1,34 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.index.BinaryDocValues;
+import org.elasticsearch.index.fielddata.ScriptDocValues;
+
+final class StringBinaryDVLeafFieldData extends AbstractBinaryDVLeafFieldData{
+    StringBinaryDVLeafFieldData(BinaryDocValues values) {
+        super(values);
+    }
+
+    @Override
+    public ScriptDocValues<String> getScriptValues() {
+        return new ScriptDocValues.Strings(getBytesValues());
+    }
+}

server/src/main/java/org/elasticsearch/index/fielddata/plain/StringBinaryIndexFieldData.java

@@ -0,0 +1,95 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.index.fielddata.plain;
+
+import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.search.SortField;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.index.Index;
+import org.elasticsearch.index.fielddata.IndexFieldData;
+import org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested;
+import org.elasticsearch.index.fielddata.fieldcomparator.BytesRefFieldComparatorSource;
+import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.MultiValueMode;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.sort.BucketedSort;
+import org.elasticsearch.search.sort.SortOrder;
+
+import java.io.IOException;
+
+public class StringBinaryIndexFieldData implements IndexFieldData<StringBinaryDVLeafFieldData> {
+
+    protected final Index index;
+    protected final String fieldName;
+    protected final ValuesSourceType valuesSourceType;
+
+    public StringBinaryIndexFieldData(Index index, String fieldName, ValuesSourceType valuesSourceType) {
+        this.index = index;
+        this.fieldName = fieldName;
+        this.valuesSourceType = valuesSourceType;
+    }
+
+    @Override
+    public final String getFieldName() {
+        return fieldName;
+    }
+
+    @Override
+    public ValuesSourceType getValuesSourceType() {
+        return valuesSourceType;
+    }
+
+    @Override
+    public final void clear() {
+        // can't do
+    }
+
+    @Override
+    public final Index index() {
+        return index;
+    }
+
+    @Override
+    public SortField sortField(Object missingValue, MultiValueMode sortMode, Nested nested, boolean reverse) {
+        XFieldComparatorSource source = new BytesRefFieldComparatorSource(this, missingValue,
+                sortMode, nested);
+        return new SortField(getFieldName(), source, reverse);
+    }
+
+    @Override
+    public StringBinaryDVLeafFieldData load(LeafReaderContext context) {
+        try {
+            return new StringBinaryDVLeafFieldData(DocValues.getBinary(context.reader(), fieldName));
+        } catch (IOException e) {
+            throw new IllegalStateException("Cannot load doc values", e);
+        }
+    } 
+    @Override
+    public BucketedSort newBucketedSort(BigArrays bigArrays, Object missingValue, MultiValueMode sortMode, Nested nested,
+            SortOrder sortOrder, DocValueFormat format, int bucketSize, BucketedSort.ExtraData extra) {
+        throw new IllegalArgumentException("can't sort on binary field");
+    }
+
+    @Override
+    public StringBinaryDVLeafFieldData loadDirect(LeafReaderContext context) throws Exception {
+        return load(context);
+    }
+}

x-pack/plugin/src/test/resources/rest-api-spec/test/wildcard/10_wildcard_basic.yml

@@ -229,6 +229,23 @@ setup:
   - length: { aggregations.top_vals.buckets: 2 }
 
 ---
+"Scripted Aggs work":
+  - do:
+      search:
+        body:
+          track_total_hits: true
+          query:
+            wildcard:
+              my_wildcard: {value: "*goodbye*" }
+          aggs:
+            top_vals:
+              terms: {script: "doc['my_wildcard']" }
+
+
+  - match: {hits.total.value: 1}
+  - length: { aggregations.top_vals.buckets: 1 }
+  - match: { aggregations.top_vals.buckets.0.key: "goodbye world" }
+---
 "Sort works":
   - do:
       search: