From 9ffbb03e37184c44c256a22a45e227032f554723 Mon Sep 17 00:00:00 2001 From: markharwood Date: Wed, 18 Nov 2020 14:36:20 +0000 Subject: [PATCH 01/31] A TermsEnum API for discovering terms in the index. Terms matching a given prefix can be returned in alphabetical or by popularity, A timeout can limit the amount of time spent looking for matches. Expected to be useful in auto-complete or regex debugging use cases. --- docs/reference/search/term-enum.asciidoc | 106 ++++ .../index/mapper/KeywordFieldMapper.java | 33 + .../index/mapper/MappedFieldType.java | 17 + .../flattened/FlattenedFieldMapper.java | 122 ++++ .../flattened/FlattenedFieldParser.java | 10 + .../elasticsearch/threadpool/ThreadPool.java | 2 + .../xpack/core/XPackClientPlugin.java | 5 +- .../elasticsearch/xpack/core/XPackPlugin.java | 5 + .../termenum/action/MultiShardTermsEnum.java | 158 +++++ .../termenum/action/NodeTermEnumRequest.java | 163 +++++ .../termenum/action/NodeTermEnumResponse.java | 68 ++ .../termenum/action/SimpleTermCountEnum.java | 121 ++++ .../xpack/core/termenum/action/TermCount.java | 107 ++++ .../core/termenum/action/TermEnumAction.java | 46 ++ .../core/termenum/action/TermEnumRequest.java | 193 ++++++ .../action/TermEnumRequestBuilder.java | 37 ++ .../termenum/action/TermEnumResponse.java | 116 ++++ .../action/TransportTermEnumAction.java | 593 ++++++++++++++++++ .../core/termenum/action/package-info.java | 10 + .../termenum/rest/RestTermEnumAction.java | 53 ++ .../termenum/MultiShardTermsEnumTests.java | 126 ++++ .../xpack/core/termenum/TermCountTests.java | 42 ++ .../core/termenum/TermEnumResponseTests.java | 87 +++ .../TransportTermEnumActionTests.java | 45 ++ .../action/RestTermEnumActionTests.java | 148 +++++ .../mapper/ConstantKeywordFieldMapper.java | 18 + .../xpack/security/operator/Constants.java | 1 + .../rest-api-spec/api/termsenum.json | 35 ++ .../rest-api-spec/test/term_enum/10_basic.yml | 82 +++ 29 files changed, 2548 insertions(+), 1 deletion(-) create mode 100644 docs/reference/search/term-enum.asciidoc create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/MultiShardTermsEnum.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumResponse.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/SimpleTermCountEnum.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermCount.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequestBuilder.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumResponse.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/package-info.java create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/rest/RestTermEnumAction.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/MultiShardTermsEnumTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermCountTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermEnumResponseTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TransportTermEnumActionTests.java create mode 100644 x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/action/RestTermEnumActionTests.java create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/api/termsenum.json create mode 100644 x-pack/plugin/src/test/resources/rest-api-spec/test/term_enum/10_basic.yml diff --git a/docs/reference/search/term-enum.asciidoc b/docs/reference/search/term-enum.asciidoc new file mode 100644 index 0000000000000..018e95891ae96 --- /dev/null +++ b/docs/reference/search/term-enum.asciidoc @@ -0,0 +1,106 @@ +[[search-term-enum]] +=== Term enum API + +The term enum API can be used to discover terms in the index that match +a partial string. This is used for features like auto-complete: + +[source,console] +-------------------------------------------------- +POST stackoverflow/_terms +{ + "field" : "tags", + "string" : "kiba" +} +-------------------------------------------------- +// TEST[setup:stackoverflow] + + +The API returns the following response: + +[source,console-result] +-------------------------------------------------- +{ + "_shards": { + "total": 1, + "successful": 1, + "failed": 0 + }, + "terms": [ + "kibana" + ], + "complete" : true +} +-------------------------------------------------- +// TESTRESPONSE[s/8/$body.terms.0.doc_count/] + +The "complete" flag is false if time or space constraints were met and the +set of terms examined was not the full set of available values. + +[[search-term-enum-api-request]] +==== {api-request-title} + +`GET //_terms` + + +[[search-term-enum-api-desc]] +==== {api-description-title} + +The termenum API can be used to discover terms in the index that begin with the provided +string. It is designed for low-latency look-ups used in auto-complete scenarios. + + +[[search-term-enum-api-path-params]] +==== {api-path-parms-title} + +``:: +(Mandatory, string) +Comma-separated list of data streams, indices, and index aliases to search. +Wildcard (`*`) expressions are supported. ++ +To search all data streams or indices in a cluster, omit this parameter or use +`_all` or `*`. + +[[search-term-enum-api-request-body]] +==== {api-request-body-title} + +[[term-enum-field-param]] +`field`:: +(Mandatory, string) +Which field to match + +[[term-enum-string-param]] +`string`:: +(Mandatory, string) +The string to match at the start of indexed terms + +[[term-enum-size-param]] +`size`:: +(Optional, integer) +How many matching terms to return. Defaults to 10 + +[[term-enum-timeout-param]] +`timeout`:: +(Optional, integer) +The maximum length of time in milliseconds to spend collecting results. Defaults to 1000. +If the timeout is exceeded a `timed_out` flag is set in the response and the results may +be partial or empty. + +[[term-enum-case_insensitive-param]] +`case_insensitive`:: +(Optional, boolean) +When true the provided search string is matched against index terms without case sensitivity. +Defaults to false. + +[[term-enum-sort_by_popularity-param]] +`sort_by_popularity`:: +(Optional, boolean) +When true terms are sorted by popularity, when false (the default) they are sorted alphabetically. +Sorting by popularity can be more costly so nodes will only consider up to a few thousand matching +terms to bound the cost of this calculation. If this term limit is reached in the analysis +the final response will have the `complete` flag set to false. + +[[term-enum-index_filter-param]] +`index_filter`:: +(Optional, <> Allows to filter an index shard if the provided +query rewrites to `match_none`. + diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 1842d5afc0387..96a62807ce285 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -17,8 +17,18 @@ import org.apache.lucene.index.LeafReaderContext; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.Query; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.MultiTerms; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Automata; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.apache.lucene.util.automaton.MinimizationOperations; +import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.common.lucene.Lucene; +import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.index.analysis.IndexAnalyzers; import org.elasticsearch.index.analysis.NamedAnalyzer; @@ -247,6 +257,27 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { this.eagerGlobalOrdinals = false; this.scriptValues = null; } + + + + @Override + public TermsEnum getTerms(boolean caseInsensitive, String string, SearchExecutionContext queryShardContext) throws IOException { + IndexReader reader = queryShardContext.searcher().getTopReaderContext().reader(); + + Terms terms = MultiTerms.getTerms(reader, name()); + if (terms == null) { + // Field does not exist on this shard. + return null; + } + Automaton a = caseInsensitive + ? AutomatonQueries.caseInsensitivePrefix(string) + : Automata.makeString(string); + a = Operations.concatenate(a, Automata.makeAnyString()); + a = MinimizationOperations.minimize(a, Integer.MAX_VALUE); + + CompiledAutomaton automaton = new CompiledAutomaton(a); + return automaton.getTermsEnum(terms); + } @Override public String typeName() { @@ -470,4 +501,6 @@ protected String contentType() { public FieldMapper.Builder getMergeBuilder() { return new Builder(simpleName(), indexAnalyzers, scriptCompiler).init(this); } + + } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java index e7bee7de8aac6..861fd0ac24971 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/MappedFieldType.java @@ -13,6 +13,7 @@ import org.apache.lucene.index.PrefixCodedTerms; import org.apache.lucene.index.PrefixCodedTerms.TermIterator; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.queries.intervals.IntervalsSource; import org.apache.lucene.search.BooleanClause.Occur; import org.apache.lucene.search.BooleanQuery; @@ -429,4 +430,20 @@ public enum CollapseType { KEYWORD, NUMERIC } + + /** + * This method is used to support auto-complete services and implementations + * are expected to find terms beginning with the provided string very quickly. + * If fields cannot look up matching terms quickly they should return null. + * The returned TermEnum should implement next(), term() and doc_freq() methods + * but postings etc are not required. + * @param caseInsensitive if matches should be case insensitive + * @param string the partially complete word the user has typed (can be empty) + * @param queryShardContext the shard context + * @return null or an enumeration of matching terms and their doc frequencies + * @throws IOException Errors accessing data + */ + public TermsEnum getTerms(boolean caseInsensitive, String string, SearchExecutionContext queryShardContext) throws IOException { + return null; + } } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 5c132b2eaae09..23451868994ff 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -9,14 +9,27 @@ package org.elasticsearch.index.mapper.flattened; import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.ImpactsEnum; +import org.apache.lucene.index.IndexReader; import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.MultiTerms; import org.apache.lucene.index.OrdinalMap; +import org.apache.lucene.index.PostingsEnum; import org.apache.lucene.index.Term; +import org.apache.lucene.index.TermState; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.MultiTermQuery; import org.apache.lucene.search.PrefixQuery; import org.apache.lucene.search.Query; import org.apache.lucene.search.SortField; +import org.apache.lucene.util.AttributeSource; import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.automaton.Automata; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.apache.lucene.util.automaton.MinimizationOperations; +import org.apache.lucene.util.automaton.Operations; import org.elasticsearch.common.lucene.Lucene; import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.common.unit.Fuzziness; @@ -241,6 +254,26 @@ public Query wildcardQuery(String value, public Query termQueryCaseInsensitive(Object value, SearchExecutionContext context) { return AutomatonQueries.caseInsensitiveTermQuery(new Term(name(), indexedValueForSearch(value))); } + + @Override + public TermsEnum getTerms(boolean caseInsensitive, String string, SearchExecutionContext queryShardContext) throws IOException { + IndexReader reader = queryShardContext.searcher().getTopReaderContext().reader(); + String searchString = FlattenedFieldParser.createKeyedValue(key, string); + Terms terms = MultiTerms.getTerms(reader, name()); + if (terms == null) { + // Field does not exist on this shard. + return null; + } + Automaton a = caseInsensitive + ? AutomatonQueries.caseInsensitivePrefix(searchString) + : Automata.makeString(searchString); + a = Operations.concatenate(a, Automata.makeAnyString()); + a = MinimizationOperations.minimize(a, Integer.MAX_VALUE); + + CompiledAutomaton automaton = new CompiledAutomaton(a); + // Wrap result in a class that strips field names from discovered terms + return new TranslatingTermsEnum(automaton.getTermsEnum(terms)); + } @Override public BytesRef indexedValueForSearch(Object value) { @@ -270,6 +303,95 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) return SourceValueFetcher.identity(rootName + "." + key, context, format); } } + + + // Wraps a raw Lucene TermsEnum to strip values of fieldnames + static class TranslatingTermsEnum extends TermsEnum{ + TermsEnum delegate; + + TranslatingTermsEnum(TermsEnum delegate) { + this.delegate = delegate; + } + + @Override + public BytesRef next() throws IOException { + // Strip the term of the fieldname value + BytesRef result = delegate.next(); + if(result != null) { + result = FlattenedFieldParser.extractValue(result); + } + return result; + } + + @Override + public BytesRef term() throws IOException { + // Strip the term of the fieldname value + BytesRef result = delegate.term(); + if(result != null) { + result = FlattenedFieldParser.extractValue(result); + } + return result; + } + + + @Override + public int docFreq() throws IOException { + return delegate.docFreq(); + } + + //=============== All other TermsEnum methods not supported ================= + + @Override + public AttributeSource attributes() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean seekExact(BytesRef text) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public SeekStatus seekCeil(BytesRef text) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void seekExact(long ord) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void seekExact(BytesRef term, TermState state) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long ord() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long totalTermFreq() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public ImpactsEnum impacts(int flags) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public TermState termState() throws IOException { + throw new UnsupportedOperationException(); + } + + } /** * A field data implementation that gives access to the values associated with diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java index 922d45ca7cf88..23420ea47171a 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java @@ -166,4 +166,14 @@ static BytesRef extractKey(BytesRef keyedValue) { } return new BytesRef(keyedValue.bytes, keyedValue.offset, length); } + static BytesRef extractValue(BytesRef keyedValue) { + int length; + for (length = 0; length < keyedValue.length; length++){ + if (keyedValue.bytes[keyedValue.offset + length] == SEPARATOR_BYTE) { + break; + } + } + int valueStart = keyedValue.offset + length + 1; + return new BytesRef(keyedValue.bytes, valueStart, keyedValue.length - valueStart ); + } } diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index 4c6a2302e3b7d..1f99ad82bbd31 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -62,6 +62,7 @@ public static class Names { public static final String ANALYZE = "analyze"; public static final String WRITE = "write"; public static final String SEARCH = "search"; + public static final String AUTO_COMPLETE = "auto_complete"; public static final String SEARCH_THROTTLED = "search_throttled"; public static final String MANAGEMENT = "management"; public static final String FLUSH = "flush"; @@ -177,6 +178,7 @@ public ThreadPool(final Settings settings, final ExecutorBuilder... customBui builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, allocatedProcessors, 1000, false)); builders.put(Names.ANALYZE, new FixedExecutorBuilder(settings, Names.ANALYZE, 1, 16, false)); builders.put(Names.SEARCH, new FixedExecutorBuilder(settings, Names.SEARCH, searchThreadPoolSize(allocatedProcessors), 1000, true)); + builders.put(Names.AUTO_COMPLETE, new FixedExecutorBuilder(settings, Names.AUTO_COMPLETE, 2, 10, true)); builders.put(Names.SEARCH_THROTTLED, new FixedExecutorBuilder(settings, Names.SEARCH_THROTTLED, 1, 100, true)); builders.put(Names.MANAGEMENT, new ScalingExecutorBuilder(Names.MANAGEMENT, 1, boundedBy(allocatedProcessors, 1, 5), TimeValue.timeValueMinutes(5))); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 9aa531a70d136..9f58637c0262a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -201,6 +201,7 @@ import org.elasticsearch.xpack.core.sql.SqlFeatureSetUsage; import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction; import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction; +import org.elasticsearch.xpack.core.termenum.action.TermEnumAction; import org.elasticsearch.xpack.core.transform.TransformFeatureSetUsage; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.TransformMetadata; @@ -413,7 +414,9 @@ public List> getClientActions() { GetAsyncSearchAction.INSTANCE, DeleteAsyncResultAction.INSTANCE, // Text Structure - FindStructureAction.INSTANCE + FindStructureAction.INSTANCE, + // Termenum API + TermEnumAction.INSTANCE )); // rollupV2 diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index ee1132c524bef..6cc0e092ea552 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -83,6 +83,9 @@ import org.elasticsearch.xpack.core.ssl.SSLConfigurationReloader; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.core.transform.TransformMetadata; +import org.elasticsearch.xpack.core.termenum.action.TermEnumAction; +import org.elasticsearch.xpack.core.termenum.action.TransportTermEnumAction; +import org.elasticsearch.xpack.core.termenum.rest.RestTermEnumAction; import org.elasticsearch.xpack.core.watcher.WatcherMetadata; import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants; @@ -291,6 +294,7 @@ public Collection createComponents(Client client, ClusterService cluster actions.add(new ActionHandler<>(XPackUsageAction.INSTANCE, getUsageAction())); actions.addAll(licensing.getActions()); actions.add(new ActionHandler<>(ReloadAnalyzerAction.INSTANCE, TransportReloadAnalyzersAction.class)); + actions.add(new ActionHandler<>(TermEnumAction.INSTANCE, TransportTermEnumAction.class)); actions.add(new ActionHandler<>(DeleteAsyncResultAction.INSTANCE, TransportDeleteAsyncResultAction.class)); actions.add(new ActionHandler<>(XPackInfoFeatureAction.DATA_TIERS, DataTiersInfoTransportAction.class)); actions.add(new ActionHandler<>(XPackUsageFeatureAction.DATA_TIERS, DataTiersUsageTransportAction.class)); @@ -330,6 +334,7 @@ public List getRestHandlers(Settings settings, RestController restC handlers.add(new RestXPackInfoAction()); handlers.add(new RestXPackUsageAction()); handlers.add(new RestReloadAnalyzersAction()); + handlers.add(new RestTermEnumAction()); handlers.addAll(licensing.getRestHandlers(settings, restController, clusterSettings, indexScopedSettings, settingsFilter, indexNameExpressionResolver, nodesInCluster)); return handlers; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/MultiShardTermsEnum.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/MultiShardTermsEnum.java new file mode 100644 index 0000000000000..43461102b4e6c --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/MultiShardTermsEnum.java @@ -0,0 +1,158 @@ +/* @notice + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache license, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the license for the specific language governing permissions and + * limitations under the license. + */ +package org.elasticsearch.xpack.core.termenum.action; + +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.PriorityQueue; + +import java.io.IOException; + +/** + * Merges terms and stats from multiple TermEnum classes + * This does a merge sort, by term text. + * Adapted from Lucene's MultiTermsEnum and differs in that: + * 1) Only next(), term() and docFreq() methods are supported + * 2) Doc counts are longs not ints. + * + */ +public final class MultiShardTermsEnum { + + private final TermMergeQueue queue; + private final TermsEnumWithCurrent[] top; + + private int numTop; + private BytesRef current; + + /** Sole constructor. + * @param enums TermsEnums from shards which we should merge + * @throws IOException Errors accessing data + **/ + public MultiShardTermsEnum(TermsEnum[] enums) throws IOException { + queue = new TermMergeQueue(enums.length); + top = new TermsEnumWithCurrent[enums.length]; + numTop = 0; + queue.clear(); + for (int i = 0; i < enums.length; i++) { + final TermsEnum termsEnum = enums[i]; + final BytesRef term = termsEnum.next(); + if (term != null) { + final TermsEnumWithCurrent entry = new TermsEnumWithCurrent(); + entry.current = term; + entry.terms = termsEnum; + queue.add(entry); + } else { + // field has no terms + } + } + } + + public BytesRef term() { + return current; + } + + private void pullTop() { + assert numTop == 0; + numTop = queue.fillTop(top); + current = top[0].current; + } + + private void pushTop() throws IOException { + // call next() on each top, and reorder queue + for (int i = 0; i < numTop; i++) { + TermsEnumWithCurrent top = queue.top(); + top.current = top.terms.next(); + if (top.current == null) { + queue.pop(); + } else { + queue.updateTop(); + } + } + numTop = 0; + } + + public BytesRef next() throws IOException { + pushTop(); + // gather equal top fields + if (queue.size() > 0) { + // TODO: we could maybe defer this somewhat costly operation until one of the APIs that + // needs to see the top is invoked (docFreq, postings, etc.) + pullTop(); + } else { + current = null; + } + + return current; + } + + public long docFreq() throws IOException { + long sum = 0; + for (int i = 0; i < numTop; i++) { + sum += top[i].terms.docFreq(); + } + return sum; + } + + static final class TermsEnumWithCurrent { + TermsEnum terms; + public BytesRef current; + } + + private static final class TermMergeQueue extends PriorityQueue { + final int[] stack; + + TermMergeQueue(int size) { + super(size); + this.stack = new int[size]; + } + + @Override + protected boolean lessThan(TermsEnumWithCurrent termsA, TermsEnumWithCurrent termsB) { + return termsA.current.compareTo(termsB.current) < 0; + } + + /** Add the {@link #top()} slice as well as all slices that are positioned + * on the same term to {@code tops} and return how many of them there are. */ + int fillTop(TermsEnumWithCurrent[] tops) { + final int size = size(); + if (size == 0) { + return 0; + } + tops[0] = top(); + int numTop = 1; + stack[0] = 1; + int stackLen = 1; + + while (stackLen != 0) { + final int index = stack[--stackLen]; + final int leftChild = index << 1; + for (int child = leftChild, end = Math.min(size, leftChild + 1); child <= end; ++child) { + TermsEnumWithCurrent te = get(child); + if (te.current.equals(tops[0].current)) { + tops[numTop++] = te; + stack[stackLen++] = child; + } + } + } + return numTop; + } + + private TermsEnumWithCurrent get(int i) { + return (TermsEnumWithCurrent) getHeapArray()[i]; + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java new file mode 100644 index 0000000000000..9e16110234420 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java @@ -0,0 +1,163 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum.action; + +import org.elasticsearch.action.IndicesRequest; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.transport.TransportRequest; + +import java.io.IOException; +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; + +/** + * Internal termenum request executed directly against a specific node, querying potentially many + * shards in one request + */ +public class NodeTermEnumRequest extends TransportRequest implements IndicesRequest { + + private String field; + private String string; + private long taskStartedTimeMillis; + private long nodeStartedTimeMillis; + private boolean caseInsensitive; + private boolean sortByPopularity; + private int size; + private long timeout; + private final QueryBuilder indexFilter; + private Set shardIds; + private String nodeId; + + + public NodeTermEnumRequest(StreamInput in) throws IOException { + super(in); + // Set the clock running as soon as we appear on a node. + nodeStartedTimeMillis = System.currentTimeMillis(); + + field = in.readString(); + string = in.readString(); + caseInsensitive = in.readBoolean(); + sortByPopularity = in.readBoolean(); + size = in.readVInt(); + timeout = in.readVLong(); + taskStartedTimeMillis = in.readVLong(); + indexFilter = in.readOptionalNamedWriteable(QueryBuilder.class); + nodeId = in.readString(); + int numShards = in.readVInt(); + shardIds = new HashSet<>(numShards); + for (int i = 0; i < numShards; i++) { + shardIds.add(new ShardId(in)); + } + } + + public NodeTermEnumRequest(final String nodeId, final Set shardIds, TermEnumRequest request) { + this.field = request.field(); + this.string = request.string(); + this.caseInsensitive = request.caseInsensitive(); + this.size = request.size(); + this.timeout = request.timeout().getMillis(); + this.sortByPopularity = request.sortByPopularity(); + this.taskStartedTimeMillis = request.taskStartTimeMillis; + this.indexFilter = request.indexFilter(); + this.nodeId = nodeId; + this.shardIds = shardIds; + + // TODO serialize shard ids + } + + public String field() { + return field; + } + + public String string() { + return string; + } + + public long taskStartedTimeMillis() { + return this.taskStartedTimeMillis; + } + + /** + * The time this request was materialized on a shard + * (defaults to "now" if serialization was not used e.g. a local request). + */ + public long shardStartedTimeMillis() { + if (nodeStartedTimeMillis == 0) { + nodeStartedTimeMillis = System.currentTimeMillis(); + } + return this.nodeStartedTimeMillis; + } + + public Set shardIds() { + return Collections.unmodifiableSet(shardIds); + } + + public boolean caseInsensitive() { + return caseInsensitive; + } + + public boolean sortByPopularity() { + return sortByPopularity; + } + public int size() { + return size; + } + + public long timeout() { + return timeout; + } + public String nodeId() { + return nodeId; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(field); + out.writeString(string); + out.writeBoolean(caseInsensitive); + out.writeBoolean(sortByPopularity); + out.writeVInt(size); + // Adjust the amount of permitted time the shard has remaining to gather terms. + long timeSpentSoFarInCoordinatingNode = System.currentTimeMillis() - taskStartedTimeMillis; + assert timeSpentSoFarInCoordinatingNode >= 0; + int remainingTimeForShardToUse = (int) (timeout - timeSpentSoFarInCoordinatingNode); + // TODO - if already timed out can we shortcut the trip somehow? Throw exception if remaining time < 0? + out.writeVLong(remainingTimeForShardToUse); + out.writeVLong(taskStartedTimeMillis); + out.writeOptionalNamedWriteable(indexFilter); + out.writeString(nodeId); + out.writeVInt(shardIds.size()); + for (ShardId shardId : shardIds) { + shardId.writeTo(out); + } + } + + public QueryBuilder indexFilter() { + return indexFilter; + } + + @Override + public String[] indices() { + HashSet indicesNames = new HashSet<>(); + for (ShardId shardId : shardIds) { + indicesNames.add(shardId.getIndexName()); + } + return indicesNames.toArray(new String[0]); + } + + @Override + public IndicesOptions indicesOptions() { + return null; + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumResponse.java new file mode 100644 index 0000000000000..7774b8c6e6bf7 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumResponse.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum.action; + +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.transport.TransportResponse; + +import java.io.IOException; +import java.util.List; + +/** + * Internal response of a term enum request executed directly against a specific shard. + * + * + */ +class NodeTermEnumResponse extends TransportResponse { + + private String error; + private boolean complete; + + private List terms; + private String nodeId; + + NodeTermEnumResponse(StreamInput in) throws IOException { + super(in); + terms = in.readList(TermCount::new); + error = in.readOptionalString(); + complete = in.readBoolean(); + nodeId = in.readString(); + } + + NodeTermEnumResponse(String nodeId, List terms, String error, boolean complete) { + this.nodeId = nodeId; + this.terms = terms; + this.error = error; + this.complete = complete; + } + + public List terms() { + return this.terms; + } + + public String getError() { + return error; + } + + public String getNodeId() { + return nodeId; + } + + public boolean getComplete() { + return complete; + } + + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeCollection(terms); + out.writeOptionalString(error); + out.writeBoolean(complete); + out.writeString(nodeId); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/SimpleTermCountEnum.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/SimpleTermCountEnum.java new file mode 100644 index 0000000000000..5c1a1b7dc63eb --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/SimpleTermCountEnum.java @@ -0,0 +1,121 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum.action; + +import org.apache.lucene.index.ImpactsEnum; +import org.apache.lucene.index.PostingsEnum; +import org.apache.lucene.index.TermState; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.AttributeSource; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.index.mapper.MappedFieldType; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Comparator; + +/** + * A utility class for fields that need to support autocomplete via + * {@link MappedFieldType#getTerms(boolean, String, org.elasticsearch.index.query.SearchExecutionContext)} + * but can't return a raw Lucene TermsEnum. + */ +public class SimpleTermCountEnum extends TermsEnum{ + int index =-1; + TermCount[] sortedTerms; + TermCount current = null; + + public SimpleTermCountEnum(TermCount[] terms) { + sortedTerms = terms; + Arrays.sort(sortedTerms, Comparator.comparing(TermCount::getTerm)); + } + + public SimpleTermCountEnum(TermCount termCount) { + sortedTerms = new TermCount[1]; + sortedTerms[0] = termCount; + } + + @Override + public BytesRef term() throws IOException { + if (current == null) { + return null; + } + return new BytesRef(current.getTerm()); + } + + @Override + public BytesRef next() throws IOException { + index++; + if(index >= sortedTerms.length) { + current = null; + } else { + current = sortedTerms[index]; + } + return term(); + } + + @Override + public int docFreq() throws IOException { + if (current == null) { + return 0; + } + return (int) current.getDocCount(); + } + + + //=============== All other TermsEnum methods not supported ================= + + @Override + public AttributeSource attributes() { + throw new UnsupportedOperationException(); + } + + @Override + public boolean seekExact(BytesRef text) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public SeekStatus seekCeil(BytesRef text) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void seekExact(long ord) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public void seekExact(BytesRef term, TermState state) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long ord() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public long totalTermFreq() throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public PostingsEnum postings(PostingsEnum reuse, int flags) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public ImpactsEnum impacts(int flags) throws IOException { + throw new UnsupportedOperationException(); + } + + @Override + public TermState termState() throws IOException { + throw new UnsupportedOperationException(); + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermCount.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermCount.java new file mode 100644 index 0000000000000..2c6c0437d229c --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermCount.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum.action; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.ToXContentFragment; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Objects; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +public class TermCount implements Writeable, ToXContentFragment { + + public static final String TERM_FIELD = "term"; + public static final String DOC_COUNT_FIELD = "doc_count"; + + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "term_count", + true, + a -> { return new TermCount((String) a[0], (long) a[1]); } + ); + static { + PARSER.declareString(optionalConstructorArg(), new ParseField(TERM_FIELD)); + PARSER.declareLong(optionalConstructorArg(), new ParseField(DOC_COUNT_FIELD)); + } + + private String term; + + private long docCount; + + public TermCount(StreamInput in) throws IOException { + term = in.readString(); + docCount = in.readLong(); + } + + public TermCount(String term, long count) { + this.term = term; + this.docCount = count; + } + + public String getTerm() { + return this.term; + } + + public long getDocCount() { + return this.docCount; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + out.writeString(term); + out.writeLong(docCount); + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.field(TERM_FIELD, getTerm()); + builder.field(DOC_COUNT_FIELD, getDocCount()); + return builder; + } + + public static TermCount fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + TermCount other = (TermCount) o; + return Objects.equals(getTerm(), other.getTerm()) && Objects.equals(getDocCount(), other.getDocCount()); + } + + @Override + public int hashCode() { + return Objects.hash(getTerm(), getDocCount()); + } + + void addToDocCount(long extra) { + docCount += extra; + } + + void setTerm(String term) { + this.term = term; + } + + void setDocCount(long docCount) { + this.docCount = docCount; + } + + @Override + public String toString() { + return term + ":" + docCount; + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumAction.java new file mode 100644 index 0000000000000..eff9a77eceb82 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumAction.java @@ -0,0 +1,46 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum.action; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; + +public class TermEnumAction extends ActionType { + + public static final TermEnumAction INSTANCE = new TermEnumAction(); + public static final String NAME = "indices:data/read/xpack/termsenum/list"; + + + static final ParseField INDEX_FILTER = new ParseField("index_filter"); + + private TermEnumAction() { + super(NAME, TermEnumResponse::new); + } + + public static TermEnumRequest fromXContent(XContentParser parser, String... indices) throws IOException { + TermEnumRequest request = new TermEnumRequest(indices); + PARSER.parse(parser, request, null); + return request; + } + + private static final ObjectParser PARSER = new ObjectParser<>("terms_enum_request"); + static { + PARSER.declareString(TermEnumRequest::field, new ParseField("field")); + PARSER.declareString(TermEnumRequest::string, new ParseField("string")); + PARSER.declareInt(TermEnumRequest::size, new ParseField("size")); + PARSER.declareBoolean(TermEnumRequest::caseInsensitive, new ParseField("case_insensitive")); + PARSER.declareBoolean(TermEnumRequest::sortByPopularity, new ParseField("sort_by_popularity")); + PARSER.declareInt(TermEnumRequest::timeoutInMillis, new ParseField("timeout")); + PARSER.declareObject(TermEnumRequest::indexFilter, (p, context) -> parseInnerQueryBuilder(p),INDEX_FILTER); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java new file mode 100644 index 0000000000000..14318ded1d47a --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java @@ -0,0 +1,193 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum.action; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ValidateActions; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.broadcast.BroadcastRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryBuilder; + +import java.io.IOException; +import java.util.Arrays; + +/** + * A request to gather terms for a given field matching a pattern + */ +public class TermEnumRequest extends BroadcastRequest implements ToXContentObject { + + public static int DEFAULT_SIZE = 10; + public static int DEFAULT_TIMEOUT_MILLIS = 1000; + + private String field; + private String string; + private int size = DEFAULT_SIZE; + private boolean caseInsensitive; + private boolean sortByPopularity; + long taskStartTimeMillis; + private QueryBuilder indexFilter; + + public TermEnumRequest() { + this(Strings.EMPTY_ARRAY); + } + + public TermEnumRequest(StreamInput in) throws IOException { + super(in); + field = in.readString(); + string = in.readString(); + caseInsensitive = in.readBoolean(); + sortByPopularity = in.readBoolean(); + size = in.readVInt(); + indexFilter = in.readOptionalNamedWriteable(QueryBuilder.class); + } + + /** + * Constructs a new term enum request against the provided indices. No indices provided means it will + * run against all indices. + */ + public TermEnumRequest(String... indices) { + super(indices); + indicesOptions(IndicesOptions.fromOptions(false, false, true, false)); + timeout(new TimeValue(DEFAULT_TIMEOUT_MILLIS)); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = super.validate(); + if (field == null) { + validationException = ValidateActions.addValidationError("field cannot be null", validationException); + } + return validationException; + } + + /** + * The field to look inside for values + */ + public void field(String field) { + this.field = field; + } + + /** + * Indicates if detailed information about query is requested + */ + public String field() { + return field; + } + + /** + * The string required in matching field values + */ + public void string(String pattern) { + this.string = pattern; + } + + /** + * The string required in matching field values + */ + public String string() { + return string; + } + + /** + * sort terms by popularity + */ + public boolean sortByPopularity() { + return sortByPopularity; + } + + /** + * sort terms by popularity + */ + public void sortByPopularity(boolean sortByPopularity) { + this.sortByPopularity = sortByPopularity; + } + + /** + * The number of terms to return + */ + public int size() { + return size; + } + + /** + * The number of terms to return + */ + public void size(int size) { + this.size = size; + } + + /** + * TThe max time in milliseconds to spend gathering terms + */ + public void timeoutInMillis(int timeout) { + timeout(new TimeValue(timeout)); + } + + /** + * If case insensitive matching is required + */ + public void caseInsensitive(boolean caseInsensitive) { + this.caseInsensitive = caseInsensitive; + } + + /** + * If case insensitive matching is required + */ + public boolean caseInsensitive() { + return caseInsensitive; + } + + /** + * Allows to filter shards if the provided {@link QueryBuilder} rewrites to `match_none`. + */ + public void indexFilter(QueryBuilder indexFilter) { + this.indexFilter = indexFilter; + } + + public QueryBuilder indexFilter() { + return indexFilter; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeString(field); + out.writeString(string); + out.writeBoolean(caseInsensitive); + out.writeBoolean(sortByPopularity); + out.writeVInt(size); + out.writeOptionalNamedWriteable(indexFilter); + } + + @Override + public String toString() { + return "[" + Arrays.toString(indices) + "] field[" + field + "], pattern[" + string + "] " + " size=" + size + " timeout=" + + timeout().getMillis() + " sort_by_popularity = " + sortByPopularity + " case_insensitive=" + + caseInsensitive + " indexFilter = "+ indexFilter; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("field", field); + builder.field("pattern", string); + builder.field("size", size); + builder.field("timeout", timeout().getMillis()); + builder.field("case_insensitive", caseInsensitive); + builder.field("sort_by_popularity", sortByPopularity); + if (indexFilter != null) { + builder.field("index_filter", indexFilter); + } + return builder.endObject(); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequestBuilder.java new file mode 100644 index 0000000000000..267a689117e38 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequestBuilder.java @@ -0,0 +1,37 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum.action; + +import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; +import org.elasticsearch.client.ElasticsearchClient; + +public class TermEnumRequestBuilder extends BroadcastOperationRequestBuilder { + + public TermEnumRequestBuilder(ElasticsearchClient client, TermEnumAction action) { + super(client, action, new TermEnumRequest()); + } + + public TermEnumRequestBuilder setField(String field) { + request.field(field); + return this; + } + + public TermEnumRequestBuilder setString(String string) { + request.string(string); + return this; + } + + public TermEnumRequestBuilder setSortByPopularity(boolean sortByPopularity) { + request.sortByPopularity(sortByPopularity); + return this; + } + public TermEnumRequestBuilder setSize(int size) { + request.size(size); + return this; + } + +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumResponse.java new file mode 100644 index 0000000000000..2d7da3abce86c --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumResponse.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum.action; + +import org.elasticsearch.action.support.DefaultShardOperationFailedException; +import org.elasticsearch.action.support.broadcast.BroadcastResponse; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * The response of the termenum/list action. + */ +public class TermEnumResponse extends BroadcastResponse { + + public static final String TERMS_FIELD = "terms"; + public static final String COMPLETE_FIELD = "complete"; + + @SuppressWarnings("unchecked") + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "term_enum_results", + true, + arg -> { + BroadcastResponse response = (BroadcastResponse) arg[0]; + return new TermEnumResponse( + (List) arg[1], + response.getTotalShards(), + response.getSuccessfulShards(), + response.getFailedShards(), + Arrays.asList(response.getShardFailures()), + (Boolean) arg[2] + ); + } + ); + static { + declareBroadcastFields(PARSER); + PARSER.declareStringArray(optionalConstructorArg(), new ParseField(TERMS_FIELD)); + PARSER.declareBoolean(optionalConstructorArg(), new ParseField(COMPLETE_FIELD)); + } + + private final List terms; + + private boolean complete; + private int skippedShards; + + TermEnumResponse(StreamInput in) throws IOException { + super(in); + terms = in.readStringList(); + complete = in.readBoolean(); + skippedShards = in.readVInt(); + } + + public TermEnumResponse( + List terms, + int totalShards, + int successfulShards, + int failedShards, + List shardFailures, boolean complete + ) { + super(totalShards, successfulShards, failedShards, shardFailures); + this.terms = terms == null ? Collections.emptyList() : terms; + this.complete = complete; + } + + /** + * The list of terms. + */ + public List getTerms() { + return terms; + } + + /** + * The number of shards skipped by the index filter + */ + public int getSkippedShards() { + return skippedShards; + } + + @Override + public void writeTo(StreamOutput out) throws IOException { + super.writeTo(out); + out.writeStringCollection(terms); + out.writeBoolean(complete); + out.writeVInt(skippedShards); + } + + @Override + protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException { + if (getTerms() != null && getTerms().isEmpty() == false) { + builder.startArray(TERMS_FIELD); + for (String term : getTerms()) { + builder.value(term); + } + builder.endArray(); + } + builder.field(COMPLETE_FIELD, complete); + } + + public static TermEnumResponse fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java new file mode 100644 index 0000000000000..06108561628d9 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java @@ -0,0 +1,593 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum.action; + +import org.apache.logging.log4j.message.ParameterizedMessage; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.PriorityQueue; +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRunnable; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.DefaultShardOperationFailedException; +import org.elasticsearch.action.support.HandledTransportAction; +import org.elasticsearch.action.support.broadcast.BroadcastShardOperationFailedException; +import org.elasticsearch.cluster.ClusterState; +import org.elasticsearch.cluster.block.ClusterBlockException; +import org.elasticsearch.cluster.block.ClusterBlockLevel; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNode; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.cluster.routing.GroupShardsIterator; +import org.elasticsearch.cluster.routing.ShardIterator; +import org.elasticsearch.cluster.routing.ShardRouting; +import org.elasticsearch.cluster.service.ClusterService; +import org.elasticsearch.common.MemoizedSupplier; +import org.elasticsearch.common.inject.Inject; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.util.concurrent.ThreadContext; +import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.IndexSettings; +import org.elasticsearch.index.engine.Engine; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.SearchExecutionContext; +import org.elasticsearch.index.shard.IndexShard; +import org.elasticsearch.index.shard.ShardId; +import org.elasticsearch.indices.IndicesService; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.license.XPackLicenseState.Feature; +import org.elasticsearch.search.SearchService; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.internal.AliasFilter; +import org.elasticsearch.search.internal.ShardSearchRequest; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.TransportChannel; +import org.elasticsearch.transport.TransportException; +import org.elasticsearch.transport.TransportRequestHandler; +import org.elasticsearch.transport.TransportResponseHandler; +import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.DataTier; +import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; +import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; + +import java.io.Closeable; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReferenceArray; + +public class TransportTermEnumAction extends HandledTransportAction { + + protected final ClusterService clusterService; + protected final TransportService transportService; + private final IndicesService indicesService; + protected final IndexNameExpressionResolver indexNameExpressionResolver; + + + final String transportShardAction; + private final String shardExecutor; + private final XPackLicenseState licenseState; + + @Inject + public TransportTermEnumAction( + // NodeClient client, + ClusterService clusterService, + TransportService transportService, + IndicesService indicesService, + ActionFilters actionFilters, + XPackLicenseState licenseState, + IndexNameExpressionResolver indexNameExpressionResolver + ) { + super(TermEnumAction.NAME, transportService, actionFilters, TermEnumRequest::new); + + this.clusterService = clusterService; + this.transportService = transportService; + this.indexNameExpressionResolver = indexNameExpressionResolver; + this.transportShardAction = actionName + "[s]"; + this.shardExecutor = ThreadPool.Names.AUTO_COMPLETE; + this.indicesService = indicesService; + this.licenseState = licenseState; + + transportService.registerRequestHandler( + transportShardAction, + ThreadPool.Names.SAME, + NodeTermEnumRequest::new, + new NodeTransportHandler() + ); + + } + + @Override + protected void doExecute(Task task, TermEnumRequest request, ActionListener listener) { + request.taskStartTimeMillis = task.getStartTime(); + new AsyncBroadcastAction(task, request, listener).start(); + } + + protected NodeTermEnumRequest newNodeRequest(final String nodeId, final Set shardIds, TermEnumRequest request) { + // Given we look terms up in the terms dictionary alias filters is another aspect of search (like DLS) that we + // currently do not support. +// final ClusterState clusterState = clusterService.state(); +// final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); +// final AliasFilter aliasFilter = searchService.buildAliasFilter(clusterState, shard.getIndexName(), indicesAndAliases); + return new NodeTermEnumRequest(nodeId, shardIds, request); + } + + protected NodeTermEnumResponse readShardResponse(StreamInput in) throws IOException { + return new NodeTermEnumResponse(in); + } + + protected Map> getNodeBundles(ClusterState clusterState, TermEnumRequest request, String[] concreteIndices) { + // Group targeted shards by nodeId + Map> fastNodeBundles = new HashMap<>(); + for (String indexName : concreteIndices) { + Settings settings = clusterState.metadata().index(indexName).getSettings(); + if (IndexSettings.INDEX_SEARCH_THROTTLED.get(settings)) { + // ignore slow throttled indices (this includes frozen) + continue; + } + + String[] singleIndex = {indexName}; + + GroupShardsIterator shards = clusterService.operationRouting() + .searchShards(clusterState, singleIndex, null, null); + assert (shards.size() == 1); // We are only considering a single concrete index + ShardIterator shardsForIndex = shards.get(0); + for (ShardRouting shardRouting : shardsForIndex.getShardRoutings()) { + String nodeId = shardRouting.currentNodeId(); + + Set bundle = null; + if (fastNodeBundles.containsKey(nodeId)){ + bundle = fastNodeBundles.get(nodeId); + } else { + DiscoveryNode node = clusterState.getNodes().getDataNodes().get(nodeId); + //Only consider hot and warm nodes + if (DataTier.isHotNode(node) || DataTier.isWarmNode(node)) { + bundle = new HashSet(); + fastNodeBundles.put(nodeId, bundle); + } + } + if (bundle != null) { + bundle.add(shardRouting.shardId()); + } + } + } + return fastNodeBundles; + } + + protected ClusterBlockException checkGlobalBlock(ClusterState state, TermEnumRequest request) { + return state.blocks().globalBlockedException(ClusterBlockLevel.READ); + } + + protected ClusterBlockException checkRequestBlock(ClusterState state, TermEnumRequest countRequest, String[] concreteIndices) { + return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices); + } + + protected TermEnumResponse newResponse(TermEnumRequest request, AtomicReferenceArray nodesResponses, + ClusterState clusterState, boolean complete) { + int successfulShards = 0; + int failedShards = 0; + List shardFailures = null; + Map combinedResults = new HashMap(); + for (int i = 0; i < nodesResponses.length(); i++) { + Object shardResponse = nodesResponses.get(i); + if (shardResponse == null) { + // simply ignore non active shards + } else if (shardResponse instanceof BroadcastShardOperationFailedException) { + failedShards++; + if (shardFailures == null) { + shardFailures = new ArrayList<>(); + } + shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse)); + } else { + NodeTermEnumResponse str = (NodeTermEnumResponse) shardResponse; + // Only one node response has to be incomplete for the entire result to be labelled incomplete. + if (str.getComplete() == false) { + complete = false; + } + for (TermCount term : str.terms()) { + TermCount existingTc = combinedResults.get(term.getTerm()); + if (existingTc == null) { + combinedResults.put(term.getTerm(), term); + } else { + // add counts + existingTc.addToDocCount(term.getDocCount()); + } + } + successfulShards++; + } + } + int size = Math.min(request.size(), combinedResults.size()); + List terms = new ArrayList<>(size); + TermCount[] sortedCombinedResults = combinedResults.values().toArray(new TermCount[0]); + if (request.sortByPopularity()) { + // Sort by doc count descending + Arrays.sort(sortedCombinedResults, new Comparator() { + public int compare(TermCount t1, TermCount t2) { + return Long.compare(t2.getDocCount(), t1.getDocCount()); + } + }); + } else { + // Sort alphabetically + Arrays.sort(sortedCombinedResults, new Comparator() { + public int compare(TermCount t1, TermCount t2) { + return t1.getTerm().compareTo(t2.getTerm()); + } + }); + } + + for (TermCount term : sortedCombinedResults) { + terms.add(term.getTerm()); + if (terms.size() == size) { + break; + } + } + return new TermEnumResponse(terms, nodesResponses.length(), successfulShards, failedShards, shardFailures, complete); + } + + static class TermCountPriorityQueue extends PriorityQueue { + + TermCountPriorityQueue(int maxSize) { + super(maxSize); + } + + @Override + protected boolean lessThan(TermCount a, TermCount b) { + return a.getDocCount() < b.getDocCount(); + } + + } + + protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Task task) throws IOException { + List termsList = new ArrayList<>(); + String error = null; + + // DLS/FLS check copied from ResizeRequestInterceptor + // MH code - not sure this is the right context + ThreadContext threadContext = transportService.getThreadPool().getThreadContext(); + final XPackLicenseState frozenLicenseState = licenseState.copyCurrentLicenseState(); + + long timeout_millis = request.timeout(); + long scheduledEnd = request.shardStartedTimeMillis() + timeout_millis; + + ArrayList shardTermsEnums = new ArrayList<>(); + ArrayList openedResources = new ArrayList<>(); + try { + for (ShardId shardId : request.shardIds()) { + // Check we haven't just arrived on a node and time is up already. + if (System.currentTimeMillis() > scheduledEnd) { + return new NodeTermEnumResponse(request.nodeId(), termsList, error, false); + } + final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); + final IndexShard indexShard = indexService.getShard(shardId.getId()); + + Engine.Searcher searcher = indexShard.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE); + openedResources.add(searcher); + final SearchExecutionContext queryShardContext = indexService.newSearchExecutionContext( + shardId.id(), + 0, + searcher, + request::shardStartedTimeMillis, + null, + Collections.emptyMap() + ); + if (canAccess(shardId.getIndexName(), request.field(), frozenLicenseState, threadContext) && canMatchShard( + shardId, + request, + queryShardContext + )) { + + final MappedFieldType mappedFieldType = indexShard.mapperService().fieldType(request.field()); + if (mappedFieldType != null) { + TermsEnum terms = mappedFieldType.getTerms(request.caseInsensitive(), request.string(), queryShardContext); + if (terms != null) { + shardTermsEnums.add(terms); + } + } + } + } + MultiShardTermsEnum te = new MultiShardTermsEnum(shardTermsEnums.toArray(new TermsEnum[0])); + + int shard_size = request.size(); + // All the above prep might take a while - do a timer check now before we continue further. + if (System.currentTimeMillis() > scheduledEnd) { + return new NodeTermEnumResponse(request.nodeId(), termsList, error, false); + } + + int numTermsBetweenClockChecks = 100; + int termCount = 0; + if (request.sortByPopularity()) { + // Collect most popular matches + TermCountPriorityQueue pq = new TermCountPriorityQueue(shard_size); + TermCount spare = null; + // TODO make this a setting (cluster or index level?) + int maxTermsConsidered = 10000; + while (te.next() != null) { + termCount++; + if (termCount >= maxTermsConsidered) { + toList(pq, termsList); + boolean complete = te.next() == null; + return new NodeTermEnumResponse(request.nodeId(), termsList, error, complete); + } + if (termCount % numTermsBetweenClockChecks == 0) { + if (System.currentTimeMillis() > scheduledEnd) { + // Gather what we have collected so far + toList(pq, termsList); + boolean complete = te.next() == null; + return new NodeTermEnumResponse(request.nodeId(), termsList, error, complete); + } + } + long df = te.docFreq(); + BytesRef bytes = te.term(); + + if (spare == null) { + spare = new TermCount(bytes.utf8ToString(), df); + } else { + spare.setTerm(bytes.utf8ToString()); + spare.setDocCount(df); + } + spare = pq.insertWithOverflow(spare); + } + toList(pq, termsList); + } else { + // Collect in alphabetical order + while (te.next() != null) { + termCount++; + if (termCount > numTermsBetweenClockChecks) { + if (System.currentTimeMillis() > scheduledEnd) { + boolean complete = te.next() == null; + return new NodeTermEnumResponse(request.nodeId(), termsList, error, complete); + } + termCount = 0; + } + // MH TODO - multireader needs to deal with counts > 2bn - how to handle? limit num shards per reader? + long df = te.docFreq(); + BytesRef bytes = te.term(); + termsList.add(new TermCount(bytes.utf8ToString(), df)); + if (termsList.size() >= shard_size) { + break; + } + } + } + + } catch (Exception e) { + error = e.getMessage(); + } + finally { + IOUtils.close(openedResources); + } + return new NodeTermEnumResponse(request.nodeId(), termsList, error, true); + } + + protected void toList(TermCountPriorityQueue pq, List termsList) { + while (pq.size() > 0) { + termsList.add(pq.pop()); + } + } + + // TODO remove this so we can shift code to server module + private boolean canAccess(String indexName, String fieldName, XPackLicenseState frozenLicenseState, ThreadContext threadContext) { + if (frozenLicenseState.isSecurityEnabled()) { + var licenseChecker = new MemoizedSupplier<>(() -> frozenLicenseState.checkFeature(Feature.SECURITY_DLS_FLS)); + IndicesAccessControl indicesAccessControl = + threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY); + IndicesAccessControl.IndexAccessControl indexAccessControl = + indicesAccessControl.getIndexPermissions(indexName); + //TODO remove FLS check (the reader does that anyway) + // TODO remove DLS check by getting a new security feature to filter at coordinating node the choice of indices + // based on if ANY DLS is present on an index. + if (indexAccessControl != null) { + final boolean fls = indexAccessControl.getFieldPermissions().grantsAccessTo(fieldName) == false; + final boolean dls = indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions(); + if ((fls || dls) && licenseChecker.get()) { + return false; + } + } + } + return true; + } + + private boolean canMatchShard(ShardId shardId, NodeTermEnumRequest req, SearchExecutionContext queryShardContext) throws IOException { + if (req.indexFilter() == null || req.indexFilter() instanceof MatchAllQueryBuilder) { + return true; + } + ShardSearchRequest searchRequest = new ShardSearchRequest(shardId, req.shardStartedTimeMillis(), AliasFilter.EMPTY); + searchRequest.source(new SearchSourceBuilder().query(req.indexFilter())); + return SearchService.queryStillMatchesAfterRewrite(searchRequest, queryShardContext); + } + + protected class AsyncBroadcastAction { + + private final Task task; + private final TermEnumRequest request; + private ActionListener listener; + private final ClusterState clusterState; + private final DiscoveryNodes nodes; + private final int expectedOps; + private final AtomicInteger counterOps = new AtomicInteger(); + private final AtomicReferenceArray nodesResponses; + private Map> nodeBundles; + + protected AsyncBroadcastAction(Task task, TermEnumRequest request, ActionListener listener) { + this.task = task; + this.request = request; + this.listener = listener; + + clusterState = clusterService.state(); + + ClusterBlockException blockException = checkGlobalBlock(clusterState, request); + if (blockException != null) { + throw blockException; + } + // update to concrete indices + String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(clusterState, request); + blockException = checkRequestBlock(clusterState, request, concreteIndices); + if (blockException != null) { + throw blockException; + } + + nodes = clusterState.nodes(); + logger.trace("resolving shards based on cluster state version [{}]", clusterState.version()); + nodeBundles = getNodeBundles(clusterState, request, concreteIndices); + expectedOps = nodeBundles.size(); + + nodesResponses = new AtomicReferenceArray<>(expectedOps); + } + + public void start() { + if (nodeBundles.size() == 0) { + // no shards + try { + listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState, true)); + } catch (Exception e) { + listener.onFailure(e); + } + // TODO or remove above try and instead just call finishHim() here? Helps keep return logic consistent + return; + } + // count the local operations, and perform the non local ones + int nodeIndex = -1; + for (final String nodeId : nodeBundles.keySet()) { + if (checkForEarlyFinish()) { + return; + } + nodeIndex++; + Set shardIds = nodeBundles.get(nodeId); + if (shardIds.size() > 0) { + performOperation(nodeId, shardIds, nodeIndex); + } else { + // really, no shards active in this group + onNoOperation(nodeId); + } + } + } + + // Returns true if we exited with a response to the caller. + boolean checkForEarlyFinish() { + long now = System.currentTimeMillis(); + if ( (now - task.getStartTime()) > request.timeout().getMillis() ) { + finishHim(false); + return true; + } + return false; + } + + protected void performOperation(final String nodeId, final Set shardIds, final int nodeIndex) { + if (shardIds.size() == 0) { + // no more active shards... (we should not really get here, just safety) + //MH TODO somewhat arbitrarily returining firsy + onNoOperation(nodeId); + } else { + try { + //TODO pass through a reduced timeout (the original time limit, minus whatever we may have + // spent already getting to this point. + final NodeTermEnumRequest nodeRequest = newNodeRequest(nodeId, shardIds, request); + nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId()); + DiscoveryNode node = nodes.get(nodeId); + if (node == null) { + // no node connected, act as failure + onNoOperation(nodeId); + } else if (checkForEarlyFinish() == false) { + transportService.sendRequest( + node, + transportShardAction, + nodeRequest, + new TransportResponseHandler() { + @Override + public NodeTermEnumResponse read(StreamInput in) throws IOException { + return readShardResponse(in); + } + + @Override + public void handleResponse(NodeTermEnumResponse response) { + onOperation(nodeId, nodeIndex, response); + } + + @Override + public void handleException(TransportException e) { + onNoOperation(nodeId); + } + } + ); + } + } catch (Exception e) { + onNoOperation(nodeId); + } + } + } + + @SuppressWarnings({ "unchecked" }) + protected void onOperation(String nodeId, int nodeIndex, NodeTermEnumResponse response) { + logger.trace("received response for node {}", nodeId); + nodesResponses.set(nodeIndex, response); + if (expectedOps == counterOps.incrementAndGet()) { + finishHim(true); + } else { + checkForEarlyFinish(); + } + } + + void onNoOperation(String nodeId) { + if (expectedOps == counterOps.incrementAndGet()) { + finishHim(true); + } + } + + // Can be called multiple times - either for early time-outs or for fully-completed collections. + protected synchronized void finishHim(boolean complete) { + if (listener == null) { + return; + } + try { + listener.onResponse(newResponse(request, nodesResponses, clusterState, complete)); + } catch (Exception e) { + listener.onFailure(e); + } + finally { + listener = null; + } + } + } + + class NodeTransportHandler implements TransportRequestHandler { + + @Override + public void messageReceived(NodeTermEnumRequest request, TransportChannel channel, Task task) throws Exception { + asyncNodeOperation(request, task, ActionListener.wrap(channel::sendResponse, e -> { + try { + channel.sendResponse(e); + } catch (Exception e1) { + logger.warn( + () -> new ParameterizedMessage( + "Failed to send error response for action [{}] and request [{}]", + actionName, + request + ), + e1 + ); + } + })); + } + } + + private void asyncNodeOperation(NodeTermEnumRequest request, Task task, ActionListener listener) { + transportService.getThreadPool() + .executor(shardExecutor) + .execute(ActionRunnable.supply(listener, () -> dataNodeOperation(request, task))); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/package-info.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/package-info.java new file mode 100644 index 0000000000000..ac11a71657094 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/package-info.java @@ -0,0 +1,10 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +/** + * Enumerate a field's terms action. + */ +package org.elasticsearch.xpack.core.termenum.action; \ No newline at end of file diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/rest/RestTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/rest/RestTermEnumAction.java new file mode 100644 index 0000000000000..ae95a4e4ef4d7 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/rest/RestTermEnumAction.java @@ -0,0 +1,53 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum.rest; + +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.action.RestToXContentListener; +import org.elasticsearch.xpack.core.termenum.action.TermEnumAction; +import org.elasticsearch.xpack.core.termenum.action.TermEnumRequest; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; +import static org.elasticsearch.rest.RestRequest.Method.POST; + +public class RestTermEnumAction extends BaseRestHandler { + + @Override + public List routes() { + return List.of( + new Route(GET, "/{index}/_terms"), + new Route(POST, "/{index}/_terms")); + } + + @Override + public String getName() { + return "term_enum_action"; + } + + @Override + public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { + try (XContentParser parser = request.contentOrSourceParamParser()) { + TermEnumRequest termEnumRequest = TermEnumAction.fromXContent(parser, + Strings.splitStringByCommaToArray(request.param("index"))); + return channel -> + client.execute(TermEnumAction.INSTANCE, termEnumRequest, new RestToXContentListener<>(channel)); + } + + + + + } + + +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/MultiShardTermsEnumTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/MultiShardTermsEnumTests.java new file mode 100644 index 0000000000000..9f416b5c29ec6 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/MultiShardTermsEnumTests.java @@ -0,0 +1,126 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum; + +import org.apache.lucene.analysis.MockAnalyzer; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.Field; +import org.apache.lucene.document.StringField; +import org.apache.lucene.index.DirectoryReader; +import org.apache.lucene.index.IndexWriter; +import org.apache.lucene.index.IndexWriterConfig; +import org.apache.lucene.index.MultiTerms; +import org.apache.lucene.index.Terms; +import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.store.ByteBuffersDirectory; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.automaton.Automata; +import org.apache.lucene.util.automaton.Automaton; +import org.apache.lucene.util.automaton.CompiledAutomaton; +import org.apache.lucene.util.automaton.MinimizationOperations; +import org.apache.lucene.util.automaton.Operations; +import org.elasticsearch.common.lucene.search.AutomatonQueries; +import org.elasticsearch.core.internal.io.IOUtils; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.core.termenum.action.MultiShardTermsEnum; +import org.elasticsearch.xpack.core.termenum.action.SimpleTermCountEnum; +import org.elasticsearch.xpack.core.termenum.action.TermCount; + +import java.io.Closeable; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; + +public class MultiShardTermsEnumTests extends ESTestCase { + + public void testRandomIndexFusion() throws Exception { + String fieldName = "foo"; + Map globalTermCounts = new HashMap<>(); + + int numShards = randomIntBetween(2, 15); + + ArrayList closeables = new ArrayList<>(); + ArrayList readers = new ArrayList<>(); + + try { + for (int s = 0; s < numShards; s++) { + Directory directory = new ByteBuffersDirectory(); + IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(new MockAnalyzer(random()))); + + int numDocs = randomIntBetween(10,200); + for (int i = 0; i < numDocs; i++) { + Document document = new Document(); + String term = randomAlphaOfLengthBetween(1,3).toLowerCase(Locale.ROOT); + document.add(new StringField(fieldName, term, Field.Store.YES)); + writer.addDocument(document); + int count = 0; + if (globalTermCounts.containsKey(term)) { + count = globalTermCounts.get(term); + } + count++; + globalTermCounts.put(term, count); + + } + DirectoryReader reader = DirectoryReader.open(writer); + readers.add(reader); + writer.close(); + closeables.add(reader); + closeables.add(directory); + } + + int numSearches = 100; + for (int q = 0; q < numSearches; q++) { + String searchPrefix = randomAlphaOfLengthBetween(0, 3).toLowerCase(Locale.ROOT); + Automaton a = AutomatonQueries.caseInsensitivePrefix(searchPrefix); + a = Operations.concatenate(a, Automata.makeAnyString()); + a = MinimizationOperations.minimize(a, Integer.MAX_VALUE); + CompiledAutomaton automaton = new CompiledAutomaton(a); + + ArrayList termsEnums = new ArrayList<>(); + for (DirectoryReader reader : readers) { + Terms terms = MultiTerms.getTerms(reader, fieldName); + TermsEnum te = automaton.getTermsEnum(terms); + if(randomBoolean()) { + // Simulate fields like constant-keyword which use a SimpleTermCountEnum to present results + // rather than the raw TermsEnum from Lucene. + ArrayList termCounts = new ArrayList<>(); + while(te.next()!=null) { + termCounts.add(new TermCount(te.term().utf8ToString(), te.docFreq())); + } + SimpleTermCountEnum simpleEnum = new SimpleTermCountEnum(termCounts.toArray(new TermCount[0])); + termsEnums.add(simpleEnum); + } else { + termsEnums.add(te); + } + } + MultiShardTermsEnum mte = new MultiShardTermsEnum(termsEnums.toArray(new TermsEnum[0])); + HashMap expecteds = new HashMap<>(); + + for (String term : globalTermCounts.keySet()) { + if(term.startsWith(searchPrefix)) { + expecteds.put(term, globalTermCounts.get(term)); + } + } + + while (mte.next() != null) { + String teString = mte.term().utf8ToString(); + long actual = mte.docFreq(); + assertTrue(expecteds.containsKey(teString)); + long expected = expecteds.get(teString); + expecteds.remove(teString); + assertEquals(mte.term().utf8ToString() + " string count wrong", expected, actual); + } + assertEquals("Expected results not found", 0, expecteds.size()); + + } + } finally { + IOUtils.close(closeables.toArray(new Closeable[0])); + } + } + +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermCountTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermCountTests.java new file mode 100644 index 0000000000000..d1b2247c67ed1 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermCountTests.java @@ -0,0 +1,42 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; +import org.elasticsearch.xpack.core.termenum.action.TermCount; + +import java.io.IOException; + +public class TermCountTests extends AbstractSerializingTestCase { + + static TermCount createRandomQueryExplanation(boolean isValid) { + int docCount = randomInt(100); + String term = randomAlphaOfLength(randomIntBetween(10, 100)); + return new TermCount(term, docCount); + } + + static TermCount createRandomQueryExplanation() { + return createRandomQueryExplanation(randomBoolean()); + } + + @Override + protected TermCount doParseInstance(XContentParser parser) throws IOException { + return TermCount.fromXContent(parser); + } + + @Override + protected TermCount createTestInstance() { + return createRandomQueryExplanation(); + } + + @Override + protected Writeable.Reader instanceReader() { + return TermCount::new; + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermEnumResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermEnumResponseTests.java new file mode 100644 index 0000000000000..b242e6176fa52 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermEnumResponseTests.java @@ -0,0 +1,87 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum; + +import org.elasticsearch.ElasticsearchException; +import org.elasticsearch.action.support.DefaultShardOperationFailedException; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractBroadcastResponseTestCase; +import org.elasticsearch.xpack.core.termenum.action.TermEnumResponse; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +public class TermEnumResponseTests extends AbstractBroadcastResponseTestCase { + + protected static List getRandomTerms() { + int termCount = randomIntBetween(0, 100); + Set uniqueTerms = new HashSet<>(termCount); + while (uniqueTerms.size() < termCount) { + String s = randomAlphaOfLengthBetween(1, 10); + uniqueTerms.add(s); + } + List terms = new ArrayList<>(uniqueTerms); + return terms; + } + + private static TermEnumResponse createRandomTermEnumResponse() { + int totalShards = randomIntBetween(1, 10); + int successfulShards = randomIntBetween(0, totalShards); + int failedShards = totalShards - successfulShards; + List shardFailures = new ArrayList<>(failedShards); + for (int i=0; i failures) { + return new TermEnumResponse(getRandomTerms(), totalShards, successfulShards, failedShards, failures, randomBoolean()); + + } + + @Override + public void testToXContent() { + String s = randomAlphaOfLengthBetween(1, 10); + List terms = new ArrayList<>(); + terms.add(s); + TermEnumResponse response = new TermEnumResponse(terms, 10, 10, 0, new ArrayList<>(), true); + + String output = Strings.toString(response); + assertEquals("{\"_shards\":{\"total\":10,\"successful\":10,\"failed\":0},\"terms\":[" + + "\""+ s +"\""+ + "],\"complete\":true}", output); + } +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TransportTermEnumActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TransportTermEnumActionTests.java new file mode 100644 index 0000000000000..9db582bd6688f --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TransportTermEnumActionTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.test.ESSingleNodeTestCase; +import org.elasticsearch.xpack.core.termenum.action.TermEnumAction; +import org.elasticsearch.xpack.core.termenum.action.TermEnumRequest; +import org.elasticsearch.xpack.core.termenum.action.TermEnumResponse; + +import java.util.concurrent.atomic.AtomicBoolean; + +import static org.hamcrest.Matchers.equalTo; + +public class TransportTermEnumActionTests extends ESSingleNodeTestCase { + + /* + * Copy of test that tripped up similarly broadcast ValidateQuery + */ + public void testListenerOnlyInvokedOnceWhenIndexDoesNotExist() { + final AtomicBoolean invoked = new AtomicBoolean(); + final ActionListener listener = new ActionListener<>() { + + @Override + public void onResponse(final TermEnumResponse validateQueryResponse) { + fail("onResponse should not be invoked in this failure case"); + } + + @Override + public void onFailure(final Exception e) { + if (invoked.compareAndSet(false, true) == false) { + fail("onFailure invoked more than once"); + } + } + + }; + client().execute(TermEnumAction.INSTANCE, new TermEnumRequest("non-existent-index"),listener); + assertThat(invoked.get(), equalTo(true)); // ensure that onFailure was invoked + } + +} diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/action/RestTermEnumActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/action/RestTermEnumActionTests.java new file mode 100644 index 0000000000000..4ee63445a4838 --- /dev/null +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/action/RestTermEnumActionTests.java @@ -0,0 +1,148 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termenum.action; + +import org.elasticsearch.action.ActionListener; +import org.elasticsearch.action.ActionRequest; +import org.elasticsearch.action.ActionType; +import org.elasticsearch.action.support.ActionFilters; +import org.elasticsearch.action.support.TransportAction; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.bytes.BytesArray; +import org.elasticsearch.common.io.stream.NamedWriteableRegistry; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.XContentType; +import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +//import org.elasticsearch.rest.CompatibleVersion; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.tasks.Task; +import org.elasticsearch.tasks.TaskManager; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestChannel; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.elasticsearch.threadpool.TestThreadPool; +import org.elasticsearch.threadpool.ThreadPool; +import org.elasticsearch.transport.Transport; +import org.elasticsearch.usage.UsageService; +import org.elasticsearch.xpack.core.termenum.rest.RestTermEnumAction; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static java.util.Collections.emptyMap; +import static java.util.Collections.emptySet; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.mockito.Mockito.mock; + +public class RestTermEnumActionTests extends ESTestCase { + + private static ThreadPool threadPool = new TestThreadPool(RestTermEnumActionTests.class.getName()); + private static NodeClient client = new NodeClient(Settings.EMPTY, threadPool); + + private static UsageService usageService = new UsageService(); + private static RestController controller = new RestController(emptySet(), null, client, + new NoneCircuitBreakerService(), usageService); + private static RestTermEnumAction action = new RestTermEnumAction(); + + /** + * Configures {@link NodeClient} to stub {@link TermEnumAction} transport action. + *

+ * This lower level of execution is out of the scope of this test. + */ + @BeforeClass + public static void stubTermEnumAction() { + final TaskManager taskManager = new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()); + + final TransportAction transportAction = new TransportAction(TermEnumAction.NAME, + new ActionFilters(Collections.emptySet()), taskManager) { + @Override + protected void doExecute(Task task, ActionRequest request, ActionListener listener) { + } + }; + + final Map actions = new HashMap<>(); + actions.put(TermEnumAction.INSTANCE, transportAction); + + client.initialize(actions, taskManager, () -> "local", + mock(Transport.Connection.class), null, new NamedWriteableRegistry(List.of())); + controller.registerHandler(action); + } + + @Override + protected NamedXContentRegistry xContentRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, Collections.emptyList()); + return new NamedXContentRegistry(searchModule.getNamedXContents()); + } + + @AfterClass + public static void terminateThreadPool() { + terminate(threadPool); + + threadPool = null; + client = null; + + usageService = null; + controller = null; + action = null; + } + + public void testRestTermEnumAction() throws Exception { + // GIVEN a valid query + final String content = "{" + + "\"field\":\"a\", " + + "\"string\":\"foo\", " + + "\"index_filter\":{\"bool\":{\"must\":{\"term\":{\"user\":\"kimchy\"}}}}}"; + + final RestRequest request = createRestRequest(content); + final FakeRestChannel channel = new FakeRestChannel(request, true, 0); + + // WHEN + action.handleRequest(request, channel, client); + + // THEN request is parsed OK + assertThat(channel.responses().get(), equalTo(0)); + assertThat(channel.errors().get(), equalTo(0)); + assertNull(channel.capturedResponse()); + } + + public void testRestTermEnumActionMissingField() throws Exception { + // GIVEN an invalid query + final String content = "{" +// + "\"field\":\"a\", " + + "\"string\":\"foo\", " + + "\"index_filter\":{\"bool\":{\"must\":{\"term\":{\"user\":\"kimchy\"}}}}}"; + + final RestRequest request = createRestRequest(content); + final FakeRestChannel channel = new FakeRestChannel(request, true, 0); + + // WHEN + action.handleRequest(request, channel, client); + + // THEN request is invalid - missing mandatory "field" parameter. + assertThat(channel.responses().get(), equalTo(0)); + assertThat(channel.errors().get(), equalTo(1)); + assertThat(channel.capturedResponse().content().utf8ToString(), containsString("field cannot be null")); + } + + + private RestRequest createRestRequest(String content) { + return new FakeRestRequest.Builder(xContentRegistry()) + .withPath("index1/_terms") + .withParams(emptyMap()) + .withContent(new BytesArray(content), XContentType.JSON) + .build(); + } + +} diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 973ab6d2c0744..5e16917876335 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.constantkeyword.mapper; +import org.apache.lucene.index.TermsEnum; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.MatchNoDocsQuery; import org.apache.lucene.search.MultiTermQuery; @@ -38,11 +39,14 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; +import org.elasticsearch.xpack.core.termenum.action.SimpleTermCountEnum; +import org.elasticsearch.xpack.core.termenum.action.TermCount; import java.io.IOException; import java.time.ZoneId; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.function.Supplier; @@ -140,6 +144,20 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) ? lookup -> List.of() : lookup -> List.of(value); } + + + + @Override + public TermsEnum getTerms(boolean caseInsensitive, String string, SearchExecutionContext queryShardContext) throws IOException { + boolean matches = caseInsensitive ? + value.toLowerCase(Locale.ROOT).startsWith(string.toLowerCase(Locale.ROOT)) : + value.startsWith(string); + if (matches == false) { + return null; + } + int docCount = queryShardContext.searcher().getIndexReader().numDocs(); + return new SimpleTermCountEnum(new TermCount(value, docCount)); + } @Override protected boolean matches(String pattern, boolean caseInsensitive, SearchExecutionContext context) { diff --git a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java index 86cfc55bd4558..6eabe470b1931 100644 --- a/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java +++ b/x-pack/plugin/security/qa/operator-privileges-tests/src/javaRestTest/java/org/elasticsearch/xpack/security/operator/Constants.java @@ -416,6 +416,7 @@ public class Constants { "indices:data/read/xpack/graph/explore", "indices:data/read/xpack/rollup/get/index/caps", "indices:data/read/xpack/rollup/search", + "indices:data/read/xpack/termsenum/list", "indices:data/write/bulk", "indices:data/write/bulk[s]", "indices:data/write/bulk_shard_operations[s]", diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/termsenum.json b/x-pack/plugin/src/test/resources/rest-api-spec/api/termsenum.json new file mode 100644 index 0000000000000..aaf0624fa5719 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/api/termsenum.json @@ -0,0 +1,35 @@ +{ + "termsenum":{ + "documentation":{ + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/term-enum.html", + "description": "The termenum API can be used to discover terms in the index that begin with the provided string. It is designed for low-latency look-ups used in auto-complete scenarios." + }, + "stability":"beta", + "visibility":"public", + "headers":{ + "accept": [ "application/json"], + "content_type": ["application/json"] + }, + "url":{ + "paths":[ + { + "path": "/{index}/_terms", + "methods": [ + "GET", + "POST" + ], + "parts": { + "index": { + "type": "list", + "description": "A comma-separated list of index names to search; use `_all` or empty string to perform the operation on all indices" + } + } + } + ] + }, + "params":{}, + "body":{ + "description":"field name, string which is the prefix expected in matching terms, timeout in milliseconds and size for max number of results" + } + } +} diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/term_enum/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/term_enum/10_basic.yml new file mode 100644 index 0000000000000..9326b0f144690 --- /dev/null +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/term_enum/10_basic.yml @@ -0,0 +1,82 @@ +--- +setup: +- do: + indices.create: + index: test_k + body: + settings: + index: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + foo: + type : keyword +- do: + indices.create: + index: test_ck + body: + settings: + index: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + foo: + type : constant_keyword + value: bar_ck + other: + type : text +- do: + indices.create: + index: test_f + body: + settings: + index: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + foo: + type : flattened + +--- +"Test basic term enumeration": + - do: + index: + index: test_k + id: 1 + body: { foo: "bar_k" } + + - do: + index: + index: test_ck + id: 2 + body: { other: "foo" } + + - do: + index: + index: test_f + id: 3 + body: { foo: { bar: "bar_f" } } + + - do: + indices.refresh: {} + + - do: + cluster.health: + index: test_f + wait_for_status: green + + - do: + termsenum: + index: test_* + body: {"field": "foo", "string":"b"} + - length: {terms: 2} + + + - do: + termsenum: + index: test_* + body: {"field": "foo.bar", "string":"b"} + - length: {terms: 1} From b95cf815b49ce78d42b51440ad983c025bb9c254 Mon Sep 17 00:00:00 2001 From: markharwood Date: Wed, 17 Feb 2021 10:38:33 +0000 Subject: [PATCH 02/31] Added HLRC support and related integration test --- .../client/RequestConverters.java | 10 +++ .../client/RestHighLevelClient.java | 31 +++++++ .../org/elasticsearch/client/TermEnumIT.java | 82 +++++++++++++++++++ .../core/termenum/action/TermEnumRequest.java | 10 +-- 4 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/TermEnumIT.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index 9807f5483c5f0..f4c4b44e03ca2 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -72,6 +72,7 @@ import org.elasticsearch.script.mustache.MultiSearchTemplateRequest; import org.elasticsearch.script.mustache.SearchTemplateRequest; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; +import org.elasticsearch.xpack.core.termenum.action.TermEnumRequest; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -536,6 +537,15 @@ static Request rankEval(RankEvalRequest rankEvalRequest) throws IOException { request.setEntity(createEntity(rankEvalRequest.getRankEvalSpec(), REQUEST_BODY_CONTENT_TYPE)); return request; } + + static Request termEnum(TermEnumRequest termEnumRequest) throws IOException { + Request request = new Request(HttpGet.METHOD_NAME, endpoint(termEnumRequest.indices(), Strings.EMPTY_ARRAY, "_terms")); + + Params params = new Params(); + params.withIndicesOptions(termEnumRequest.indicesOptions()); + request.setEntity(createEntity(termEnumRequest, REQUEST_BODY_CONTENT_TYPE)); + return request; + } static Request reindex(ReindexRequest reindexRequest) throws IOException { return prepareReindexRequest(reindexRequest, true); diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 455343d99627a..f92c18b6e592f 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -189,6 +189,8 @@ import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; import org.elasticsearch.search.suggest.term.TermSuggestion; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; +import org.elasticsearch.xpack.core.termenum.action.TermEnumRequest; +import org.elasticsearch.xpack.core.termenum.action.TermEnumResponse; import java.io.Closeable; import java.io.IOException; @@ -1423,6 +1425,19 @@ public final RankEvalResponse rankEval(RankEvalRequest rankEvalRequest, RequestO return performRequestAndParseEntity(rankEvalRequest, RequestConverters::rankEval, options, RankEvalResponse::fromXContent, emptySet()); } + + /** + * Executes a request using the TermEnum API. + * See Term enum API + * on elastic.co + * @param termEnumRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @return the response + */ + public final TermEnumResponse terms(TermEnumRequest termEnumRequest, RequestOptions options) throws IOException { + return performRequestAndParseEntity(termEnumRequest, RequestConverters::termEnum, options, TermEnumResponse::fromXContent, + emptySet()); + } /** @@ -1466,6 +1481,22 @@ public final Cancellable rankEvalAsync(RankEvalRequest rankEvalRequest, RequestO RankEvalResponse::fromXContent, listener, emptySet()); } + + /** + * Asynchronously executes a request using the TermEnum API. + * See Term enum API + * on elastic.co + * @param termEnumRequest the request + * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized + * @param listener the listener to be notified upon request completion + * @return cancellable that may be used to cancel the request + */ + public final Cancellable termEnumAsync(TermEnumRequest termEnumRequest, RequestOptions options, + ActionListener listener) { + return performRequestAsyncAndParseEntity(termEnumRequest, RequestConverters::termEnum, options, + RankEvalResponse::fromXContent, listener, + emptySet()); + } /** * Executes a request using the Field Capabilities API. diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/TermEnumIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/TermEnumIT.java new file mode 100644 index 0000000000000..acf08745fe131 --- /dev/null +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/TermEnumIT.java @@ -0,0 +1,82 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ +package org.elasticsearch.client; + +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.elasticsearch.xpack.core.termenum.action.TermEnumRequest; +import org.elasticsearch.xpack.core.termenum.action.TermEnumResponse; +import org.junit.Before; + +import java.io.IOException; +import java.util.List; + +public class TermEnumIT extends ESRestHighLevelClientTestCase { + + @Before + public void indexDocuments() throws IOException { + // Create chain of doc IDs across indices 1->2->3 + Request doc1 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/1"); + doc1.setJsonEntity("{ \"const\":\"aaa\"}"); + client().performRequest(doc1); + + Request doc2 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/2"); + doc2.setJsonEntity("{ \"const\":\"aba\"}"); + client().performRequest(doc2); + + Request doc3 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/3"); + doc3.setJsonEntity("{ \"const\":\"aba\"}"); + client().performRequest(doc3); + + Request doc4 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/4"); + doc4.setJsonEntity("{ \"const\":\"abc\"}"); + client().performRequest(doc4); + + Request doc5 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/5"); + doc5.setJsonEntity("{ \"const\":\"def\"}"); + client().performRequest(doc5); + + + client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh")); + } + + public void testSuggests() throws Exception { + checkResults("index*", "a", "aaa", "aba", "abc"); + checkResults("index1", "a", "aaa"); + checkResults("index2", "a", "aba", "abc"); + checkResults("index*", "ab", "aba", "abc"); + checkResults("index*", "d", "def"); + checkResults("index*", "z"); + checkResults("index*", "Ab", true, "aba", "abc"); + } + + public void testCaseInsensitive() throws Exception { + checkResults("index*", "Ab", true, "aba", "abc"); + } + + public void checkResults(String indices, String prefix, String... expecteds) throws IOException { + checkResults(indices, prefix, false, expecteds); + } + + public void checkResults(String indices, String prefix, boolean caseInsensitive, String... expecteds) throws IOException { + TermEnumRequest ter = new TermEnumRequest(indices); + ter.field("const.keyword"); + ter.string(prefix); + ter.caseInsensitive(caseInsensitive); + TermEnumResponse result = highLevelClient().terms(ter, RequestOptions.DEFAULT); + if (expecteds == null) { + assertEquals(0, result.getTerms().size()); + return; + } + List terms = result.getTerms(); + assertEquals(expecteds.length, terms.size()); + for (int i = 0; i < expecteds.length; i++) { + assertEquals(expecteds[i], terms.get(i)); + } + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java index 14318ded1d47a..f3ec25c7a11dd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java @@ -22,7 +22,7 @@ import java.util.Arrays; /** - * A request to gather terms for a given field matching a pattern + * A request to gather terms for a given field matching a string prefix */ public class TermEnumRequest extends BroadcastRequest implements ToXContentObject { @@ -87,8 +87,8 @@ public String field() { /** * The string required in matching field values */ - public void string(String pattern) { - this.string = pattern; + public void string(String string) { + this.string = string; } /** @@ -171,7 +171,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public String toString() { - return "[" + Arrays.toString(indices) + "] field[" + field + "], pattern[" + string + "] " + " size=" + size + " timeout=" + return "[" + Arrays.toString(indices) + "] field[" + field + "], string[" + string + "] " + " size=" + size + " timeout=" + timeout().getMillis() + " sort_by_popularity = " + sortByPopularity + " case_insensitive=" + caseInsensitive + " indexFilter = "+ indexFilter; } @@ -180,7 +180,7 @@ public String toString() { public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { builder.startObject(); builder.field("field", field); - builder.field("pattern", string); + builder.field("string", string); builder.field("size", size); builder.field("timeout", timeout().getMillis()); builder.field("case_insensitive", caseInsensitive); From ab826b43205d0bfd257353fb46e234dc43103b3d Mon Sep 17 00:00:00 2001 From: markharwood Date: Wed, 17 Feb 2021 17:31:11 +0000 Subject: [PATCH 03/31] Added client classes for HLRC. Added yaml test with index filter --- .../client/RequestConverters.java | 2 +- .../client/RestHighLevelClient.java | 14 +- .../client/termenum/TermEnumRequest.java | 183 ++++++++++++++++++ .../client/termenum/TermEnumResponse.java | 112 +++++++++++ .../org/elasticsearch/client/TermEnumIT.java | 4 +- .../action/TransportTermEnumAction.java | 3 +- .../rest-api-spec/test/term_enum/10_basic.yml | 18 +- 7 files changed, 322 insertions(+), 14 deletions(-) create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumRequest.java create mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumResponse.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index f4c4b44e03ca2..aa5de2be97fc4 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -44,6 +44,7 @@ import org.elasticsearch.client.indices.AnalyzeRequest; import org.elasticsearch.client.security.RefreshPolicy; import org.elasticsearch.client.tasks.TaskId; +import org.elasticsearch.client.termenum.TermEnumRequest; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Priority; @@ -72,7 +73,6 @@ import org.elasticsearch.script.mustache.MultiSearchTemplateRequest; import org.elasticsearch.script.mustache.SearchTemplateRequest; import org.elasticsearch.search.fetch.subphase.FetchSourceContext; -import org.elasticsearch.xpack.core.termenum.action.TermEnumRequest; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index f92c18b6e592f..8e44ef5fe9eb2 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -60,6 +60,8 @@ import org.elasticsearch.client.core.TermVectorsRequest; import org.elasticsearch.client.core.TermVectorsResponse; import org.elasticsearch.client.tasks.TaskSubmissionResponse; +import org.elasticsearch.client.termenum.TermEnumRequest; +import org.elasticsearch.client.termenum.TermEnumResponse; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParseField; @@ -120,18 +122,18 @@ import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler; import org.elasticsearch.search.aggregations.bucket.sampler.ParsedSampler; +import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms; import org.elasticsearch.search.aggregations.bucket.terms.LongRareTerms; +import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms; import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongRareTerms; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms; import org.elasticsearch.search.aggregations.bucket.terms.ParsedSignificantLongTerms; import org.elasticsearch.search.aggregations.bucket.terms.ParsedSignificantStringTerms; import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringRareTerms; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; import org.elasticsearch.search.aggregations.bucket.terms.SignificantLongTerms; import org.elasticsearch.search.aggregations.bucket.terms.SignificantStringTerms; -import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms; -import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; import org.elasticsearch.search.aggregations.bucket.terms.StringRareTerms; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; @@ -189,8 +191,6 @@ import org.elasticsearch.search.suggest.phrase.PhraseSuggestionBuilder; import org.elasticsearch.search.suggest.term.TermSuggestion; import org.elasticsearch.search.suggest.term.TermSuggestionBuilder; -import org.elasticsearch.xpack.core.termenum.action.TermEnumRequest; -import org.elasticsearch.xpack.core.termenum.action.TermEnumResponse; import java.io.Closeable; import java.io.IOException; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumRequest.java new file mode 100644 index 0000000000000..205e0f021377a --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumRequest.java @@ -0,0 +1,183 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.client.termenum; + +import org.elasticsearch.action.ActionRequestValidationException; +import org.elasticsearch.action.ValidateActions; +import org.elasticsearch.action.support.IndicesOptions; +import org.elasticsearch.action.support.broadcast.BroadcastRequest; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.unit.TimeValue; +import org.elasticsearch.common.xcontent.ToXContentObject; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.index.query.QueryBuilder; + +import java.io.IOException; +import java.util.Arrays; + +/** + * A request to gather terms for a given field matching a string prefix + */ +public class TermEnumRequest extends BroadcastRequest implements ToXContentObject { + + public static int DEFAULT_SIZE = 10; + public static int DEFAULT_TIMEOUT_MILLIS = 1000; + + private String field; + private String string; + private int size = DEFAULT_SIZE; + private boolean caseInsensitive; + private boolean sortByPopularity; + long taskStartTimeMillis; + private QueryBuilder indexFilter; + + public TermEnumRequest() { + this(Strings.EMPTY_ARRAY); + } + + /** + * Constructs a new term enum request against the provided indices. No indices provided means it will + * run against all indices. + */ + public TermEnumRequest(String... indices) { + super(indices); + indicesOptions(IndicesOptions.fromOptions(false, false, true, false)); + timeout(new TimeValue(DEFAULT_TIMEOUT_MILLIS)); + } + + @Override + public ActionRequestValidationException validate() { + ActionRequestValidationException validationException = super.validate(); + if (field == null) { + validationException = ValidateActions.addValidationError("field cannot be null", validationException); + } + return validationException; + } + + /** + * The field to look inside for values + */ + public void field(String field) { + this.field = field; + } + + /** + * Indicates if detailed information about query is requested + */ + public String field() { + return field; + } + + /** + * The string required in matching field values + */ + public void string(String string) { + this.string = string; + } + + /** + * The string required in matching field values + */ + public String string() { + return string; + } + + /** + * sort terms by popularity + */ + public boolean sortByPopularity() { + return sortByPopularity; + } + + /** + * sort terms by popularity + */ + public void sortByPopularity(boolean sortByPopularity) { + this.sortByPopularity = sortByPopularity; + } + + /** + * The number of terms to return + */ + public int size() { + return size; + } + + /** + * The number of terms to return + */ + public void size(int size) { + this.size = size; + } + + /** + * TThe max time in milliseconds to spend gathering terms + */ + public void timeoutInMillis(int timeout) { + timeout(new TimeValue(timeout)); + } + + /** + * If case insensitive matching is required + */ + public void caseInsensitive(boolean caseInsensitive) { + this.caseInsensitive = caseInsensitive; + } + + /** + * If case insensitive matching is required + */ + public boolean caseInsensitive() { + return caseInsensitive; + } + + /** + * Allows to filter shards if the provided {@link QueryBuilder} rewrites to `match_none`. + */ + public void indexFilter(QueryBuilder indexFilter) { + this.indexFilter = indexFilter; + } + + public QueryBuilder indexFilter() { + return indexFilter; + } + + @Override + public String toString() { + return "[" + Arrays.toString(indices) + "] field[" + field + "], string[" + string + "] " + " size=" + size + " timeout=" + + timeout().getMillis() + " sort_by_popularity = " + sortByPopularity + " case_insensitive=" + + caseInsensitive + " indexFilter = "+ indexFilter; + } + + @Override + public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { + builder.startObject(); + builder.field("field", field); + builder.field("string", string); + builder.field("size", size); + builder.field("timeout", timeout().getMillis()); + builder.field("case_insensitive", caseInsensitive); + builder.field("sort_by_popularity", sortByPopularity); + if (indexFilter != null) { + builder.field("index_filter", indexFilter); + } + return builder.endObject(); + } +} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumResponse.java new file mode 100644 index 0000000000000..05d97fe740578 --- /dev/null +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumResponse.java @@ -0,0 +1,112 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.elasticsearch.client.termenum; + +import org.elasticsearch.action.support.DefaultShardOperationFailedException; +import org.elasticsearch.action.support.broadcast.BroadcastResponse; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ConstructingObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; + +/** + * The response of the termenum/list action. + */ +public class TermEnumResponse extends BroadcastResponse { + + public static final String TERMS_FIELD = "terms"; + public static final String COMPLETE_FIELD = "complete"; + + @SuppressWarnings("unchecked") + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + "term_enum_results", + true, + arg -> { + BroadcastResponse response = (BroadcastResponse) arg[0]; + return new TermEnumResponse( + (List) arg[1], + response.getTotalShards(), + response.getSuccessfulShards(), + response.getFailedShards(), + Arrays.asList(response.getShardFailures()), + (Boolean) arg[2] + ); + } + ); + static { + declareBroadcastFields(PARSER); + PARSER.declareStringArray(optionalConstructorArg(), new ParseField(TERMS_FIELD)); + PARSER.declareBoolean(optionalConstructorArg(), new ParseField(COMPLETE_FIELD)); + } + + private final List terms; + + private boolean complete; + private int skippedShards; + + public TermEnumResponse( + List terms, + int totalShards, + int successfulShards, + int failedShards, + List shardFailures, boolean complete + ) { + super(totalShards, successfulShards, failedShards, shardFailures); + this.terms = terms == null ? Collections.emptyList() : terms; + this.complete = complete; + } + + /** + * The list of terms. + */ + public List getTerms() { + return terms; + } + + /** + * The number of shards skipped by the index filter + */ + public int getSkippedShards() { + return skippedShards; + } + + @Override + protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException { + if (getTerms() != null && getTerms().isEmpty() == false) { + builder.startArray(TERMS_FIELD); + for (String term : getTerms()) { + builder.value(term); + } + builder.endArray(); + } + builder.field(COMPLETE_FIELD, complete); + } + + public static TermEnumResponse fromXContent(XContentParser parser) { + return PARSER.apply(parser, null); + } + +} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/TermEnumIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/TermEnumIT.java index acf08745fe131..7fcc277ea3321 100644 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/TermEnumIT.java +++ b/client/rest-high-level/src/test/java/org/elasticsearch/client/TermEnumIT.java @@ -9,8 +9,8 @@ import org.apache.http.client.methods.HttpPost; import org.apache.http.client.methods.HttpPut; -import org.elasticsearch.xpack.core.termenum.action.TermEnumRequest; -import org.elasticsearch.xpack.core.termenum.action.TermEnumResponse; +import org.elasticsearch.client.termenum.TermEnumRequest; +import org.elasticsearch.client.termenum.TermEnumResponse; import org.junit.Before; import java.io.IOException; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java index 06108561628d9..1d863c3f93fc6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java @@ -388,7 +388,8 @@ private boolean canAccess(String indexName, String fieldName, XPackLicenseState threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY); IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(indexName); - //TODO remove FLS check (the reader does that anyway) + //TODO There was a suggestion we could remove FLS check because the reader does that filtering anyway - didn't + // turn out that way when I tested it. // TODO remove DLS check by getting a new security feature to filter at coordinating node the choice of indices // based on if ANY DLS is present on an index. if (indexAccessControl != null) { diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/term_enum/10_basic.yml b/x-pack/plugin/src/test/resources/rest-api-spec/test/term_enum/10_basic.yml index 9326b0f144690..bbee32d01de57 100644 --- a/x-pack/plugin/src/test/resources/rest-api-spec/test/term_enum/10_basic.yml +++ b/x-pack/plugin/src/test/resources/rest-api-spec/test/term_enum/10_basic.yml @@ -12,6 +12,8 @@ setup: properties: foo: type : keyword + timestamp: + type : date - do: indices.create: index: test_ck @@ -27,6 +29,8 @@ setup: value: bar_ck other: type : text + timestamp: + type : date - do: indices.create: index: test_f @@ -39,6 +43,8 @@ setup: properties: foo: type : flattened + timestamp: + type : date --- "Test basic term enumeration": @@ -46,19 +52,19 @@ setup: index: index: test_k id: 1 - body: { foo: "bar_k" } + body: { foo: "bar_k", "timestamp":"2021-01-01T01:01:01.000Z" } - do: index: index: test_ck id: 2 - body: { other: "foo" } + body: { other: "foo", "timestamp":"2020-01-01T01:01:01.000Z" } - do: index: index: test_f id: 3 - body: { foo: { bar: "bar_f" } } + body: { foo: { bar: "bar_f" }, "timestamp":"2019-01-01T01:01:01.000Z" } - do: indices.refresh: {} @@ -80,3 +86,9 @@ setup: index: test_* body: {"field": "foo.bar", "string":"b"} - length: {terms: 1} + + - do: + termsenum: + index: test_* + body: {"field": "foo", "string":"b", "index_filter":{"range":{"timestamp":{"gte":"2021-01-01T01:01:01.000Z"}}}} + - length: {terms: 1} From 7a5e654f83912d0a27e87c0dca0b2ea89ede51d2 Mon Sep 17 00:00:00 2001 From: markharwood Date: Wed, 17 Feb 2021 18:35:16 +0000 Subject: [PATCH 04/31] License fix --- .../client/termenum/TermEnumRequest.java | 21 +++++-------------- .../client/termenum/TermEnumResponse.java | 21 +++++-------------- 2 files changed, 10 insertions(+), 32 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumRequest.java index 205e0f021377a..65c69f9d844c1 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumRequest.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumRequest.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.client.termenum; diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumResponse.java index 05d97fe740578..b1fd669218c25 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumResponse.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumResponse.java @@ -1,20 +1,9 @@ /* - * Licensed to Elasticsearch under one or more contributor - * license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright - * ownership. Elasticsearch licenses this file to you under - * the Apache License, Version 2.0 (the "License"); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. */ package org.elasticsearch.client.termenum; From 93dfe30fca29bffc372feda1bc44d3e317e6e973 Mon Sep 17 00:00:00 2001 From: markharwood Date: Thu, 18 Feb 2021 10:50:42 +0000 Subject: [PATCH 05/31] =?UTF-8?q?Remove=20HLRC=20code=20for=20now=20-=20re?= =?UTF-8?q?quires=20less-than-ideal=20package=20names=20while=20we?= =?UTF-8?q?=E2=80=99re=20unable=20to=20move=20main=20server=20implementati?= =?UTF-8?q?on=20out=20of=20xpack=20(due=20to=20security=20dependency).=20D?= =?UTF-8?q?on=E2=80=99t=20want=20clients=20to=20build=20dependencies=20on?= =?UTF-8?q?=20the=20wrong=20package=20names=20and=20the=20Java=20client=20?= =?UTF-8?q?changes=20are=20underway=20anyway.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/RequestConverters.java | 10 - .../client/RestHighLevelClient.java | 32 ---- .../client/termenum/TermEnumRequest.java | 172 ------------------ .../client/termenum/TermEnumResponse.java | 101 ---------- .../org/elasticsearch/client/TermEnumIT.java | 82 --------- 5 files changed, 397 deletions(-) delete mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumRequest.java delete mode 100644 client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumResponse.java delete mode 100644 client/rest-high-level/src/test/java/org/elasticsearch/client/TermEnumIT.java diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index aa5de2be97fc4..cfa721b49973c 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -44,7 +44,6 @@ import org.elasticsearch.client.indices.AnalyzeRequest; import org.elasticsearch.client.security.RefreshPolicy; import org.elasticsearch.client.tasks.TaskId; -import org.elasticsearch.client.termenum.TermEnumRequest; import org.elasticsearch.cluster.health.ClusterHealthStatus; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.Priority; @@ -538,15 +537,6 @@ static Request rankEval(RankEvalRequest rankEvalRequest) throws IOException { return request; } - static Request termEnum(TermEnumRequest termEnumRequest) throws IOException { - Request request = new Request(HttpGet.METHOD_NAME, endpoint(termEnumRequest.indices(), Strings.EMPTY_ARRAY, "_terms")); - - Params params = new Params(); - params.withIndicesOptions(termEnumRequest.indicesOptions()); - request.setEntity(createEntity(termEnumRequest, REQUEST_BODY_CONTENT_TYPE)); - return request; - } - static Request reindex(ReindexRequest reindexRequest) throws IOException { return prepareReindexRequest(reindexRequest, true); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 8e44ef5fe9eb2..5747490a5d24b 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -60,8 +60,6 @@ import org.elasticsearch.client.core.TermVectorsRequest; import org.elasticsearch.client.core.TermVectorsResponse; import org.elasticsearch.client.tasks.TaskSubmissionResponse; -import org.elasticsearch.client.termenum.TermEnumRequest; -import org.elasticsearch.client.termenum.TermEnumResponse; import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.ParseField; @@ -1426,20 +1424,6 @@ public final RankEvalResponse rankEval(RankEvalRequest rankEvalRequest, RequestO emptySet()); } - /** - * Executes a request using the TermEnum API. - * See Term enum API - * on elastic.co - * @param termEnumRequest the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @return the response - */ - public final TermEnumResponse terms(TermEnumRequest termEnumRequest, RequestOptions options) throws IOException { - return performRequestAndParseEntity(termEnumRequest, RequestConverters::termEnum, options, TermEnumResponse::fromXContent, - emptySet()); - } - - /** * Executes a request using the Multi Search Template API. * @@ -1482,22 +1466,6 @@ public final Cancellable rankEvalAsync(RankEvalRequest rankEvalRequest, RequestO emptySet()); } - /** - * Asynchronously executes a request using the TermEnum API. - * See Term enum API - * on elastic.co - * @param termEnumRequest the request - * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized - * @param listener the listener to be notified upon request completion - * @return cancellable that may be used to cancel the request - */ - public final Cancellable termEnumAsync(TermEnumRequest termEnumRequest, RequestOptions options, - ActionListener listener) { - return performRequestAsyncAndParseEntity(termEnumRequest, RequestConverters::termEnum, options, - RankEvalResponse::fromXContent, listener, - emptySet()); - } - /** * Executes a request using the Field Capabilities API. * See Field Capabilities API diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumRequest.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumRequest.java deleted file mode 100644 index 65c69f9d844c1..0000000000000 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumRequest.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ - -package org.elasticsearch.client.termenum; - -import org.elasticsearch.action.ActionRequestValidationException; -import org.elasticsearch.action.ValidateActions; -import org.elasticsearch.action.support.IndicesOptions; -import org.elasticsearch.action.support.broadcast.BroadcastRequest; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.unit.TimeValue; -import org.elasticsearch.common.xcontent.ToXContentObject; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.index.query.QueryBuilder; - -import java.io.IOException; -import java.util.Arrays; - -/** - * A request to gather terms for a given field matching a string prefix - */ -public class TermEnumRequest extends BroadcastRequest implements ToXContentObject { - - public static int DEFAULT_SIZE = 10; - public static int DEFAULT_TIMEOUT_MILLIS = 1000; - - private String field; - private String string; - private int size = DEFAULT_SIZE; - private boolean caseInsensitive; - private boolean sortByPopularity; - long taskStartTimeMillis; - private QueryBuilder indexFilter; - - public TermEnumRequest() { - this(Strings.EMPTY_ARRAY); - } - - /** - * Constructs a new term enum request against the provided indices. No indices provided means it will - * run against all indices. - */ - public TermEnumRequest(String... indices) { - super(indices); - indicesOptions(IndicesOptions.fromOptions(false, false, true, false)); - timeout(new TimeValue(DEFAULT_TIMEOUT_MILLIS)); - } - - @Override - public ActionRequestValidationException validate() { - ActionRequestValidationException validationException = super.validate(); - if (field == null) { - validationException = ValidateActions.addValidationError("field cannot be null", validationException); - } - return validationException; - } - - /** - * The field to look inside for values - */ - public void field(String field) { - this.field = field; - } - - /** - * Indicates if detailed information about query is requested - */ - public String field() { - return field; - } - - /** - * The string required in matching field values - */ - public void string(String string) { - this.string = string; - } - - /** - * The string required in matching field values - */ - public String string() { - return string; - } - - /** - * sort terms by popularity - */ - public boolean sortByPopularity() { - return sortByPopularity; - } - - /** - * sort terms by popularity - */ - public void sortByPopularity(boolean sortByPopularity) { - this.sortByPopularity = sortByPopularity; - } - - /** - * The number of terms to return - */ - public int size() { - return size; - } - - /** - * The number of terms to return - */ - public void size(int size) { - this.size = size; - } - - /** - * TThe max time in milliseconds to spend gathering terms - */ - public void timeoutInMillis(int timeout) { - timeout(new TimeValue(timeout)); - } - - /** - * If case insensitive matching is required - */ - public void caseInsensitive(boolean caseInsensitive) { - this.caseInsensitive = caseInsensitive; - } - - /** - * If case insensitive matching is required - */ - public boolean caseInsensitive() { - return caseInsensitive; - } - - /** - * Allows to filter shards if the provided {@link QueryBuilder} rewrites to `match_none`. - */ - public void indexFilter(QueryBuilder indexFilter) { - this.indexFilter = indexFilter; - } - - public QueryBuilder indexFilter() { - return indexFilter; - } - - @Override - public String toString() { - return "[" + Arrays.toString(indices) + "] field[" + field + "], string[" + string + "] " + " size=" + size + " timeout=" - + timeout().getMillis() + " sort_by_popularity = " + sortByPopularity + " case_insensitive=" - + caseInsensitive + " indexFilter = "+ indexFilter; - } - - @Override - public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException { - builder.startObject(); - builder.field("field", field); - builder.field("string", string); - builder.field("size", size); - builder.field("timeout", timeout().getMillis()); - builder.field("case_insensitive", caseInsensitive); - builder.field("sort_by_popularity", sortByPopularity); - if (indexFilter != null) { - builder.field("index_filter", indexFilter); - } - return builder.endObject(); - } -} diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumResponse.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumResponse.java deleted file mode 100644 index b1fd669218c25..0000000000000 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/termenum/TermEnumResponse.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.client.termenum; - -import org.elasticsearch.action.support.DefaultShardOperationFailedException; -import org.elasticsearch.action.support.broadcast.BroadcastResponse; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.xcontent.ConstructingObjectParser; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.common.xcontent.XContentParser; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collections; -import java.util.List; - -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; - -/** - * The response of the termenum/list action. - */ -public class TermEnumResponse extends BroadcastResponse { - - public static final String TERMS_FIELD = "terms"; - public static final String COMPLETE_FIELD = "complete"; - - @SuppressWarnings("unchecked") - static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( - "term_enum_results", - true, - arg -> { - BroadcastResponse response = (BroadcastResponse) arg[0]; - return new TermEnumResponse( - (List) arg[1], - response.getTotalShards(), - response.getSuccessfulShards(), - response.getFailedShards(), - Arrays.asList(response.getShardFailures()), - (Boolean) arg[2] - ); - } - ); - static { - declareBroadcastFields(PARSER); - PARSER.declareStringArray(optionalConstructorArg(), new ParseField(TERMS_FIELD)); - PARSER.declareBoolean(optionalConstructorArg(), new ParseField(COMPLETE_FIELD)); - } - - private final List terms; - - private boolean complete; - private int skippedShards; - - public TermEnumResponse( - List terms, - int totalShards, - int successfulShards, - int failedShards, - List shardFailures, boolean complete - ) { - super(totalShards, successfulShards, failedShards, shardFailures); - this.terms = terms == null ? Collections.emptyList() : terms; - this.complete = complete; - } - - /** - * The list of terms. - */ - public List getTerms() { - return terms; - } - - /** - * The number of shards skipped by the index filter - */ - public int getSkippedShards() { - return skippedShards; - } - - @Override - protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException { - if (getTerms() != null && getTerms().isEmpty() == false) { - builder.startArray(TERMS_FIELD); - for (String term : getTerms()) { - builder.value(term); - } - builder.endArray(); - } - builder.field(COMPLETE_FIELD, complete); - } - - public static TermEnumResponse fromXContent(XContentParser parser) { - return PARSER.apply(parser, null); - } - -} diff --git a/client/rest-high-level/src/test/java/org/elasticsearch/client/TermEnumIT.java b/client/rest-high-level/src/test/java/org/elasticsearch/client/TermEnumIT.java deleted file mode 100644 index 7fcc277ea3321..0000000000000 --- a/client/rest-high-level/src/test/java/org/elasticsearch/client/TermEnumIT.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0 and the Server Side Public License, v 1; you may not use this file except - * in compliance with, at your election, the Elastic License 2.0 or the Server - * Side Public License, v 1. - */ -package org.elasticsearch.client; - -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.elasticsearch.client.termenum.TermEnumRequest; -import org.elasticsearch.client.termenum.TermEnumResponse; -import org.junit.Before; - -import java.io.IOException; -import java.util.List; - -public class TermEnumIT extends ESRestHighLevelClientTestCase { - - @Before - public void indexDocuments() throws IOException { - // Create chain of doc IDs across indices 1->2->3 - Request doc1 = new Request(HttpPut.METHOD_NAME, "/index1/_doc/1"); - doc1.setJsonEntity("{ \"const\":\"aaa\"}"); - client().performRequest(doc1); - - Request doc2 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/2"); - doc2.setJsonEntity("{ \"const\":\"aba\"}"); - client().performRequest(doc2); - - Request doc3 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/3"); - doc3.setJsonEntity("{ \"const\":\"aba\"}"); - client().performRequest(doc3); - - Request doc4 = new Request(HttpPut.METHOD_NAME, "/index2/_doc/4"); - doc4.setJsonEntity("{ \"const\":\"abc\"}"); - client().performRequest(doc4); - - Request doc5 = new Request(HttpPut.METHOD_NAME, "/index3/_doc/5"); - doc5.setJsonEntity("{ \"const\":\"def\"}"); - client().performRequest(doc5); - - - client().performRequest(new Request(HttpPost.METHOD_NAME, "/_refresh")); - } - - public void testSuggests() throws Exception { - checkResults("index*", "a", "aaa", "aba", "abc"); - checkResults("index1", "a", "aaa"); - checkResults("index2", "a", "aba", "abc"); - checkResults("index*", "ab", "aba", "abc"); - checkResults("index*", "d", "def"); - checkResults("index*", "z"); - checkResults("index*", "Ab", true, "aba", "abc"); - } - - public void testCaseInsensitive() throws Exception { - checkResults("index*", "Ab", true, "aba", "abc"); - } - - public void checkResults(String indices, String prefix, String... expecteds) throws IOException { - checkResults(indices, prefix, false, expecteds); - } - - public void checkResults(String indices, String prefix, boolean caseInsensitive, String... expecteds) throws IOException { - TermEnumRequest ter = new TermEnumRequest(indices); - ter.field("const.keyword"); - ter.string(prefix); - ter.caseInsensitive(caseInsensitive); - TermEnumResponse result = highLevelClient().terms(ter, RequestOptions.DEFAULT); - if (expecteds == null) { - assertEquals(0, result.getTerms().size()); - return; - } - List terms = result.getTerms(); - assertEquals(expecteds.length, terms.size()); - for (int i = 0; i < expecteds.length; i++) { - assertEquals(expecteds[i], terms.get(i)); - } - } -} From 58bbf413044259897bd8c54e4013e6243c3eb630 Mon Sep 17 00:00:00 2001 From: markharwood Date: Wed, 10 Mar 2021 10:56:48 +0000 Subject: [PATCH 06/31] Return empty arrays when no results rather than no `terms` property at all --- .../xpack/core/termenum/action/TermEnumResponse.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumResponse.java index 2d7da3abce86c..61125261e6a87 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumResponse.java @@ -100,13 +100,13 @@ public void writeTo(StreamOutput out) throws IOException { @Override protected void addCustomXContentFields(XContentBuilder builder, Params params) throws IOException { + builder.startArray(TERMS_FIELD); if (getTerms() != null && getTerms().isEmpty() == false) { - builder.startArray(TERMS_FIELD); for (String term : getTerms()) { builder.value(term); } - builder.endArray(); } + builder.endArray(); builder.field(COMPLETE_FIELD, complete); } From 7fb781d5f55fdf345242ac73ca9165ee998e4768 Mon Sep 17 00:00:00 2001 From: markharwood Date: Thu, 11 Mar 2021 16:36:31 +0000 Subject: [PATCH 07/31] Fix bundling of shardIds for nodes, add success/fail accounting of numbers of shards --- .../action/TransportTermEnumAction.java | 90 +++++++++++++------ 1 file changed, 64 insertions(+), 26 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java index 1d863c3f93fc6..7f26a61db8d4a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java @@ -66,6 +66,7 @@ import java.util.Comparator; import java.util.HashMap; import java.util.HashSet; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Set; @@ -136,6 +137,9 @@ protected Map> getNodeBundles(ClusterState clusterState, Te // Group targeted shards by nodeId Map> fastNodeBundles = new HashMap<>(); for (String indexName : concreteIndices) { + + + //TODO remove this node filtering - rely on clients index_filters on _tier field instead? Settings settings = clusterState.metadata().index(indexName).getSettings(); if (IndexSettings.INDEX_SEARCH_THROTTLED.get(settings)) { // ignore slow throttled indices (this includes frozen) @@ -146,26 +150,40 @@ protected Map> getNodeBundles(ClusterState clusterState, Te GroupShardsIterator shards = clusterService.operationRouting() .searchShards(clusterState, singleIndex, null, null); - assert (shards.size() == 1); // We are only considering a single concrete index - ShardIterator shardsForIndex = shards.get(0); - for (ShardRouting shardRouting : shardsForIndex.getShardRoutings()) { - String nodeId = shardRouting.currentNodeId(); - + + Iterator shardsForIndex = shards.iterator(); + while (shardsForIndex.hasNext()) { + ShardIterator copiesOfShard = shardsForIndex.next(); + ShardRouting selectedCopyOfShard = null; + for(ShardRouting copy: copiesOfShard) { + // Pick the first active node with a copy of the shard on a node in the hot or warm tiers + if (copy.active() && copy.assignedToNode()) { + DiscoveryNode node = clusterState.getNodes().getDataNodes().get(copy.currentNodeId()); + // TODO remove this once we have queryable _tier field. + // (perhaps not as efficient though to hit data nodes with a canmatch query rather than + // avoid nodes based on tags here at coordinating node) + //Only consider hot and warm nodes + if (DataTier.isHotNode(node) || DataTier.isWarmNode(node)) { + selectedCopyOfShard = copy; + break; + } + } + } + if (selectedCopyOfShard == null) { + break; + } + String nodeId = selectedCopyOfShard.currentNodeId(); Set bundle = null; if (fastNodeBundles.containsKey(nodeId)){ bundle = fastNodeBundles.get(nodeId); } else { - DiscoveryNode node = clusterState.getNodes().getDataNodes().get(nodeId); - //Only consider hot and warm nodes - if (DataTier.isHotNode(node) || DataTier.isWarmNode(node)) { - bundle = new HashSet(); - fastNodeBundles.put(nodeId, bundle); - } + bundle = new HashSet(); + fastNodeBundles.put(nodeId, bundle); } if (bundle != null) { - bundle.add(shardRouting.shardId()); - } - } + bundle.add(selectedCopyOfShard.shardId()); + } + } } return fastNodeBundles; } @@ -179,27 +197,49 @@ protected ClusterBlockException checkRequestBlock(ClusterState state, TermEnumRe } protected TermEnumResponse newResponse(TermEnumRequest request, AtomicReferenceArray nodesResponses, - ClusterState clusterState, boolean complete) { + ClusterState clusterState, boolean complete, Map> nodeBundles) { int successfulShards = 0; int failedShards = 0; List shardFailures = null; Map combinedResults = new HashMap(); for (int i = 0; i < nodesResponses.length(); i++) { - Object shardResponse = nodesResponses.get(i); - if (shardResponse == null) { + Object nodeResponse = nodesResponses.get(i); + if (nodeResponse == null) { // simply ignore non active shards - } else if (shardResponse instanceof BroadcastShardOperationFailedException) { + } else if (nodeResponse instanceof BroadcastShardOperationFailedException) { + complete = false; failedShards++; if (shardFailures == null) { shardFailures = new ArrayList<>(); } - shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) shardResponse)); + shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) nodeResponse)); } else { - NodeTermEnumResponse str = (NodeTermEnumResponse) shardResponse; + NodeTermEnumResponse str = (NodeTermEnumResponse) nodeResponse; // Only one node response has to be incomplete for the entire result to be labelled incomplete. if (str.getComplete() == false) { complete = false; } + + Set shards = nodeBundles.get(str.getNodeId()); + if (str.getError() != null) { + complete = false; + // A single reported error is assumed to be for all shards queried on that node. + // When reading we read from multiple Lucene indices in one unified view so any error is + // assumed to be all shards on that node. + failedShards += shards.size(); + if (shardFailures == null) { + shardFailures = new ArrayList<>(); + } + for (ShardId failedShard : shards) { + shardFailures.add( + new DefaultShardOperationFailedException( + new BroadcastShardOperationFailedException(failedShard, str.getError()) + ) + ); + } + } else { + successfulShards += shards.size(); + } for (TermCount term : str.terms()) { TermCount existingTc = combinedResults.get(term.getTerm()); if (existingTc == null) { @@ -209,7 +249,6 @@ protected TermEnumResponse newResponse(TermEnumRequest request, AtomicReferenceA existingTc.addToDocCount(term.getDocCount()); } } - successfulShards++; } } int size = Math.min(request.size(), combinedResults.size()); @@ -237,7 +276,7 @@ public int compare(TermCount t1, TermCount t2) { break; } } - return new TermEnumResponse(terms, nodesResponses.length(), successfulShards, failedShards, shardFailures, complete); + return new TermEnumResponse(terms, (failedShards + successfulShards), successfulShards, failedShards, shardFailures, complete); } static class TermCountPriorityQueue extends PriorityQueue { @@ -355,7 +394,6 @@ protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Ta } termCount = 0; } - // MH TODO - multireader needs to deal with counts > 2bn - how to handle? limit num shards per reader? long df = te.docFreq(); BytesRef bytes = te.term(); termsList.add(new TermCount(bytes.utf8ToString(), df)); @@ -380,7 +418,7 @@ protected void toList(TermCountPriorityQueue pq, List termsList) { } } - // TODO remove this so we can shift code to server module + // TODO remove this so we can shift code to server module - see https://github.com/elastic/elasticsearch/issues/70221 private boolean canAccess(String indexName, String fieldName, XPackLicenseState frozenLicenseState, ThreadContext threadContext) { if (frozenLicenseState.isSecurityEnabled()) { var licenseChecker = new MemoizedSupplier<>(() -> frozenLicenseState.checkFeature(Feature.SECURITY_DLS_FLS)); @@ -454,7 +492,7 @@ public void start() { if (nodeBundles.size() == 0) { // no shards try { - listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState, true)); + listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState, true, nodeBundles)); } catch (Exception e) { listener.onFailure(e); } @@ -555,7 +593,7 @@ protected synchronized void finishHim(boolean complete) { return; } try { - listener.onResponse(newResponse(request, nodesResponses, clusterState, complete)); + listener.onResponse(newResponse(request, nodesResponses, clusterState, complete, nodeBundles)); } catch (Exception e) { listener.onFailure(e); } From 4e9da3918ad693375203246f2bb73303007156f3 Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 23 Mar 2021 12:26:15 +0000 Subject: [PATCH 08/31] Type fixes --- .../core/termenum/action/TransportTermEnumAction.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java index 7f26a61db8d4a..6a1efffb73c8e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java @@ -196,7 +196,7 @@ protected ClusterBlockException checkRequestBlock(ClusterState state, TermEnumRe return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices); } - protected TermEnumResponse newResponse(TermEnumRequest request, AtomicReferenceArray nodesResponses, + protected TermEnumResponse newResponse(TermEnumRequest request, AtomicReferenceArray nodesResponses, ClusterState clusterState, boolean complete, Map> nodeBundles) { int successfulShards = 0; int failedShards = 0; @@ -459,7 +459,7 @@ protected class AsyncBroadcastAction { private final DiscoveryNodes nodes; private final int expectedOps; private final AtomicInteger counterOps = new AtomicInteger(); - private final AtomicReferenceArray nodesResponses; + private final AtomicReferenceArray nodesResponses; private Map> nodeBundles; protected AsyncBroadcastAction(Task task, TermEnumRequest request, ActionListener listener) { @@ -492,7 +492,7 @@ public void start() { if (nodeBundles.size() == 0) { // no shards try { - listener.onResponse(newResponse(request, new AtomicReferenceArray(0), clusterState, true, nodeBundles)); + listener.onResponse(newResponse(request, new AtomicReferenceArray<>(0), clusterState, true, nodeBundles)); } catch (Exception e) { listener.onFailure(e); } @@ -570,7 +570,6 @@ public void handleException(TransportException e) { } } - @SuppressWarnings({ "unchecked" }) protected void onOperation(String nodeId, int nodeIndex, NodeTermEnumResponse response) { logger.trace("received response for node {}", nodeId); nodesResponses.set(nodeIndex, response); From cac1bb32d38a4481dd6812adb80790dbfa8784e9 Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 23 Mar 2021 15:32:04 +0000 Subject: [PATCH 09/31] Types warning --- .../xpack/core/termenum/action/RestTermEnumActionTests.java | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/action/RestTermEnumActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/action/RestTermEnumActionTests.java index 4ee63445a4838..62499c2e9bab9 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/action/RestTermEnumActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/action/RestTermEnumActionTests.java @@ -62,6 +62,7 @@ public class RestTermEnumActionTests extends ESTestCase { * This lower level of execution is out of the scope of this test. */ @BeforeClass + @SuppressWarnings("rawtypes") public static void stubTermEnumAction() { final TaskManager taskManager = new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()); From 810d638656e36752cbe3f7c97a4dbdc6e8eeff1b Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 6 Apr 2021 18:34:54 +0100 Subject: [PATCH 10/31] Removed hot/warm tier tests (in anticipation of new queryable _tier field) Move canMatch logic to run on network thread Injected searchService so we can use its canMatch method --- .../termenum/action/NodeTermEnumRequest.java | 4 + .../action/TransportTermEnumAction.java | 142 +++++++++--------- 2 files changed, 74 insertions(+), 72 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java index 9e16110234420..7edd8a7324541 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java @@ -160,4 +160,8 @@ public IndicesOptions indicesOptions() { return null; } + public boolean remove(ShardId shardId) { + return shardIds.remove(shardId); + } + } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java index 6a1efffb73c8e..7fe5aba9dc44c 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java @@ -77,9 +77,9 @@ public class TransportTermEnumAction extends HandledTransportAction shardIds, TermEnumRequest request) { // Given we look terms up in the terms dictionary alias filters is another aspect of search (like DLS) that we // currently do not support. -// final ClusterState clusterState = clusterService.state(); -// final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); -// final AliasFilter aliasFilter = searchService.buildAliasFilter(clusterState, shard.getIndexName(), indicesAndAliases); + // final ClusterState clusterState = clusterService.state(); + // final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); + // final AliasFilter aliasFilter = searchService.buildAliasFilter(clusterState, shard.getIndexName(), indicesAndAliases); return new NodeTermEnumRequest(nodeId, shardIds, request); } @@ -137,44 +139,36 @@ protected Map> getNodeBundles(ClusterState clusterState, Te // Group targeted shards by nodeId Map> fastNodeBundles = new HashMap<>(); for (String indexName : concreteIndices) { - - - //TODO remove this node filtering - rely on clients index_filters on _tier field instead? + + // TODO remove this node filtering - rely on clients index_filters on _tier field instead? Settings settings = clusterState.metadata().index(indexName).getSettings(); if (IndexSettings.INDEX_SEARCH_THROTTLED.get(settings)) { // ignore slow throttled indices (this includes frozen) continue; } - - String[] singleIndex = {indexName}; - + + String[] singleIndex = { indexName }; + GroupShardsIterator shards = clusterService.operationRouting() .searchShards(clusterState, singleIndex, null, null); - + Iterator shardsForIndex = shards.iterator(); while (shardsForIndex.hasNext()) { ShardIterator copiesOfShard = shardsForIndex.next(); ShardRouting selectedCopyOfShard = null; - for(ShardRouting copy: copiesOfShard) { - // Pick the first active node with a copy of the shard on a node in the hot or warm tiers + for (ShardRouting copy : copiesOfShard) { + // Pick the first active node with a copy of the shard if (copy.active() && copy.assignedToNode()) { - DiscoveryNode node = clusterState.getNodes().getDataNodes().get(copy.currentNodeId()); - // TODO remove this once we have queryable _tier field. - // (perhaps not as efficient though to hit data nodes with a canmatch query rather than - // avoid nodes based on tags here at coordinating node) - //Only consider hot and warm nodes - if (DataTier.isHotNode(node) || DataTier.isWarmNode(node)) { - selectedCopyOfShard = copy; - break; - } + selectedCopyOfShard = copy; + break; } } if (selectedCopyOfShard == null) { break; } - String nodeId = selectedCopyOfShard.currentNodeId(); + String nodeId = selectedCopyOfShard.currentNodeId(); Set bundle = null; - if (fastNodeBundles.containsKey(nodeId)){ + if (fastNodeBundles.containsKey(nodeId)) { bundle = fastNodeBundles.get(nodeId); } else { bundle = new HashSet(); @@ -182,8 +176,8 @@ protected Map> getNodeBundles(ClusterState clusterState, Te } if (bundle != null) { bundle.add(selectedCopyOfShard.shardId()); - } - } + } + } } return fastNodeBundles; } @@ -196,8 +190,13 @@ protected ClusterBlockException checkRequestBlock(ClusterState state, TermEnumRe return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices); } - protected TermEnumResponse newResponse(TermEnumRequest request, AtomicReferenceArray nodesResponses, - ClusterState clusterState, boolean complete, Map> nodeBundles) { + protected TermEnumResponse newResponse( + TermEnumRequest request, + AtomicReferenceArray nodesResponses, + ClusterState clusterState, + boolean complete, + Map> nodeBundles + ) { int successfulShards = 0; int failedShards = 0; List shardFailures = null; @@ -219,12 +218,12 @@ protected TermEnumResponse newResponse(TermEnumRequest request, AtomicReferenceA if (str.getComplete() == false) { complete = false; } - + Set shards = nodeBundles.get(str.getNodeId()); if (str.getError() != null) { complete = false; - // A single reported error is assumed to be for all shards queried on that node. - // When reading we read from multiple Lucene indices in one unified view so any error is + // A single reported error is assumed to be for all shards queried on that node. + // When reading we read from multiple Lucene indices in one unified view so any error is // assumed to be all shards on that node. failedShards += shards.size(); if (shardFailures == null) { @@ -291,15 +290,10 @@ protected boolean lessThan(TermCount a, TermCount b) { } } - + protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Task task) throws IOException { List termsList = new ArrayList<>(); String error = null; - - // DLS/FLS check copied from ResizeRequestInterceptor - // MH code - not sure this is the right context - ThreadContext threadContext = transportService.getThreadPool().getThreadContext(); - final XPackLicenseState frozenLicenseState = licenseState.copyCurrentLicenseState(); long timeout_millis = request.timeout(); long scheduledEnd = request.shardStartedTimeMillis() + timeout_millis; @@ -325,23 +319,16 @@ protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Ta null, Collections.emptyMap() ); - if (canAccess(shardId.getIndexName(), request.field(), frozenLicenseState, threadContext) && canMatchShard( - shardId, - request, - queryShardContext - )) { - - final MappedFieldType mappedFieldType = indexShard.mapperService().fieldType(request.field()); - if (mappedFieldType != null) { - TermsEnum terms = mappedFieldType.getTerms(request.caseInsensitive(), request.string(), queryShardContext); - if (terms != null) { - shardTermsEnums.add(terms); - } + final MappedFieldType mappedFieldType = indexShard.mapperService().fieldType(request.field()); + if (mappedFieldType != null) { + TermsEnum terms = mappedFieldType.getTerms(request.caseInsensitive(), request.string(), queryShardContext); + if (terms != null) { + shardTermsEnums.add(terms); } } } MultiShardTermsEnum te = new MultiShardTermsEnum(shardTermsEnums.toArray(new TermsEnum[0])); - + int shard_size = request.size(); // All the above prep might take a while - do a timer check now before we continue further. if (System.currentTimeMillis() > scheduledEnd) { @@ -405,8 +392,7 @@ protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Ta } catch (Exception e) { error = e.getMessage(); - } - finally { + } finally { IOUtils.close(openedResources); } return new NodeTermEnumResponse(request.nodeId(), termsList, error, true); @@ -417,18 +403,16 @@ protected void toList(TermCountPriorityQueue pq, List termsList) { termsList.add(pq.pop()); } } - + // TODO remove this so we can shift code to server module - see https://github.com/elastic/elasticsearch/issues/70221 private boolean canAccess(String indexName, String fieldName, XPackLicenseState frozenLicenseState, ThreadContext threadContext) { if (frozenLicenseState.isSecurityEnabled()) { var licenseChecker = new MemoizedSupplier<>(() -> frozenLicenseState.checkFeature(Feature.SECURITY_DLS_FLS)); - IndicesAccessControl indicesAccessControl = - threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY); - IndicesAccessControl.IndexAccessControl indexAccessControl = - indicesAccessControl.getIndexPermissions(indexName); - //TODO There was a suggestion we could remove FLS check because the reader does that filtering anyway - didn't + IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY); + IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(indexName); + // TODO There was a suggestion we could remove FLS check because the reader does that filtering anyway - didn't // turn out that way when I tested it. - // TODO remove DLS check by getting a new security feature to filter at coordinating node the choice of indices + // TODO remove DLS check by getting a new security feature to filter at coordinating node the choice of indices // based on if ANY DLS is present on an index. if (indexAccessControl != null) { final boolean fls = indexAccessControl.getFieldPermissions().grantsAccessTo(fieldName) == false; @@ -441,14 +425,14 @@ private boolean canAccess(String indexName, String fieldName, XPackLicenseState return true; } - private boolean canMatchShard(ShardId shardId, NodeTermEnumRequest req, SearchExecutionContext queryShardContext) throws IOException { + private boolean canMatchShard(ShardId shardId, NodeTermEnumRequest req) throws IOException { if (req.indexFilter() == null || req.indexFilter() instanceof MatchAllQueryBuilder) { return true; } ShardSearchRequest searchRequest = new ShardSearchRequest(shardId, req.shardStartedTimeMillis(), AliasFilter.EMPTY); searchRequest.source(new SearchSourceBuilder().query(req.indexFilter())); - return SearchService.queryStillMatchesAfterRewrite(searchRequest, queryShardContext); - } + return searchService.canMatch(searchRequest).canMatch(); + } protected class AsyncBroadcastAction { @@ -515,11 +499,11 @@ public void start() { } } } - + // Returns true if we exited with a response to the caller. boolean checkForEarlyFinish() { long now = System.currentTimeMillis(); - if ( (now - task.getStartTime()) > request.timeout().getMillis() ) { + if ((now - task.getStartTime()) > request.timeout().getMillis()) { finishHim(false); return true; } @@ -529,11 +513,11 @@ boolean checkForEarlyFinish() { protected void performOperation(final String nodeId, final Set shardIds, final int nodeIndex) { if (shardIds.size() == 0) { // no more active shards... (we should not really get here, just safety) - //MH TODO somewhat arbitrarily returining firsy + // MH TODO somewhat arbitrarily returining firsy onNoOperation(nodeId); } else { try { - //TODO pass through a reduced timeout (the original time limit, minus whatever we may have + // TODO pass through a reduced timeout (the original time limit, minus whatever we may have // spent already getting to this point. final NodeTermEnumRequest nodeRequest = newNodeRequest(nodeId, shardIds, request); nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId()); @@ -577,7 +561,7 @@ protected void onOperation(String nodeId, int nodeIndex, NodeTermEnumResponse re finishHim(true); } else { checkForEarlyFinish(); - } + } } void onNoOperation(String nodeId) { @@ -595,8 +579,7 @@ protected synchronized void finishHim(boolean complete) { listener.onResponse(newResponse(request, nodesResponses, clusterState, complete, nodeBundles)); } catch (Exception e) { listener.onFailure(e); - } - finally { + } finally { listener = null; } } @@ -623,7 +606,22 @@ public void messageReceived(NodeTermEnumRequest request, TransportChannel channe } } - private void asyncNodeOperation(NodeTermEnumRequest request, Task task, ActionListener listener) { + private void asyncNodeOperation(NodeTermEnumRequest request, Task task, ActionListener listener) + throws IOException { + // DLS/FLS check copied from ResizeRequestInterceptor - check permissions and + // any index_filter canMatch checks on network thread before allocating work + ThreadContext threadContext = transportService.getThreadPool().getThreadContext(); + final XPackLicenseState frozenLicenseState = licenseState.copyCurrentLicenseState(); + for (ShardId shardId : request.shardIds().toArray(new ShardId[0])) { + if (canAccess(shardId.getIndexName(), request.field(), frozenLicenseState, threadContext) == false || canMatchShard( + shardId, + request + ) == false) { + // Permission denied or can't match, remove shardID from request + request.remove(shardId); + } + } + // TODO avoid making request if no shards searchable? transportService.getThreadPool() .executor(shardExecutor) .execute(ActionRunnable.supply(listener, () -> dataNodeOperation(request, task))); From 156302f3edfd1032c92b4e766eaacaaa3fe983a5 Mon Sep 17 00:00:00 2001 From: markharwood Date: Thu, 8 Apr 2021 15:58:41 +0100 Subject: [PATCH 11/31] Move rest-api-spec and related YML test to new standard home for this stuff. --- .../src/main}/resources/rest-api-spec/api/termsenum.json | 0 .../resources/rest-api-spec/test/term_enum/10_basic.yml | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {x-pack/plugin/src/test => rest-api-spec/src/main}/resources/rest-api-spec/api/termsenum.json (100%) rename {x-pack/plugin/src/test => rest-api-spec/src/yamlRestTest}/resources/rest-api-spec/test/term_enum/10_basic.yml (100%) diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/api/termsenum.json b/rest-api-spec/src/main/resources/rest-api-spec/api/termsenum.json similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/api/termsenum.json rename to rest-api-spec/src/main/resources/rest-api-spec/api/termsenum.json diff --git a/x-pack/plugin/src/test/resources/rest-api-spec/test/term_enum/10_basic.yml b/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml similarity index 100% rename from x-pack/plugin/src/test/resources/rest-api-spec/test/term_enum/10_basic.yml rename to rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml From 62af758dc220f2a25d825c4e809403be73dfcb4d Mon Sep 17 00:00:00 2001 From: markharwood Date: Thu, 8 Apr 2021 16:16:12 +0100 Subject: [PATCH 12/31] Unused import --- .../xpack/core/termenum/action/TransportTermEnumAction.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java index 7fe5aba9dc44c..31747d6226e6b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java @@ -54,7 +54,6 @@ import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; -import org.elasticsearch.xpack.core.DataTier; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; From 03f79deb3b10a9907699a8ca981a6b6c77b8bf75 Mon Sep 17 00:00:00 2001 From: markharwood Date: Thu, 8 Apr 2021 17:01:12 +0100 Subject: [PATCH 13/31] Move test to xpack --- .../resources/rest-api-spec/test/term_enum/10_basic.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {rest-api-spec => x-pack/plugin/core}/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml (100%) diff --git a/rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml b/x-pack/plugin/core/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml similarity index 100% rename from rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml rename to x-pack/plugin/core/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml From 53e12cdbbd0a1d884da6b44d19eaabf59eaaa35f Mon Sep 17 00:00:00 2001 From: markharwood Date: Mon, 12 Apr 2021 15:05:05 +0100 Subject: [PATCH 14/31] =?UTF-8?q?Return=20early=20on=20network=20thread=20?= =?UTF-8?q?if=20can=E2=80=99t=20match=20any=20shards.=20Remove=20FLS=20log?= =?UTF-8?q?ic=20now=20we=20use=20the=20right=20searcher.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../action/TransportTermEnumAction.java | 25 ++++++++----------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java index 31747d6226e6b..e764184f1378b 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java @@ -192,7 +192,6 @@ protected ClusterBlockException checkRequestBlock(ClusterState state, TermEnumRe protected TermEnumResponse newResponse( TermEnumRequest request, AtomicReferenceArray nodesResponses, - ClusterState clusterState, boolean complete, Map> nodeBundles ) { @@ -308,7 +307,7 @@ protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Ta final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); final IndexShard indexShard = indexService.getShard(shardId.getId()); - Engine.Searcher searcher = indexShard.acquireSearcher(Engine.CAN_MATCH_SEARCH_SOURCE); + Engine.Searcher searcher = indexShard.acquireSearcher(Engine.SEARCH_SOURCE); openedResources.add(searcher); final SearchExecutionContext queryShardContext = indexService.newSearchExecutionContext( shardId.id(), @@ -409,14 +408,9 @@ private boolean canAccess(String indexName, String fieldName, XPackLicenseState var licenseChecker = new MemoizedSupplier<>(() -> frozenLicenseState.checkFeature(Feature.SECURITY_DLS_FLS)); IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY); IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(indexName); - // TODO There was a suggestion we could remove FLS check because the reader does that filtering anyway - didn't - // turn out that way when I tested it. - // TODO remove DLS check by getting a new security feature to filter at coordinating node the choice of indices - // based on if ANY DLS is present on an index. if (indexAccessControl != null) { - final boolean fls = indexAccessControl.getFieldPermissions().grantsAccessTo(fieldName) == false; final boolean dls = indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions(); - if ((fls || dls) && licenseChecker.get()) { + if ( dls && licenseChecker.get()) { return false; } } @@ -475,7 +469,7 @@ public void start() { if (nodeBundles.size() == 0) { // no shards try { - listener.onResponse(newResponse(request, new AtomicReferenceArray<>(0), clusterState, true, nodeBundles)); + listener.onResponse(newResponse(request, new AtomicReferenceArray<>(0), true, nodeBundles)); } catch (Exception e) { listener.onFailure(e); } @@ -575,7 +569,7 @@ protected synchronized void finishHim(boolean complete) { return; } try { - listener.onResponse(newResponse(request, nodesResponses, clusterState, complete, nodeBundles)); + listener.onResponse(newResponse(request, nodesResponses, complete, nodeBundles)); } catch (Exception e) { listener.onFailure(e); } finally { @@ -620,9 +614,12 @@ private void asyncNodeOperation(NodeTermEnumRequest request, Task task, ActionLi request.remove(shardId); } } - // TODO avoid making request if no shards searchable? - transportService.getThreadPool() - .executor(shardExecutor) - .execute(ActionRunnable.supply(listener, () -> dataNodeOperation(request, task))); + if (request.shardIds().size() == 0) { + listener.onResponse(new NodeTermEnumResponse(request.nodeId(), Collections.emptyList(), null, true)); + } else { + transportService.getThreadPool() + .executor(shardExecutor) + .execute(ActionRunnable.supply(listener, () -> dataNodeOperation(request, task))); + } } } From b36f477f5901ed4f5eaeb1686fbe9b2676a88e3d Mon Sep 17 00:00:00 2001 From: markharwood Date: Mon, 12 Apr 2021 15:34:58 +0100 Subject: [PATCH 15/31] Removed sort by popularity option --- docs/reference/search/term-enum.asciidoc | 10 +- .../termenum/action/NodeTermEnumRequest.java | 7 -- .../core/termenum/action/TermEnumAction.java | 1 - .../core/termenum/action/TermEnumRequest.java | 20 +--- .../action/TermEnumRequestBuilder.java | 4 - .../action/TransportTermEnumAction.java | 106 +++--------------- 6 files changed, 19 insertions(+), 129 deletions(-) diff --git a/docs/reference/search/term-enum.asciidoc b/docs/reference/search/term-enum.asciidoc index 018e95891ae96..2e981dc8b236f 100644 --- a/docs/reference/search/term-enum.asciidoc +++ b/docs/reference/search/term-enum.asciidoc @@ -82,7 +82,7 @@ How many matching terms to return. Defaults to 10 `timeout`:: (Optional, integer) The maximum length of time in milliseconds to spend collecting results. Defaults to 1000. -If the timeout is exceeded a `timed_out` flag is set in the response and the results may +If the timeout is exceeded the `complete` flag set to false in the response and the results may be partial or empty. [[term-enum-case_insensitive-param]] @@ -91,14 +91,6 @@ be partial or empty. When true the provided search string is matched against index terms without case sensitivity. Defaults to false. -[[term-enum-sort_by_popularity-param]] -`sort_by_popularity`:: -(Optional, boolean) -When true terms are sorted by popularity, when false (the default) they are sorted alphabetically. -Sorting by popularity can be more costly so nodes will only consider up to a few thousand matching -terms to bound the cost of this calculation. If this term limit is reached in the analysis -the final response will have the `complete` flag set to false. - [[term-enum-index_filter-param]] `index_filter`:: (Optional, <> Allows to filter an index shard if the provided diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java index 7edd8a7324541..f2017f798f047 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java @@ -30,7 +30,6 @@ public class NodeTermEnumRequest extends TransportRequest implements IndicesRequ private long taskStartedTimeMillis; private long nodeStartedTimeMillis; private boolean caseInsensitive; - private boolean sortByPopularity; private int size; private long timeout; private final QueryBuilder indexFilter; @@ -46,7 +45,6 @@ public NodeTermEnumRequest(StreamInput in) throws IOException { field = in.readString(); string = in.readString(); caseInsensitive = in.readBoolean(); - sortByPopularity = in.readBoolean(); size = in.readVInt(); timeout = in.readVLong(); taskStartedTimeMillis = in.readVLong(); @@ -65,7 +63,6 @@ public NodeTermEnumRequest(final String nodeId, final Set shardIds, Ter this.caseInsensitive = request.caseInsensitive(); this.size = request.size(); this.timeout = request.timeout().getMillis(); - this.sortByPopularity = request.sortByPopularity(); this.taskStartedTimeMillis = request.taskStartTimeMillis; this.indexFilter = request.indexFilter(); this.nodeId = nodeId; @@ -105,9 +102,6 @@ public boolean caseInsensitive() { return caseInsensitive; } - public boolean sortByPopularity() { - return sortByPopularity; - } public int size() { return size; } @@ -125,7 +119,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(field); out.writeString(string); out.writeBoolean(caseInsensitive); - out.writeBoolean(sortByPopularity); out.writeVInt(size); // Adjust the amount of permitted time the shard has remaining to gather terms. long timeSpentSoFarInCoordinatingNode = System.currentTimeMillis() - taskStartedTimeMillis; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumAction.java index eff9a77eceb82..872523caaf670 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumAction.java @@ -39,7 +39,6 @@ public static TermEnumRequest fromXContent(XContentParser parser, String... indi PARSER.declareString(TermEnumRequest::string, new ParseField("string")); PARSER.declareInt(TermEnumRequest::size, new ParseField("size")); PARSER.declareBoolean(TermEnumRequest::caseInsensitive, new ParseField("case_insensitive")); - PARSER.declareBoolean(TermEnumRequest::sortByPopularity, new ParseField("sort_by_popularity")); PARSER.declareInt(TermEnumRequest::timeoutInMillis, new ParseField("timeout")); PARSER.declareObject(TermEnumRequest::indexFilter, (p, context) -> parseInnerQueryBuilder(p),INDEX_FILTER); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java index f3ec25c7a11dd..31d349f1a8cc8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java @@ -33,7 +33,6 @@ public class TermEnumRequest extends BroadcastRequest implement private String string; private int size = DEFAULT_SIZE; private boolean caseInsensitive; - private boolean sortByPopularity; long taskStartTimeMillis; private QueryBuilder indexFilter; @@ -46,7 +45,6 @@ public TermEnumRequest(StreamInput in) throws IOException { field = in.readString(); string = in.readString(); caseInsensitive = in.readBoolean(); - sortByPopularity = in.readBoolean(); size = in.readVInt(); indexFilter = in.readOptionalNamedWriteable(QueryBuilder.class); } @@ -98,20 +96,6 @@ public String string() { return string; } - /** - * sort terms by popularity - */ - public boolean sortByPopularity() { - return sortByPopularity; - } - - /** - * sort terms by popularity - */ - public void sortByPopularity(boolean sortByPopularity) { - this.sortByPopularity = sortByPopularity; - } - /** * The number of terms to return */ @@ -164,7 +148,6 @@ public void writeTo(StreamOutput out) throws IOException { out.writeString(field); out.writeString(string); out.writeBoolean(caseInsensitive); - out.writeBoolean(sortByPopularity); out.writeVInt(size); out.writeOptionalNamedWriteable(indexFilter); } @@ -172,7 +155,7 @@ public void writeTo(StreamOutput out) throws IOException { @Override public String toString() { return "[" + Arrays.toString(indices) + "] field[" + field + "], string[" + string + "] " + " size=" + size + " timeout=" - + timeout().getMillis() + " sort_by_popularity = " + sortByPopularity + " case_insensitive=" + + timeout().getMillis() + " case_insensitive=" + caseInsensitive + " indexFilter = "+ indexFilter; } @@ -184,7 +167,6 @@ public XContentBuilder toXContent(XContentBuilder builder, Params params) throws builder.field("size", size); builder.field("timeout", timeout().getMillis()); builder.field("case_insensitive", caseInsensitive); - builder.field("sort_by_popularity", sortByPopularity); if (indexFilter != null) { builder.field("index_filter", indexFilter); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequestBuilder.java index 267a689117e38..995047fb6cc37 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequestBuilder.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequestBuilder.java @@ -25,10 +25,6 @@ public TermEnumRequestBuilder setString(String string) { return this; } - public TermEnumRequestBuilder setSortByPopularity(boolean sortByPopularity) { - request.sortByPopularity(sortByPopularity); - return this; - } public TermEnumRequestBuilder setSize(int size) { request.size(size); return this; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java index e764184f1378b..35b622cb0a014 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java @@ -29,11 +29,9 @@ import org.elasticsearch.common.MemoizedSupplier; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.IndexService; -import org.elasticsearch.index.IndexSettings; import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.MatchAllQueryBuilder; @@ -139,13 +137,6 @@ protected Map> getNodeBundles(ClusterState clusterState, Te Map> fastNodeBundles = new HashMap<>(); for (String indexName : concreteIndices) { - // TODO remove this node filtering - rely on clients index_filters on _tier field instead? - Settings settings = clusterState.metadata().index(indexName).getSettings(); - if (IndexSettings.INDEX_SEARCH_THROTTLED.get(settings)) { - // ignore slow throttled indices (this includes frozen) - continue; - } - String[] singleIndex = { indexName }; GroupShardsIterator shards = clusterService.operationRouting() @@ -251,21 +242,12 @@ protected TermEnumResponse newResponse( int size = Math.min(request.size(), combinedResults.size()); List terms = new ArrayList<>(size); TermCount[] sortedCombinedResults = combinedResults.values().toArray(new TermCount[0]); - if (request.sortByPopularity()) { - // Sort by doc count descending - Arrays.sort(sortedCombinedResults, new Comparator() { - public int compare(TermCount t1, TermCount t2) { - return Long.compare(t2.getDocCount(), t1.getDocCount()); - } - }); - } else { - // Sort alphabetically - Arrays.sort(sortedCombinedResults, new Comparator() { - public int compare(TermCount t1, TermCount t2) { - return t1.getTerm().compareTo(t2.getTerm()); - } - }); - } + // Sort alphabetically + Arrays.sort(sortedCombinedResults, new Comparator() { + public int compare(TermCount t1, TermCount t2) { + return t1.getTerm().compareTo(t2.getTerm()); + } + }); for (TermCount term : sortedCombinedResults) { terms.add(term.getTerm()); @@ -276,19 +258,6 @@ public int compare(TermCount t1, TermCount t2) { return new TermEnumResponse(terms, (failedShards + successfulShards), successfulShards, failedShards, shardFailures, complete); } - static class TermCountPriorityQueue extends PriorityQueue { - - TermCountPriorityQueue(int maxSize) { - super(maxSize); - } - - @Override - protected boolean lessThan(TermCount a, TermCount b) { - return a.getDocCount() < b.getDocCount(); - } - - } - protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Task task) throws IOException { List termsList = new ArrayList<>(); String error = null; @@ -335,56 +304,21 @@ protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Ta int numTermsBetweenClockChecks = 100; int termCount = 0; - if (request.sortByPopularity()) { - // Collect most popular matches - TermCountPriorityQueue pq = new TermCountPriorityQueue(shard_size); - TermCount spare = null; - // TODO make this a setting (cluster or index level?) - int maxTermsConsidered = 10000; - while (te.next() != null) { - termCount++; - if (termCount >= maxTermsConsidered) { - toList(pq, termsList); + // Collect in alphabetical order + while (te.next() != null) { + termCount++; + if (termCount > numTermsBetweenClockChecks) { + if (System.currentTimeMillis() > scheduledEnd) { boolean complete = te.next() == null; return new NodeTermEnumResponse(request.nodeId(), termsList, error, complete); } - if (termCount % numTermsBetweenClockChecks == 0) { - if (System.currentTimeMillis() > scheduledEnd) { - // Gather what we have collected so far - toList(pq, termsList); - boolean complete = te.next() == null; - return new NodeTermEnumResponse(request.nodeId(), termsList, error, complete); - } - } - long df = te.docFreq(); - BytesRef bytes = te.term(); - - if (spare == null) { - spare = new TermCount(bytes.utf8ToString(), df); - } else { - spare.setTerm(bytes.utf8ToString()); - spare.setDocCount(df); - } - spare = pq.insertWithOverflow(spare); + termCount = 0; } - toList(pq, termsList); - } else { - // Collect in alphabetical order - while (te.next() != null) { - termCount++; - if (termCount > numTermsBetweenClockChecks) { - if (System.currentTimeMillis() > scheduledEnd) { - boolean complete = te.next() == null; - return new NodeTermEnumResponse(request.nodeId(), termsList, error, complete); - } - termCount = 0; - } - long df = te.docFreq(); - BytesRef bytes = te.term(); - termsList.add(new TermCount(bytes.utf8ToString(), df)); - if (termsList.size() >= shard_size) { - break; - } + long df = te.docFreq(); + BytesRef bytes = te.term(); + termsList.add(new TermCount(bytes.utf8ToString(), df)); + if (termsList.size() >= shard_size) { + break; } } @@ -396,12 +330,6 @@ protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Ta return new NodeTermEnumResponse(request.nodeId(), termsList, error, true); } - protected void toList(TermCountPriorityQueue pq, List termsList) { - while (pq.size() > 0) { - termsList.add(pq.pop()); - } - } - // TODO remove this so we can shift code to server module - see https://github.com/elastic/elasticsearch/issues/70221 private boolean canAccess(String indexName, String fieldName, XPackLicenseState frozenLicenseState, ThreadContext threadContext) { if (frozenLicenseState.isSecurityEnabled()) { From 3b1b6d935a51a4940e6ea7d05d97e4b3d87b1614 Mon Sep 17 00:00:00 2001 From: markharwood Date: Mon, 12 Apr 2021 15:58:25 +0100 Subject: [PATCH 16/31] Unused import --- .../xpack/core/termenum/action/TransportTermEnumAction.java | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java index 35b622cb0a014..d935cdbc40cec 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java @@ -9,7 +9,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; -import org.apache.lucene.util.PriorityQueue; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.ActionFilters; From 5250cc7e524b1019121feadbde24b80167a8a0da Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 13 Apr 2021 14:59:23 +0100 Subject: [PATCH 17/31] Addressing some review comments (thanks Jim/Adrien!) --- .../elasticsearch/client/RequestConverters.java | 2 +- .../elasticsearch/client/RestHighLevelClient.java | 15 ++++++++------- docs/reference/search/term-enum.asciidoc | 1 - .../core/termenum/rest/RestTermEnumAction.java | 5 ----- .../core/termenum/MultiShardTermsEnumTests.java | 11 ++++++----- .../mapper/ConstantKeywordFieldMapper.java | 2 +- 6 files changed, 16 insertions(+), 20 deletions(-) diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java index cfa721b49973c..9807f5483c5f0 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RequestConverters.java @@ -536,7 +536,7 @@ static Request rankEval(RankEvalRequest rankEvalRequest) throws IOException { request.setEntity(createEntity(rankEvalRequest.getRankEvalSpec(), REQUEST_BODY_CONTENT_TYPE)); return request; } - + static Request reindex(ReindexRequest reindexRequest) throws IOException { return prepareReindexRequest(reindexRequest, true); } diff --git a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java index 5747490a5d24b..455343d99627a 100644 --- a/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java +++ b/client/rest-high-level/src/main/java/org/elasticsearch/client/RestHighLevelClient.java @@ -120,18 +120,18 @@ import org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.sampler.InternalSampler; import org.elasticsearch.search.aggregations.bucket.sampler.ParsedSampler; -import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms; import org.elasticsearch.search.aggregations.bucket.terms.LongRareTerms; -import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms; import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongRareTerms; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms; import org.elasticsearch.search.aggregations.bucket.terms.ParsedSignificantLongTerms; import org.elasticsearch.search.aggregations.bucket.terms.ParsedSignificantStringTerms; import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringRareTerms; -import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; import org.elasticsearch.search.aggregations.bucket.terms.SignificantLongTerms; import org.elasticsearch.search.aggregations.bucket.terms.SignificantStringTerms; +import org.elasticsearch.search.aggregations.bucket.terms.DoubleTerms; +import org.elasticsearch.search.aggregations.bucket.terms.LongTerms; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedDoubleTerms; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedLongTerms; +import org.elasticsearch.search.aggregations.bucket.terms.ParsedStringTerms; import org.elasticsearch.search.aggregations.bucket.terms.StringRareTerms; import org.elasticsearch.search.aggregations.bucket.terms.StringTerms; import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; @@ -1423,7 +1423,8 @@ public final RankEvalResponse rankEval(RankEvalRequest rankEvalRequest, RequestO return performRequestAndParseEntity(rankEvalRequest, RequestConverters::rankEval, options, RankEvalResponse::fromXContent, emptySet()); } - + + /** * Executes a request using the Multi Search Template API. * @@ -1465,7 +1466,7 @@ public final Cancellable rankEvalAsync(RankEvalRequest rankEvalRequest, RequestO RankEvalResponse::fromXContent, listener, emptySet()); } - + /** * Executes a request using the Field Capabilities API. * See Field Capabilities API diff --git a/docs/reference/search/term-enum.asciidoc b/docs/reference/search/term-enum.asciidoc index 2e981dc8b236f..fa9e09056b4bd 100644 --- a/docs/reference/search/term-enum.asciidoc +++ b/docs/reference/search/term-enum.asciidoc @@ -31,7 +31,6 @@ The API returns the following response: "complete" : true } -------------------------------------------------- -// TESTRESPONSE[s/8/$body.terms.0.doc_count/] The "complete" flag is false if time or space constraints were met and the set of terms examined was not the full set of available values. diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/rest/RestTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/rest/RestTermEnumAction.java index ae95a4e4ef4d7..a890a554b40b9 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/rest/RestTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/rest/RestTermEnumAction.java @@ -43,11 +43,6 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC return channel -> client.execute(TermEnumAction.INSTANCE, termEnumRequest, new RestToXContentListener<>(channel)); } - - - - } - } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/MultiShardTermsEnumTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/MultiShardTermsEnumTests.java index 9f416b5c29ec6..0868a16bae16f 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/MultiShardTermsEnumTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/MultiShardTermsEnumTests.java @@ -35,6 +35,7 @@ import java.util.HashMap; import java.util.Locale; import java.util.Map; +import java.util.Map.Entry; public class MultiShardTermsEnumTests extends ESTestCase { @@ -89,7 +90,7 @@ public void testRandomIndexFusion() throws Exception { // Simulate fields like constant-keyword which use a SimpleTermCountEnum to present results // rather than the raw TermsEnum from Lucene. ArrayList termCounts = new ArrayList<>(); - while(te.next()!=null) { + while (te.next() != null) { termCounts.add(new TermCount(te.term().utf8ToString(), te.docFreq())); } SimpleTermCountEnum simpleEnum = new SimpleTermCountEnum(termCounts.toArray(new TermCount[0])); @@ -101,10 +102,10 @@ public void testRandomIndexFusion() throws Exception { MultiShardTermsEnum mte = new MultiShardTermsEnum(termsEnums.toArray(new TermsEnum[0])); HashMap expecteds = new HashMap<>(); - for (String term : globalTermCounts.keySet()) { - if(term.startsWith(searchPrefix)) { - expecteds.put(term, globalTermCounts.get(term)); - } + for (Entry termCount : globalTermCounts.entrySet()) { + if (termCount.getKey().startsWith(searchPrefix)) { + expecteds.put(termCount.getKey(), termCount.getValue()); + } } while (mte.next() != null) { diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index 5e16917876335..d97287c3d5922 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -155,7 +155,7 @@ public TermsEnum getTerms(boolean caseInsensitive, String string, SearchExecutio if (matches == false) { return null; } - int docCount = queryShardContext.searcher().getIndexReader().numDocs(); + int docCount = queryShardContext.searcher().getIndexReader().maxDoc(); return new SimpleTermCountEnum(new TermCount(value, docCount)); } From 3288641ae857d9aae7096f704ade6fdd9eac53e8 Mon Sep 17 00:00:00 2001 From: markharwood Date: Wed, 14 Apr 2021 11:20:30 +0100 Subject: [PATCH 18/31] Docs tidy up --- docs/reference/search/term-enum.asciidoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/search/term-enum.asciidoc b/docs/reference/search/term-enum.asciidoc index fa9e09056b4bd..54b00cdd4ec9a 100644 --- a/docs/reference/search/term-enum.asciidoc +++ b/docs/reference/search/term-enum.asciidoc @@ -2,7 +2,7 @@ === Term enum API The term enum API can be used to discover terms in the index that match -a partial string. This is used for features like auto-complete: +a partial string. This is used for auto-complete: [source,console] -------------------------------------------------- From 2c0096859600a7e8a8a3f36a92ab5a205dcd8dd2 Mon Sep 17 00:00:00 2001 From: markharwood Date: Wed, 14 Apr 2021 16:50:09 +0100 Subject: [PATCH 19/31] Provide full stack traces for errors, change TODO comment --- .../xpack/core/termenum/action/TransportTermEnumAction.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java index d935cdbc40cec..a318f5d2a1de3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java @@ -9,6 +9,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; +import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRunnable; import org.elasticsearch.action.support.ActionFilters; @@ -322,14 +323,15 @@ protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Ta } } catch (Exception e) { - error = e.getMessage(); + error = ExceptionsHelper.stackTrace(e); } finally { IOUtils.close(openedResources); } return new NodeTermEnumResponse(request.nodeId(), termsList, error, true); } - // TODO remove this so we can shift code to server module - see https://github.com/elastic/elasticsearch/issues/70221 + // TODO remove this so we can shift code to server module - write a separate Interceptor class to rewrite requests according to + // security rules (only allowing access to shards where role query rewrites to a match_all private boolean canAccess(String indexName, String fieldName, XPackLicenseState frozenLicenseState, ThreadContext threadContext) { if (frozenLicenseState.isSecurityEnabled()) { var licenseChecker = new MemoizedSupplier<>(() -> frozenLicenseState.checkFeature(Feature.SECURITY_DLS_FLS)); From 6a68b70631cf226f2130069d4a369b50f3c7f01a Mon Sep 17 00:00:00 2001 From: markharwood Date: Mon, 19 Apr 2021 11:15:57 +0100 Subject: [PATCH 20/31] Move location of YAML test - was causing errors when seated alongside core/src/yamlRestTest --- .../resources/rest-api-spec/test/term_enum/10_basic.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename x-pack/plugin/{core => }/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml (100%) diff --git a/x-pack/plugin/core/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml similarity index 100% rename from x-pack/plugin/core/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml From 1fe0a11b41924d69a2fbea9e2d1b19c0cd3ff8b6 Mon Sep 17 00:00:00 2001 From: markharwood Date: Fri, 23 Apr 2021 11:06:07 +0100 Subject: [PATCH 21/31] Security enhancement - allow access where DLS rewrites to match_all. Added security tests --- .../action/TransportTermEnumAction.java | 67 +++- .../rest-api-spec/test/term_enum/10_basic.yml | 295 +++++++++++++++--- 2 files changed, 305 insertions(+), 57 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java index a318f5d2a1de3..b42b6b7b08aed 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java @@ -8,6 +8,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.TermsEnum; +import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; @@ -27,6 +28,7 @@ import org.elasticsearch.cluster.routing.ShardRouting; import org.elasticsearch.cluster.service.ClusterService; import org.elasticsearch.common.MemoizedSupplier; +import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.util.concurrent.ThreadContext; @@ -35,12 +37,16 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.MatchAllQueryBuilder; +import org.elasticsearch.index.query.MatchNoneQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.index.shard.IndexShard; import org.elasticsearch.index.shard.ShardId; import org.elasticsearch.indices.IndicesService; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.license.XPackLicenseState.Feature; +import org.elasticsearch.script.ScriptService; import org.elasticsearch.search.SearchService; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.internal.AliasFilter; @@ -52,8 +58,10 @@ import org.elasticsearch.transport.TransportRequestHandler; import org.elasticsearch.transport.TransportResponseHandler; import org.elasticsearch.transport.TransportService; +import org.elasticsearch.xpack.core.security.SecurityContext; import org.elasticsearch.xpack.core.security.authz.AuthorizationServiceField; import org.elasticsearch.xpack.core.security.authz.accesscontrol.IndicesAccessControl; +import org.elasticsearch.xpack.core.security.authz.support.DLSRoleQueryValidator; import java.io.Closeable; import java.io.IOException; @@ -76,6 +84,7 @@ public class TransportTermEnumAction extends HandledTransportAction(() -> frozenLicenseState.checkFeature(Feature.SECURITY_DLS_FLS)); IndicesAccessControl indicesAccessControl = threadContext.getTransient(AuthorizationServiceField.INDICES_PERMISSIONS_KEY); - IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(indexName); + IndicesAccessControl.IndexAccessControl indexAccessControl = indicesAccessControl.getIndexPermissions(shardId.getIndexName()); + + if (indexAccessControl != null) { final boolean dls = indexAccessControl.getDocumentPermissions().hasDocumentLevelPermissions(); if ( dls && licenseChecker.get()) { + // Check to see if any of the roles defined for the current user rewrite to match_all + + SecurityContext securityContext = new SecurityContext(clusterService.getSettings(), threadContext); + final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); + final IndexShard indexShard = indexService.getShard(shardId.getId()); + ArrayList openedResources = new ArrayList<>(); + try { + + Engine.Searcher searcher = indexShard.acquireSearcher(Engine.SEARCH_SOURCE); + openedResources.add(searcher); + final SearchExecutionContext queryShardContext = indexService.newSearchExecutionContext( + shardId.id(), + 0, + searcher, + request::shardStartedTimeMillis, + null, + Collections.emptyMap() + ); + + // Current user has potentially many roles and therefore potentially many queries + // defining sets of docs accessible + Set queries = indexAccessControl.getDocumentPermissions().getQueries(); + for (BytesReference querySource : queries) { + QueryBuilder queryBuilder = DLSRoleQueryValidator.evaluateAndVerifyRoleQuery( + querySource, + scriptService, + queryShardContext.getXContentRegistry(), + securityContext.getUser() + ); + QueryBuilder rewrittenQueryBuilder = Rewriteable.rewrite(queryBuilder, queryShardContext); + if (rewrittenQueryBuilder instanceof MatchAllQueryBuilder) { + // One of the roles assigned has "all" permissions - allow unfettered access to termsDict + return true; + } + + } + } finally { + IOUtils.close(openedResources); + } return false; } } @@ -535,7 +592,7 @@ private void asyncNodeOperation(NodeTermEnumRequest request, Task task, ActionLi ThreadContext threadContext = transportService.getThreadPool().getThreadContext(); final XPackLicenseState frozenLicenseState = licenseState.copyCurrentLicenseState(); for (ShardId shardId : request.shardIds().toArray(new ShardId[0])) { - if (canAccess(shardId.getIndexName(), request.field(), frozenLicenseState, threadContext) == false || canMatchShard( + if (canAccess(shardId, request, frozenLicenseState, threadContext) == false || canMatchShard( shardId, request ) == false) { diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml index bbee32d01de57..6d9ed51f5f9cf 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml @@ -1,53 +1,158 @@ --- setup: -- do: - indices.create: - index: test_k - body: - settings: - index: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - foo: - type : keyword - timestamp: - type : date -- do: - indices.create: - index: test_ck - body: - settings: - index: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - foo: - type : constant_keyword - value: bar_ck - other: - type : text - timestamp: - type : date -- do: - indices.create: - index: test_f - body: - settings: - index: - number_of_shards: 1 - number_of_replicas: 0 - mappings: - properties: - foo: - type : flattened - timestamp: - type : date + - skip: + features: headers + - do: + cluster.health: + wait_for_status: yellow + + - do: + security.put_role: + name: "test_admin_role" + body: > + { + "indices": [ + { "names": ["*"], "privileges": ["all"] } + ] + } + + - do: + security.put_user: + username: "test_admin" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "test_admin_role" ], + "full_name" : "user with full privileges to multiple indices" + } + + - do: + security.put_role: + name: "dls_all_role" + body: > + { + "indices": [ + { "names": ["test_security"], "privileges": ["read"], "query": "{\"term\": {\"ck\": \"const\"}}" } + ] + } + + - do: + security.put_role: + name: "dls_none_role" + body: > + { + "indices": [ + { "names": ["test_security"], "privileges": ["read"], "query": "{\"term\": {\"foo\": \"does_not_exist\"}}" } + ] + } + - do: + security.put_user: + username: "dls_all_user" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "dls_none_role", "dls_all_role" ], + "full_name" : "user with access to all docs in test_security index (using DLS)" + } + + - do: + security.put_role: + name: "dls_some_role" + body: > + { + "indices": [ + { "names": ["test_security"], "privileges": ["read"], "query": "{\"term\": {\"foo\": \"bar_dls\"}}" } + ] + } + - do: + security.put_user: + username: "dls_some_user" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "dls_some_role" ], + "full_name" : "user with access to selected docs in index" + } + - do: + security.put_role: + name: "fls_role" + body: > + { + "indices": [ + { "names": ["test_security"], "privileges": ["read"], "field_security" : {"grant" : [ "*"],"except": [ "foo" ]} } + ] + } + + - do: + security.put_user: + username: "fls_user" + body: > + { + "password" : "x-pack-test-password", + "roles" : [ "fls_role" ], + "full_name" : "user with access to selected docs in index" + } + - do: + indices.create: + index: test_k + body: + settings: + index: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + foo: + type : keyword + timestamp: + type : date + - do: + indices.create: + index: test_ck + body: + settings: + index: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + foo: + type : constant_keyword + value: bar_ck + other: + type : text + timestamp: + type : date + - do: + indices.create: + index: test_f + body: + settings: + index: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + foo: + type : flattened + timestamp: + type : date + - do: + indices.create: + index: test_security + body: + settings: + index: + number_of_shards: 1 + number_of_replicas: 0 + mappings: + properties: + ck: + type : constant_keyword + value: const + foo: + type : keyword ---- -"Test basic term enumeration": - do: index: index: test_k @@ -65,20 +170,72 @@ setup: index: test_f id: 3 body: { foo: { bar: "bar_f" }, "timestamp":"2019-01-01T01:01:01.000Z" } - - do: + index: + index: test_security + id: 4 + body: { foo: "bar_dls"} + + - do: #superuser + headers: { Authorization: "Basic dGVzdF9hZG1pbjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # admin indices.refresh: {} + - do: #superuser + cluster.health: + index: test_f + wait_for_status: green + +--- +teardown: - do: - cluster.health: - index: test_f - wait_for_status: green + security.delete_user: + username: "dls_all_user" + ignore: 404 + + - do: + security.delete_role: + name: "dls_all_role" + ignore: 404 + - do: + security.delete_role: + name: "dls_none_role" + ignore: 404 + + - do: + security.delete_user: + username: "dls_some_user" + ignore: 404 + - do: + security.delete_role: + name: "dls_some_role" + ignore: 404 + - do: + security.delete_user: + username: "fls_user" + ignore: 404 + + - do: + security.delete_role: + name: "fls_role" + ignore: 404 + - do: + security.delete_user: + username: "test_admin" + ignore: 404 + + - do: + security.delete_role: + name: "test_admin_role" + ignore: 404 + +--- +"Test basic term enumeration": - do: termsenum: index: test_* body: {"field": "foo", "string":"b"} - - length: {terms: 2} + - length: {terms: 3} - do: @@ -87,8 +244,42 @@ setup: body: {"field": "foo.bar", "string":"b"} - length: {terms: 1} +--- +"Test index filtering": - do: termsenum: index: test_* body: {"field": "foo", "string":"b", "index_filter":{"range":{"timestamp":{"gte":"2021-01-01T01:01:01.000Z"}}}} - length: {terms: 1} +--- +"Test security": + + - do: + headers: { Authorization: "Basic dGVzdF9hZG1pbjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # admin_user sees all docs + termsenum: + index: test_security + body: {"field": "foo", "string":"b"} + - length: {terms: 1} + + - do: + headers: { Authorization: "Basic ZGxzX2FsbF91c2VyOngtcGFjay10ZXN0LXBhc3N3b3Jk" } # dls_all_user sees all docs + termsenum: + index: test_security + body: {"field": "foo", "string":"b"} + - length: {terms: 1} + + - do: + headers: { Authorization: "Basic ZGxzX3NvbWVfdXNlcjp4LXBhY2stdGVzdC1wYXNzd29yZA==" } # dls_some_user sees selected docs + termsenum: + index: test_security + body: {"field": "foo", "string":"b"} + - length: {terms: 0} + + - do: + headers: { Authorization: "Basic ZmxzX3VzZXI6eC1wYWNrLXRlc3QtcGFzc3dvcmQ=" } # fls_user can't see field + termsenum: + index: test_security + body: {"field": "foo", "string":"b"} + - length: {terms: 0} + + From 2f598607ac112be5158e0c8654810fbdff006045 Mon Sep 17 00:00:00 2001 From: markharwood Date: Fri, 23 Apr 2021 11:24:34 +0100 Subject: [PATCH 22/31] Remove acquisition of searcher from security check code --- .../action/TransportTermEnumAction.java | 58 ++++++++----------- 1 file changed, 23 insertions(+), 35 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java index b42b6b7b08aed..1efcb2b742201 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java @@ -8,7 +8,6 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.TermsEnum; -import org.apache.lucene.search.BooleanQuery; import org.apache.lucene.util.BytesRef; import org.elasticsearch.ExceptionsHelper; import org.elasticsearch.action.ActionListener; @@ -37,7 +36,6 @@ import org.elasticsearch.index.engine.Engine; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.query.MatchAllQueryBuilder; -import org.elasticsearch.index.query.MatchNoneQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.Rewriteable; import org.elasticsearch.index.query.SearchExecutionContext; @@ -362,41 +360,31 @@ private boolean canAccess( SecurityContext securityContext = new SecurityContext(clusterService.getSettings(), threadContext); final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); - final IndexShard indexShard = indexService.getShard(shardId.getId()); - ArrayList openedResources = new ArrayList<>(); - try { - - Engine.Searcher searcher = indexShard.acquireSearcher(Engine.SEARCH_SOURCE); - openedResources.add(searcher); - final SearchExecutionContext queryShardContext = indexService.newSearchExecutionContext( - shardId.id(), - 0, - searcher, - request::shardStartedTimeMillis, - null, - Collections.emptyMap() - ); - - // Current user has potentially many roles and therefore potentially many queries - // defining sets of docs accessible - Set queries = indexAccessControl.getDocumentPermissions().getQueries(); - for (BytesReference querySource : queries) { - QueryBuilder queryBuilder = DLSRoleQueryValidator.evaluateAndVerifyRoleQuery( - querySource, - scriptService, - queryShardContext.getXContentRegistry(), - securityContext.getUser() - ); - QueryBuilder rewrittenQueryBuilder = Rewriteable.rewrite(queryBuilder, queryShardContext); - if (rewrittenQueryBuilder instanceof MatchAllQueryBuilder) { - // One of the roles assigned has "all" permissions - allow unfettered access to termsDict - return true; - } + final SearchExecutionContext queryShardContext = indexService.newSearchExecutionContext( + shardId.id(), + 0, + null, + request::shardStartedTimeMillis, + null, + Collections.emptyMap() + ); + // Current user has potentially many roles and therefore potentially many queries + // defining sets of docs accessible + Set queries = indexAccessControl.getDocumentPermissions().getQueries(); + for (BytesReference querySource : queries) { + QueryBuilder queryBuilder = DLSRoleQueryValidator.evaluateAndVerifyRoleQuery( + querySource, + scriptService, + queryShardContext.getXContentRegistry(), + securityContext.getUser() + ); + QueryBuilder rewrittenQueryBuilder = Rewriteable.rewrite(queryBuilder, queryShardContext); + if (rewrittenQueryBuilder instanceof MatchAllQueryBuilder) { + // One of the roles assigned has "all" permissions - allow unfettered access to termsDict + return true; } - } finally { - IOUtils.close(openedResources); - } + } return false; } } From 4c38b78005934cbd8c6b83e4ba65d74b98c6fd62 Mon Sep 17 00:00:00 2001 From: markharwood Date: Mon, 26 Apr 2021 15:46:53 +0100 Subject: [PATCH 23/31] Changed termenum to termsenum. REST endpoint is now _terms_enum --- ...term-enum.asciidoc => terms-enum.asciidoc} | 32 ++++---- .../rest-api-spec/api/termsenum.json | 6 +- .../xpack/core/XPackClientPlugin.java | 6 +- .../elasticsearch/xpack/core/XPackPlugin.java | 10 +-- .../core/termenum/action/TermEnumAction.java | 45 ----------- .../action/MultiShardTermsEnum.java | 2 +- .../action/NodeTermsEnumRequest.java} | 10 +-- .../action/NodeTermsEnumResponse.java} | 10 +-- .../action/SimpleTermCountEnum.java | 2 +- .../action/TermCount.java | 2 +- .../termsenum/action/TermsEnumAction.java | 45 +++++++++++ .../action/TermsEnumRequest.java} | 10 +-- .../action/TermsEnumRequestBuilder.java} | 14 ++-- .../action/TermsEnumResponse.java} | 16 ++-- .../action/TransportTermsEnumAction.java} | 74 +++++++++---------- .../action/package-info.java | 2 +- .../rest/RestTermsEnumAction.java} | 16 ++-- .../MultiShardTermsEnumTests.java | 8 +- .../TermCountTests.java | 4 +- .../TermsEnumResponseTests.java} | 24 +++--- .../TransportTermsEnumActionTests.java} | 16 ++-- .../action/RestTermsEnumActionTests.java} | 19 ++--- .../mapper/ConstantKeywordFieldMapper.java | 4 +- .../{term_enum => terms_enum}/10_basic.yml | 0 24 files changed, 189 insertions(+), 188 deletions(-) rename docs/reference/search/{term-enum.asciidoc => terms-enum.asciidoc} (76%) delete mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumAction.java rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{termenum => termsenum}/action/MultiShardTermsEnum.java (98%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{termenum/action/NodeTermEnumRequest.java => termsenum/action/NodeTermsEnumRequest.java} (91%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{termenum/action/NodeTermEnumResponse.java => termsenum/action/NodeTermsEnumResponse.java} (79%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{termenum => termsenum}/action/SimpleTermCountEnum.java (98%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{termenum => termsenum}/action/TermCount.java (98%) create mode 100644 x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumAction.java rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{termenum/action/TermEnumRequest.java => termsenum/action/TermsEnumRequest.java} (94%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{termenum/action/TermEnumRequestBuilder.java => termsenum/action/TermsEnumRequestBuilder.java} (52%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{termenum/action/TermEnumResponse.java => termsenum/action/TermsEnumResponse.java} (87%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{termenum/action/TransportTermEnumAction.java => termsenum/action/TransportTermsEnumAction.java} (89%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{termenum => termsenum}/action/package-info.java (84%) rename x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/{termenum/rest/RestTermEnumAction.java => termsenum/rest/RestTermsEnumAction.java} (69%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/{termenum => termsenum}/MultiShardTermsEnumTests.java (95%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/{termenum => termsenum}/TermCountTests.java (91%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/{termenum/TermEnumResponseTests.java => termsenum/TermsEnumResponseTests.java} (70%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/{termenum/TransportTermEnumActionTests.java => termsenum/TransportTermsEnumActionTests.java} (64%) rename x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/{termenum/action/RestTermEnumActionTests.java => termsenum/action/RestTermsEnumActionTests.java} (90%) rename x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/{term_enum => terms_enum}/10_basic.yml (100%) diff --git a/docs/reference/search/term-enum.asciidoc b/docs/reference/search/terms-enum.asciidoc similarity index 76% rename from docs/reference/search/term-enum.asciidoc rename to docs/reference/search/terms-enum.asciidoc index 54b00cdd4ec9a..c555806679f92 100644 --- a/docs/reference/search/term-enum.asciidoc +++ b/docs/reference/search/terms-enum.asciidoc @@ -1,12 +1,12 @@ -[[search-term-enum]] -=== Term enum API +[[search-terms-enum]] +=== Terms enum API -The term enum API can be used to discover terms in the index that match +The terms enum API can be used to discover terms in the index that match a partial string. This is used for auto-complete: [source,console] -------------------------------------------------- -POST stackoverflow/_terms +POST stackoverflow/_terms_enum { "field" : "tags", "string" : "kiba" @@ -35,20 +35,20 @@ The API returns the following response: The "complete" flag is false if time or space constraints were met and the set of terms examined was not the full set of available values. -[[search-term-enum-api-request]] +[[search-terms-enum-api-request]] ==== {api-request-title} -`GET //_terms` +`GET //_terms_enum` -[[search-term-enum-api-desc]] +[[search-terms-enum-api-desc]] ==== {api-description-title} -The termenum API can be used to discover terms in the index that begin with the provided +The termsenum API can be used to discover terms in the index that begin with the provided string. It is designed for low-latency look-ups used in auto-complete scenarios. -[[search-term-enum-api-path-params]] +[[search-terms-enum-api-path-params]] ==== {api-path-parms-title} ``:: @@ -59,38 +59,38 @@ Wildcard (`*`) expressions are supported. To search all data streams or indices in a cluster, omit this parameter or use `_all` or `*`. -[[search-term-enum-api-request-body]] +[[search-terms-enum-api-request-body]] ==== {api-request-body-title} -[[term-enum-field-param]] +[[terms-enum-field-param]] `field`:: (Mandatory, string) Which field to match -[[term-enum-string-param]] +[[terms-enum-string-param]] `string`:: (Mandatory, string) The string to match at the start of indexed terms -[[term-enum-size-param]] +[[terms-enum-size-param]] `size`:: (Optional, integer) How many matching terms to return. Defaults to 10 -[[term-enum-timeout-param]] +[[terms-enum-timeout-param]] `timeout`:: (Optional, integer) The maximum length of time in milliseconds to spend collecting results. Defaults to 1000. If the timeout is exceeded the `complete` flag set to false in the response and the results may be partial or empty. -[[term-enum-case_insensitive-param]] +[[terms-enum-case_insensitive-param]] `case_insensitive`:: (Optional, boolean) When true the provided search string is matched against index terms without case sensitivity. Defaults to false. -[[term-enum-index_filter-param]] +[[terms-enum-index_filter-param]] `index_filter`:: (Optional, <> Allows to filter an index shard if the provided query rewrites to `match_none`. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/termsenum.json b/rest-api-spec/src/main/resources/rest-api-spec/api/termsenum.json index aaf0624fa5719..0d790c8b1bcad 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/termsenum.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/termsenum.json @@ -1,8 +1,8 @@ { "termsenum":{ "documentation":{ - "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/term-enum.html", - "description": "The termenum API can be used to discover terms in the index that begin with the provided string. It is designed for low-latency look-ups used in auto-complete scenarios." + "url":"https://www.elastic.co/guide/en/elasticsearch/reference/current/terms-enum.html", + "description": "The terms enum API can be used to discover terms in the index that begin with the provided string. It is designed for low-latency look-ups used in auto-complete scenarios." }, "stability":"beta", "visibility":"public", @@ -13,7 +13,7 @@ "url":{ "paths":[ { - "path": "/{index}/_terms", + "path": "/{index}/_terms_enum", "methods": [ "GET", "POST" diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java index 9f58637c0262a..4d3712f324df0 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackClientPlugin.java @@ -200,8 +200,8 @@ import org.elasticsearch.xpack.core.spatial.SpatialFeatureSetUsage; import org.elasticsearch.xpack.core.sql.SqlFeatureSetUsage; import org.elasticsearch.xpack.core.ssl.action.GetCertificateInfoAction; +import org.elasticsearch.xpack.core.termsenum.action.TermsEnumAction; import org.elasticsearch.xpack.core.textstructure.action.FindStructureAction; -import org.elasticsearch.xpack.core.termenum.action.TermEnumAction; import org.elasticsearch.xpack.core.transform.TransformFeatureSetUsage; import org.elasticsearch.xpack.core.transform.TransformField; import org.elasticsearch.xpack.core.transform.TransformMetadata; @@ -415,8 +415,8 @@ public List> getClientActions() { DeleteAsyncResultAction.INSTANCE, // Text Structure FindStructureAction.INSTANCE, - // Termenum API - TermEnumAction.INSTANCE + // Terms enum API + TermsEnumAction.INSTANCE )); // rollupV2 diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java index 6cc0e092ea552..e990b29845b18 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/XPackPlugin.java @@ -83,9 +83,9 @@ import org.elasticsearch.xpack.core.ssl.SSLConfigurationReloader; import org.elasticsearch.xpack.core.ssl.SSLService; import org.elasticsearch.xpack.core.transform.TransformMetadata; -import org.elasticsearch.xpack.core.termenum.action.TermEnumAction; -import org.elasticsearch.xpack.core.termenum.action.TransportTermEnumAction; -import org.elasticsearch.xpack.core.termenum.rest.RestTermEnumAction; +import org.elasticsearch.xpack.core.termsenum.action.TermsEnumAction; +import org.elasticsearch.xpack.core.termsenum.action.TransportTermsEnumAction; +import org.elasticsearch.xpack.core.termsenum.rest.RestTermsEnumAction; import org.elasticsearch.xpack.core.watcher.WatcherMetadata; import org.elasticsearch.xpack.searchablesnapshots.SearchableSnapshotsConstants; @@ -294,7 +294,7 @@ public Collection createComponents(Client client, ClusterService cluster actions.add(new ActionHandler<>(XPackUsageAction.INSTANCE, getUsageAction())); actions.addAll(licensing.getActions()); actions.add(new ActionHandler<>(ReloadAnalyzerAction.INSTANCE, TransportReloadAnalyzersAction.class)); - actions.add(new ActionHandler<>(TermEnumAction.INSTANCE, TransportTermEnumAction.class)); + actions.add(new ActionHandler<>(TermsEnumAction.INSTANCE, TransportTermsEnumAction.class)); actions.add(new ActionHandler<>(DeleteAsyncResultAction.INSTANCE, TransportDeleteAsyncResultAction.class)); actions.add(new ActionHandler<>(XPackInfoFeatureAction.DATA_TIERS, DataTiersInfoTransportAction.class)); actions.add(new ActionHandler<>(XPackUsageFeatureAction.DATA_TIERS, DataTiersUsageTransportAction.class)); @@ -334,7 +334,7 @@ public List getRestHandlers(Settings settings, RestController restC handlers.add(new RestXPackInfoAction()); handlers.add(new RestXPackUsageAction()); handlers.add(new RestReloadAnalyzersAction()); - handlers.add(new RestTermEnumAction()); + handlers.add(new RestTermsEnumAction()); handlers.addAll(licensing.getRestHandlers(settings, restController, clusterSettings, indexScopedSettings, settingsFilter, indexNameExpressionResolver, nodesInCluster)); return handlers; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumAction.java deleted file mode 100644 index 872523caaf670..0000000000000 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumAction.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.core.termenum.action; - -import org.elasticsearch.action.ActionType; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.XContentParser; - -import java.io.IOException; - -import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; - -public class TermEnumAction extends ActionType { - - public static final TermEnumAction INSTANCE = new TermEnumAction(); - public static final String NAME = "indices:data/read/xpack/termsenum/list"; - - - static final ParseField INDEX_FILTER = new ParseField("index_filter"); - - private TermEnumAction() { - super(NAME, TermEnumResponse::new); - } - - public static TermEnumRequest fromXContent(XContentParser parser, String... indices) throws IOException { - TermEnumRequest request = new TermEnumRequest(indices); - PARSER.parse(parser, request, null); - return request; - } - - private static final ObjectParser PARSER = new ObjectParser<>("terms_enum_request"); - static { - PARSER.declareString(TermEnumRequest::field, new ParseField("field")); - PARSER.declareString(TermEnumRequest::string, new ParseField("string")); - PARSER.declareInt(TermEnumRequest::size, new ParseField("size")); - PARSER.declareBoolean(TermEnumRequest::caseInsensitive, new ParseField("case_insensitive")); - PARSER.declareInt(TermEnumRequest::timeoutInMillis, new ParseField("timeout")); - PARSER.declareObject(TermEnumRequest::indexFilter, (p, context) -> parseInnerQueryBuilder(p),INDEX_FILTER); - } -} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/MultiShardTermsEnum.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/MultiShardTermsEnum.java similarity index 98% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/MultiShardTermsEnum.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/MultiShardTermsEnum.java index 43461102b4e6c..2a7794df124e6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/MultiShardTermsEnum.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/MultiShardTermsEnum.java @@ -14,7 +14,7 @@ * See the license for the specific language governing permissions and * limitations under the license. */ -package org.elasticsearch.xpack.core.termenum.action; +package org.elasticsearch.xpack.core.termsenum.action; import org.apache.lucene.index.TermsEnum; import org.apache.lucene.util.BytesRef; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java similarity index 91% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java index f2017f798f047..1abb71a1da713 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum.action; +package org.elasticsearch.xpack.core.termsenum.action; import org.elasticsearch.action.IndicesRequest; import org.elasticsearch.action.support.IndicesOptions; @@ -20,10 +20,10 @@ import java.util.Set; /** - * Internal termenum request executed directly against a specific node, querying potentially many + * Internal terms enum request executed directly against a specific node, querying potentially many * shards in one request */ -public class NodeTermEnumRequest extends TransportRequest implements IndicesRequest { +public class NodeTermsEnumRequest extends TransportRequest implements IndicesRequest { private String field; private String string; @@ -37,7 +37,7 @@ public class NodeTermEnumRequest extends TransportRequest implements IndicesRequ private String nodeId; - public NodeTermEnumRequest(StreamInput in) throws IOException { + public NodeTermsEnumRequest(StreamInput in) throws IOException { super(in); // Set the clock running as soon as we appear on a node. nodeStartedTimeMillis = System.currentTimeMillis(); @@ -57,7 +57,7 @@ public NodeTermEnumRequest(StreamInput in) throws IOException { } } - public NodeTermEnumRequest(final String nodeId, final Set shardIds, TermEnumRequest request) { + public NodeTermsEnumRequest(final String nodeId, final Set shardIds, TermsEnumRequest request) { this.field = request.field(); this.string = request.string(); this.caseInsensitive = request.caseInsensitive(); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumResponse.java similarity index 79% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumResponse.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumResponse.java index 7774b8c6e6bf7..728a6504c0eb3 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/NodeTermEnumResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumResponse.java @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum.action; +package org.elasticsearch.xpack.core.termsenum.action; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; @@ -14,11 +14,11 @@ import java.util.List; /** - * Internal response of a term enum request executed directly against a specific shard. + * Internal response of a terms enum request executed directly against a specific shard. * * */ -class NodeTermEnumResponse extends TransportResponse { +class NodeTermsEnumResponse extends TransportResponse { private String error; private boolean complete; @@ -26,7 +26,7 @@ class NodeTermEnumResponse extends TransportResponse { private List terms; private String nodeId; - NodeTermEnumResponse(StreamInput in) throws IOException { + NodeTermsEnumResponse(StreamInput in) throws IOException { super(in); terms = in.readList(TermCount::new); error = in.readOptionalString(); @@ -34,7 +34,7 @@ class NodeTermEnumResponse extends TransportResponse { nodeId = in.readString(); } - NodeTermEnumResponse(String nodeId, List terms, String error, boolean complete) { + NodeTermsEnumResponse(String nodeId, List terms, String error, boolean complete) { this.nodeId = nodeId; this.terms = terms; this.error = error; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/SimpleTermCountEnum.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/SimpleTermCountEnum.java similarity index 98% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/SimpleTermCountEnum.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/SimpleTermCountEnum.java index 5c1a1b7dc63eb..8fb2ae1b8081f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/SimpleTermCountEnum.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/SimpleTermCountEnum.java @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum.action; +package org.elasticsearch.xpack.core.termsenum.action; import org.apache.lucene.index.ImpactsEnum; import org.apache.lucene.index.PostingsEnum; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermCount.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermCount.java similarity index 98% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermCount.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermCount.java index 2c6c0437d229c..32083c995b519 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermCount.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermCount.java @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum.action; +package org.elasticsearch.xpack.core.termsenum.action; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.io.stream.StreamInput; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumAction.java new file mode 100644 index 0000000000000..d21c1157276f7 --- /dev/null +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumAction.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.core.termsenum.action; + +import org.elasticsearch.action.ActionType; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; + +import java.io.IOException; + +import static org.elasticsearch.index.query.AbstractQueryBuilder.parseInnerQueryBuilder; + +public class TermsEnumAction extends ActionType { + + public static final TermsEnumAction INSTANCE = new TermsEnumAction(); + public static final String NAME = "indices:data/read/xpack/termsenum/list"; + + + static final ParseField INDEX_FILTER = new ParseField("index_filter"); + + private TermsEnumAction() { + super(NAME, TermsEnumResponse::new); + } + + public static TermsEnumRequest fromXContent(XContentParser parser, String... indices) throws IOException { + TermsEnumRequest request = new TermsEnumRequest(indices); + PARSER.parse(parser, request, null); + return request; + } + + private static final ObjectParser PARSER = new ObjectParser<>("terms_enum_request"); + static { + PARSER.declareString(TermsEnumRequest::field, new ParseField("field")); + PARSER.declareString(TermsEnumRequest::string, new ParseField("string")); + PARSER.declareInt(TermsEnumRequest::size, new ParseField("size")); + PARSER.declareBoolean(TermsEnumRequest::caseInsensitive, new ParseField("case_insensitive")); + PARSER.declareInt(TermsEnumRequest::timeoutInMillis, new ParseField("timeout")); + PARSER.declareObject(TermsEnumRequest::indexFilter, (p, context) -> parseInnerQueryBuilder(p),INDEX_FILTER); + } +} diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java similarity index 94% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java index 31d349f1a8cc8..6d4b9de8791e4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum.action; +package org.elasticsearch.xpack.core.termsenum.action; import org.elasticsearch.action.ActionRequestValidationException; import org.elasticsearch.action.ValidateActions; @@ -24,7 +24,7 @@ /** * A request to gather terms for a given field matching a string prefix */ -public class TermEnumRequest extends BroadcastRequest implements ToXContentObject { +public class TermsEnumRequest extends BroadcastRequest implements ToXContentObject { public static int DEFAULT_SIZE = 10; public static int DEFAULT_TIMEOUT_MILLIS = 1000; @@ -36,11 +36,11 @@ public class TermEnumRequest extends BroadcastRequest implement long taskStartTimeMillis; private QueryBuilder indexFilter; - public TermEnumRequest() { + public TermsEnumRequest() { this(Strings.EMPTY_ARRAY); } - public TermEnumRequest(StreamInput in) throws IOException { + public TermsEnumRequest(StreamInput in) throws IOException { super(in); field = in.readString(); string = in.readString(); @@ -53,7 +53,7 @@ public TermEnumRequest(StreamInput in) throws IOException { * Constructs a new term enum request against the provided indices. No indices provided means it will * run against all indices. */ - public TermEnumRequest(String... indices) { + public TermsEnumRequest(String... indices) { super(indices); indicesOptions(IndicesOptions.fromOptions(false, false, true, false)); timeout(new TimeValue(DEFAULT_TIMEOUT_MILLIS)); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequestBuilder.java similarity index 52% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequestBuilder.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequestBuilder.java index 995047fb6cc37..573162b17b758 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumRequestBuilder.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequestBuilder.java @@ -4,28 +4,28 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum.action; +package org.elasticsearch.xpack.core.termsenum.action; import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -public class TermEnumRequestBuilder extends BroadcastOperationRequestBuilder { +public class TermsEnumRequestBuilder extends BroadcastOperationRequestBuilder { - public TermEnumRequestBuilder(ElasticsearchClient client, TermEnumAction action) { - super(client, action, new TermEnumRequest()); + public TermsEnumRequestBuilder(ElasticsearchClient client, TermsEnumAction action) { + super(client, action, new TermsEnumRequest()); } - public TermEnumRequestBuilder setField(String field) { + public TermsEnumRequestBuilder setField(String field) { request.field(field); return this; } - public TermEnumRequestBuilder setString(String string) { + public TermsEnumRequestBuilder setString(String string) { request.string(string); return this; } - public TermEnumRequestBuilder setSize(int size) { + public TermsEnumRequestBuilder setSize(int size) { request.size(size); return this; } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumResponse.java similarity index 87% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumResponse.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumResponse.java index 61125261e6a87..5bd95a1da8fce 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TermEnumResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumResponse.java @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum.action; +package org.elasticsearch.xpack.core.termsenum.action; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.action.support.broadcast.BroadcastResponse; @@ -23,20 +23,20 @@ import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; /** - * The response of the termenum/list action. + * The response of the _terms_enum action. */ -public class TermEnumResponse extends BroadcastResponse { +public class TermsEnumResponse extends BroadcastResponse { public static final String TERMS_FIELD = "terms"; public static final String COMPLETE_FIELD = "complete"; @SuppressWarnings("unchecked") - static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( + static final ConstructingObjectParser PARSER = new ConstructingObjectParser<>( "term_enum_results", true, arg -> { BroadcastResponse response = (BroadcastResponse) arg[0]; - return new TermEnumResponse( + return new TermsEnumResponse( (List) arg[1], response.getTotalShards(), response.getSuccessfulShards(), @@ -57,14 +57,14 @@ public class TermEnumResponse extends BroadcastResponse { private boolean complete; private int skippedShards; - TermEnumResponse(StreamInput in) throws IOException { + TermsEnumResponse(StreamInput in) throws IOException { super(in); terms = in.readStringList(); complete = in.readBoolean(); skippedShards = in.readVInt(); } - public TermEnumResponse( + public TermsEnumResponse( List terms, int totalShards, int successfulShards, @@ -110,7 +110,7 @@ protected void addCustomXContentFields(XContentBuilder builder, Params params) t builder.field(COMPLETE_FIELD, complete); } - public static TermEnumResponse fromXContent(XContentParser parser) { + public static TermsEnumResponse fromXContent(XContentParser parser) { return PARSER.apply(parser, null); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java similarity index 89% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java index 1efcb2b742201..405cb56400336 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/TransportTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum.action; +package org.elasticsearch.xpack.core.termsenum.action; import org.apache.logging.log4j.message.ParameterizedMessage; import org.apache.lucene.index.TermsEnum; @@ -76,7 +76,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReferenceArray; -public class TransportTermEnumAction extends HandledTransportAction { +public class TransportTermsEnumAction extends HandledTransportAction { protected final ClusterService clusterService; protected final TransportService transportService; @@ -90,7 +90,7 @@ public class TransportTermEnumAction extends HandledTransportAction listener) { + protected void doExecute(Task task, TermsEnumRequest request, ActionListener listener) { request.taskStartTimeMillis = task.getStartTime(); new AsyncBroadcastAction(task, request, listener).start(); } - protected NodeTermEnumRequest newNodeRequest(final String nodeId, final Set shardIds, TermEnumRequest request) { + protected NodeTermsEnumRequest newNodeRequest(final String nodeId, final Set shardIds, TermsEnumRequest request) { // Given we look terms up in the terms dictionary alias filters is another aspect of search (like DLS) that we // currently do not support. // final ClusterState clusterState = clusterService.state(); // final Set indicesAndAliases = indexNameExpressionResolver.resolveExpressions(clusterState, request.indices()); // final AliasFilter aliasFilter = searchService.buildAliasFilter(clusterState, shard.getIndexName(), indicesAndAliases); - return new NodeTermEnumRequest(nodeId, shardIds, request); + return new NodeTermsEnumRequest(nodeId, shardIds, request); } - protected NodeTermEnumResponse readShardResponse(StreamInput in) throws IOException { - return new NodeTermEnumResponse(in); + protected NodeTermsEnumResponse readShardResponse(StreamInput in) throws IOException { + return new NodeTermsEnumResponse(in); } - protected Map> getNodeBundles(ClusterState clusterState, TermEnumRequest request, String[] concreteIndices) { + protected Map> getNodeBundles(ClusterState clusterState, TermsEnumRequest request, String[] concreteIndices) { // Group targeted shards by nodeId Map> fastNodeBundles = new HashMap<>(); for (String indexName : concreteIndices) { @@ -181,16 +181,16 @@ protected Map> getNodeBundles(ClusterState clusterState, Te return fastNodeBundles; } - protected ClusterBlockException checkGlobalBlock(ClusterState state, TermEnumRequest request) { + protected ClusterBlockException checkGlobalBlock(ClusterState state, TermsEnumRequest request) { return state.blocks().globalBlockedException(ClusterBlockLevel.READ); } - protected ClusterBlockException checkRequestBlock(ClusterState state, TermEnumRequest countRequest, String[] concreteIndices) { + protected ClusterBlockException checkRequestBlock(ClusterState state, TermsEnumRequest countRequest, String[] concreteIndices) { return state.blocks().indicesBlockedException(ClusterBlockLevel.READ, concreteIndices); } - protected TermEnumResponse newResponse( - TermEnumRequest request, + protected TermsEnumResponse newResponse( + TermsEnumRequest request, AtomicReferenceArray nodesResponses, boolean complete, Map> nodeBundles @@ -211,7 +211,7 @@ protected TermEnumResponse newResponse( } shardFailures.add(new DefaultShardOperationFailedException((BroadcastShardOperationFailedException) nodeResponse)); } else { - NodeTermEnumResponse str = (NodeTermEnumResponse) nodeResponse; + NodeTermsEnumResponse str = (NodeTermsEnumResponse) nodeResponse; // Only one node response has to be incomplete for the entire result to be labelled incomplete. if (str.getComplete() == false) { complete = false; @@ -264,10 +264,10 @@ public int compare(TermCount t1, TermCount t2) { break; } } - return new TermEnumResponse(terms, (failedShards + successfulShards), successfulShards, failedShards, shardFailures, complete); + return new TermsEnumResponse(terms, (failedShards + successfulShards), successfulShards, failedShards, shardFailures, complete); } - protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Task task) throws IOException { + protected NodeTermsEnumResponse dataNodeOperation(NodeTermsEnumRequest request, Task task) throws IOException { List termsList = new ArrayList<>(); String error = null; @@ -280,7 +280,7 @@ protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Ta for (ShardId shardId : request.shardIds()) { // Check we haven't just arrived on a node and time is up already. if (System.currentTimeMillis() > scheduledEnd) { - return new NodeTermEnumResponse(request.nodeId(), termsList, error, false); + return new NodeTermsEnumResponse(request.nodeId(), termsList, error, false); } final IndexService indexService = indicesService.indexServiceSafe(shardId.getIndex()); final IndexShard indexShard = indexService.getShard(shardId.getId()); @@ -308,7 +308,7 @@ protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Ta int shard_size = request.size(); // All the above prep might take a while - do a timer check now before we continue further. if (System.currentTimeMillis() > scheduledEnd) { - return new NodeTermEnumResponse(request.nodeId(), termsList, error, false); + return new NodeTermsEnumResponse(request.nodeId(), termsList, error, false); } int numTermsBetweenClockChecks = 100; @@ -319,7 +319,7 @@ protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Ta if (termCount > numTermsBetweenClockChecks) { if (System.currentTimeMillis() > scheduledEnd) { boolean complete = te.next() == null; - return new NodeTermEnumResponse(request.nodeId(), termsList, error, complete); + return new NodeTermsEnumResponse(request.nodeId(), termsList, error, complete); } termCount = 0; } @@ -336,14 +336,14 @@ protected NodeTermEnumResponse dataNodeOperation(NodeTermEnumRequest request, Ta } finally { IOUtils.close(openedResources); } - return new NodeTermEnumResponse(request.nodeId(), termsList, error, true); + return new NodeTermsEnumResponse(request.nodeId(), termsList, error, true); } // TODO remove this so we can shift code to server module - write a separate Interceptor class to // rewrite requests according to security rules private boolean canAccess( ShardId shardId, - NodeTermEnumRequest request, + NodeTermsEnumRequest request, XPackLicenseState frozenLicenseState, ThreadContext threadContext ) throws IOException { @@ -392,7 +392,7 @@ private boolean canAccess( return true; } - private boolean canMatchShard(ShardId shardId, NodeTermEnumRequest req) throws IOException { + private boolean canMatchShard(ShardId shardId, NodeTermsEnumRequest req) throws IOException { if (req.indexFilter() == null || req.indexFilter() instanceof MatchAllQueryBuilder) { return true; } @@ -404,16 +404,16 @@ private boolean canMatchShard(ShardId shardId, NodeTermEnumRequest req) throws I protected class AsyncBroadcastAction { private final Task task; - private final TermEnumRequest request; - private ActionListener listener; + private final TermsEnumRequest request; + private ActionListener listener; private final ClusterState clusterState; private final DiscoveryNodes nodes; private final int expectedOps; private final AtomicInteger counterOps = new AtomicInteger(); - private final AtomicReferenceArray nodesResponses; + private final AtomicReferenceArray nodesResponses; private Map> nodeBundles; - protected AsyncBroadcastAction(Task task, TermEnumRequest request, ActionListener listener) { + protected AsyncBroadcastAction(Task task, TermsEnumRequest request, ActionListener listener) { this.task = task; this.request = request; this.listener = listener; @@ -486,7 +486,7 @@ protected void performOperation(final String nodeId, final Set shardIds try { // TODO pass through a reduced timeout (the original time limit, minus whatever we may have // spent already getting to this point. - final NodeTermEnumRequest nodeRequest = newNodeRequest(nodeId, shardIds, request); + final NodeTermsEnumRequest nodeRequest = newNodeRequest(nodeId, shardIds, request); nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId()); DiscoveryNode node = nodes.get(nodeId); if (node == null) { @@ -497,14 +497,14 @@ protected void performOperation(final String nodeId, final Set shardIds node, transportShardAction, nodeRequest, - new TransportResponseHandler() { + new TransportResponseHandler() { @Override - public NodeTermEnumResponse read(StreamInput in) throws IOException { + public NodeTermsEnumResponse read(StreamInput in) throws IOException { return readShardResponse(in); } @Override - public void handleResponse(NodeTermEnumResponse response) { + public void handleResponse(NodeTermsEnumResponse response) { onOperation(nodeId, nodeIndex, response); } @@ -521,7 +521,7 @@ public void handleException(TransportException e) { } } - protected void onOperation(String nodeId, int nodeIndex, NodeTermEnumResponse response) { + protected void onOperation(String nodeId, int nodeIndex, NodeTermsEnumResponse response) { logger.trace("received response for node {}", nodeId); nodesResponses.set(nodeIndex, response); if (expectedOps == counterOps.incrementAndGet()) { @@ -552,10 +552,10 @@ protected synchronized void finishHim(boolean complete) { } } - class NodeTransportHandler implements TransportRequestHandler { + class NodeTransportHandler implements TransportRequestHandler { @Override - public void messageReceived(NodeTermEnumRequest request, TransportChannel channel, Task task) throws Exception { + public void messageReceived(NodeTermsEnumRequest request, TransportChannel channel, Task task) throws Exception { asyncNodeOperation(request, task, ActionListener.wrap(channel::sendResponse, e -> { try { channel.sendResponse(e); @@ -573,7 +573,7 @@ public void messageReceived(NodeTermEnumRequest request, TransportChannel channe } } - private void asyncNodeOperation(NodeTermEnumRequest request, Task task, ActionListener listener) + private void asyncNodeOperation(NodeTermsEnumRequest request, Task task, ActionListener listener) throws IOException { // DLS/FLS check copied from ResizeRequestInterceptor - check permissions and // any index_filter canMatch checks on network thread before allocating work @@ -589,7 +589,7 @@ private void asyncNodeOperation(NodeTermEnumRequest request, Task task, ActionLi } } if (request.shardIds().size() == 0) { - listener.onResponse(new NodeTermEnumResponse(request.nodeId(), Collections.emptyList(), null, true)); + listener.onResponse(new NodeTermsEnumResponse(request.nodeId(), Collections.emptyList(), null, true)); } else { transportService.getThreadPool() .executor(shardExecutor) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/package-info.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/package-info.java similarity index 84% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/package-info.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/package-info.java index ac11a71657094..4042ef981827f 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/action/package-info.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/package-info.java @@ -7,4 +7,4 @@ /** * Enumerate a field's terms action. */ -package org.elasticsearch.xpack.core.termenum.action; \ No newline at end of file +package org.elasticsearch.xpack.core.termsenum.action; \ No newline at end of file diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/rest/RestTermEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/rest/RestTermsEnumAction.java similarity index 69% rename from x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/rest/RestTermEnumAction.java rename to x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/rest/RestTermsEnumAction.java index a890a554b40b9..4c771995d085e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termenum/rest/RestTermEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/rest/RestTermsEnumAction.java @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum.rest; +package org.elasticsearch.xpack.core.termsenum.rest; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.Strings; @@ -12,8 +12,8 @@ import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.action.RestToXContentListener; -import org.elasticsearch.xpack.core.termenum.action.TermEnumAction; -import org.elasticsearch.xpack.core.termenum.action.TermEnumRequest; +import org.elasticsearch.xpack.core.termsenum.action.TermsEnumAction; +import org.elasticsearch.xpack.core.termsenum.action.TermsEnumRequest; import java.io.IOException; import java.util.List; @@ -21,13 +21,13 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; import static org.elasticsearch.rest.RestRequest.Method.POST; -public class RestTermEnumAction extends BaseRestHandler { +public class RestTermsEnumAction extends BaseRestHandler { @Override public List routes() { return List.of( - new Route(GET, "/{index}/_terms"), - new Route(POST, "/{index}/_terms")); + new Route(GET, "/{index}/_terms_enum"), + new Route(POST, "/{index}/_terms_enum")); } @Override @@ -38,10 +38,10 @@ public String getName() { @Override public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException { try (XContentParser parser = request.contentOrSourceParamParser()) { - TermEnumRequest termEnumRequest = TermEnumAction.fromXContent(parser, + TermsEnumRequest termEnumRequest = TermsEnumAction.fromXContent(parser, Strings.splitStringByCommaToArray(request.param("index"))); return channel -> - client.execute(TermEnumAction.INSTANCE, termEnumRequest, new RestToXContentListener<>(channel)); + client.execute(TermsEnumAction.INSTANCE, termEnumRequest, new RestToXContentListener<>(channel)); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/MultiShardTermsEnumTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/MultiShardTermsEnumTests.java similarity index 95% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/MultiShardTermsEnumTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/MultiShardTermsEnumTests.java index 0868a16bae16f..f31b9d33c05e6 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/MultiShardTermsEnumTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/MultiShardTermsEnumTests.java @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum; +package org.elasticsearch.xpack.core.termsenum; import org.apache.lucene.analysis.MockAnalyzer; import org.apache.lucene.document.Document; @@ -26,9 +26,9 @@ import org.elasticsearch.common.lucene.search.AutomatonQueries; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.core.termenum.action.MultiShardTermsEnum; -import org.elasticsearch.xpack.core.termenum.action.SimpleTermCountEnum; -import org.elasticsearch.xpack.core.termenum.action.TermCount; +import org.elasticsearch.xpack.core.termsenum.action.MultiShardTermsEnum; +import org.elasticsearch.xpack.core.termsenum.action.SimpleTermCountEnum; +import org.elasticsearch.xpack.core.termsenum.action.TermCount; import java.io.Closeable; import java.util.ArrayList; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermCountTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermCountTests.java similarity index 91% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermCountTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermCountTests.java index d1b2247c67ed1..c6368db221e29 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermCountTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermCountTests.java @@ -4,12 +4,12 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum; +package org.elasticsearch.xpack.core.termsenum; import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractSerializingTestCase; -import org.elasticsearch.xpack.core.termenum.action.TermCount; +import org.elasticsearch.xpack.core.termsenum.action.TermCount; import java.io.IOException; diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermEnumResponseTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermsEnumResponseTests.java similarity index 70% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermEnumResponseTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermsEnumResponseTests.java index b242e6176fa52..e166599f591c0 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TermEnumResponseTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TermsEnumResponseTests.java @@ -4,14 +4,14 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum; +package org.elasticsearch.xpack.core.termsenum; import org.elasticsearch.ElasticsearchException; import org.elasticsearch.action.support.DefaultShardOperationFailedException; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractBroadcastResponseTestCase; -import org.elasticsearch.xpack.core.termenum.action.TermEnumResponse; +import org.elasticsearch.xpack.core.termsenum.action.TermsEnumResponse; import java.io.IOException; import java.util.ArrayList; @@ -19,7 +19,7 @@ import java.util.List; import java.util.Set; -public class TermEnumResponseTests extends AbstractBroadcastResponseTestCase { +public class TermsEnumResponseTests extends AbstractBroadcastResponseTestCase { protected static List getRandomTerms() { int termCount = randomIntBetween(0, 100); @@ -32,7 +32,7 @@ protected static List getRandomTerms() { return terms; } - private static TermEnumResponse createRandomTermEnumResponse() { + private static TermsEnumResponse createRandomTermEnumResponse() { int totalShards = randomIntBetween(1, 10); int successfulShards = randomIntBetween(0, totalShards); int failedShards = totalShards - successfulShards; @@ -45,30 +45,30 @@ private static TermEnumResponse createRandomTermEnumResponse() { new DefaultShardOperationFailedException(index, shard, exc) ); } - return new TermEnumResponse(getRandomTerms(), totalShards, successfulShards, failedShards, shardFailures, randomBoolean()); + return new TermsEnumResponse(getRandomTerms(), totalShards, successfulShards, failedShards, shardFailures, randomBoolean()); } @Override - protected TermEnumResponse doParseInstance(XContentParser parser) throws IOException { - return TermEnumResponse.fromXContent(parser); + protected TermsEnumResponse doParseInstance(XContentParser parser) throws IOException { + return TermsEnumResponse.fromXContent(parser); } @Override - protected TermEnumResponse createTestInstance() { + protected TermsEnumResponse createTestInstance() { return createRandomTermEnumResponse(); } @Override - protected void assertEqualInstances(TermEnumResponse response, TermEnumResponse parsedResponse) { + protected void assertEqualInstances(TermsEnumResponse response, TermsEnumResponse parsedResponse) { super.assertEqualInstances(response, parsedResponse); assertEquals(response.getTerms().size(), parsedResponse.getTerms().size()); assertTrue(response.getTerms().containsAll(parsedResponse.getTerms())); } @Override - protected TermEnumResponse createTestInstance(int totalShards, int successfulShards, int failedShards, + protected TermsEnumResponse createTestInstance(int totalShards, int successfulShards, int failedShards, List failures) { - return new TermEnumResponse(getRandomTerms(), totalShards, successfulShards, failedShards, failures, randomBoolean()); + return new TermsEnumResponse(getRandomTerms(), totalShards, successfulShards, failedShards, failures, randomBoolean()); } @@ -77,7 +77,7 @@ public void testToXContent() { String s = randomAlphaOfLengthBetween(1, 10); List terms = new ArrayList<>(); terms.add(s); - TermEnumResponse response = new TermEnumResponse(terms, 10, 10, 0, new ArrayList<>(), true); + TermsEnumResponse response = new TermsEnumResponse(terms, 10, 10, 0, new ArrayList<>(), true); String output = Strings.toString(response); assertEquals("{\"_shards\":{\"total\":10,\"successful\":10,\"failed\":0},\"terms\":[" + diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TransportTermEnumActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java similarity index 64% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TransportTermEnumActionTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java index 9db582bd6688f..ca9c46e9f6e7d 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/TransportTermEnumActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/TransportTermsEnumActionTests.java @@ -4,29 +4,29 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum; +package org.elasticsearch.xpack.core.termsenum; import org.elasticsearch.action.ActionListener; import org.elasticsearch.test.ESSingleNodeTestCase; -import org.elasticsearch.xpack.core.termenum.action.TermEnumAction; -import org.elasticsearch.xpack.core.termenum.action.TermEnumRequest; -import org.elasticsearch.xpack.core.termenum.action.TermEnumResponse; +import org.elasticsearch.xpack.core.termsenum.action.TermsEnumAction; +import org.elasticsearch.xpack.core.termsenum.action.TermsEnumRequest; +import org.elasticsearch.xpack.core.termsenum.action.TermsEnumResponse; import java.util.concurrent.atomic.AtomicBoolean; import static org.hamcrest.Matchers.equalTo; -public class TransportTermEnumActionTests extends ESSingleNodeTestCase { +public class TransportTermsEnumActionTests extends ESSingleNodeTestCase { /* * Copy of test that tripped up similarly broadcast ValidateQuery */ public void testListenerOnlyInvokedOnceWhenIndexDoesNotExist() { final AtomicBoolean invoked = new AtomicBoolean(); - final ActionListener listener = new ActionListener<>() { + final ActionListener listener = new ActionListener<>() { @Override - public void onResponse(final TermEnumResponse validateQueryResponse) { + public void onResponse(final TermsEnumResponse validateQueryResponse) { fail("onResponse should not be invoked in this failure case"); } @@ -38,7 +38,7 @@ public void onFailure(final Exception e) { } }; - client().execute(TermEnumAction.INSTANCE, new TermEnumRequest("non-existent-index"),listener); + client().execute(TermsEnumAction.INSTANCE, new TermsEnumRequest("non-existent-index"),listener); assertThat(invoked.get(), equalTo(true)); // ensure that onFailure was invoked } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/action/RestTermEnumActionTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/action/RestTermsEnumActionTests.java similarity index 90% rename from x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/action/RestTermEnumActionTests.java rename to x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/action/RestTermsEnumActionTests.java index 62499c2e9bab9..93e15f141634c 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termenum/action/RestTermEnumActionTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/action/RestTermsEnumActionTests.java @@ -4,7 +4,7 @@ * 2.0; you may not use this file except in compliance with the Elastic License * 2.0. */ -package org.elasticsearch.xpack.core.termenum.action; +package org.elasticsearch.xpack.core.termsenum.action; import org.elasticsearch.action.ActionListener; import org.elasticsearch.action.ActionRequest; @@ -31,7 +31,8 @@ import org.elasticsearch.threadpool.ThreadPool; import org.elasticsearch.transport.Transport; import org.elasticsearch.usage.UsageService; -import org.elasticsearch.xpack.core.termenum.rest.RestTermEnumAction; +import org.elasticsearch.xpack.core.termsenum.action.TermsEnumAction; +import org.elasticsearch.xpack.core.termsenum.rest.RestTermsEnumAction; import org.junit.AfterClass; import org.junit.BeforeClass; @@ -46,18 +47,18 @@ import static org.hamcrest.Matchers.equalTo; import static org.mockito.Mockito.mock; -public class RestTermEnumActionTests extends ESTestCase { +public class RestTermsEnumActionTests extends ESTestCase { - private static ThreadPool threadPool = new TestThreadPool(RestTermEnumActionTests.class.getName()); + private static ThreadPool threadPool = new TestThreadPool(RestTermsEnumActionTests.class.getName()); private static NodeClient client = new NodeClient(Settings.EMPTY, threadPool); private static UsageService usageService = new UsageService(); private static RestController controller = new RestController(emptySet(), null, client, new NoneCircuitBreakerService(), usageService); - private static RestTermEnumAction action = new RestTermEnumAction(); + private static RestTermsEnumAction action = new RestTermsEnumAction(); /** - * Configures {@link NodeClient} to stub {@link TermEnumAction} transport action. + * Configures {@link NodeClient} to stub {@link TermsEnumAction} transport action. *

* This lower level of execution is out of the scope of this test. */ @@ -66,7 +67,7 @@ public class RestTermEnumActionTests extends ESTestCase { public static void stubTermEnumAction() { final TaskManager taskManager = new TaskManager(Settings.EMPTY, threadPool, Collections.emptySet()); - final TransportAction transportAction = new TransportAction(TermEnumAction.NAME, + final TransportAction transportAction = new TransportAction(TermsEnumAction.NAME, new ActionFilters(Collections.emptySet()), taskManager) { @Override protected void doExecute(Task task, ActionRequest request, ActionListener listener) { @@ -74,7 +75,7 @@ protected void doExecute(Task task, ActionRequest request, ActionListener listen }; final Map actions = new HashMap<>(); - actions.put(TermEnumAction.INSTANCE, transportAction); + actions.put(TermsEnumAction.INSTANCE, transportAction); client.initialize(actions, taskManager, () -> "local", mock(Transport.Connection.class), null, new NamedWriteableRegistry(List.of())); @@ -140,7 +141,7 @@ public void testRestTermEnumActionMissingField() throws Exception { private RestRequest createRestRequest(String content) { return new FakeRestRequest.Builder(xContentRegistry()) - .withPath("index1/_terms") + .withPath("index1/_terms_enum") .withParams(emptyMap()) .withContent(new BytesArray(content), XContentType.JSON) .build(); diff --git a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java index d97287c3d5922..a309fec91a663 100644 --- a/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java +++ b/x-pack/plugin/mapper-constant-keyword/src/main/java/org/elasticsearch/xpack/constantkeyword/mapper/ConstantKeywordFieldMapper.java @@ -39,8 +39,8 @@ import org.elasticsearch.index.query.SearchExecutionContext; import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; import org.elasticsearch.search.lookup.SearchLookup; -import org.elasticsearch.xpack.core.termenum.action.SimpleTermCountEnum; -import org.elasticsearch.xpack.core.termenum.action.TermCount; +import org.elasticsearch.xpack.core.termsenum.action.SimpleTermCountEnum; +import org.elasticsearch.xpack.core.termsenum.action.TermCount; import java.io.IOException; import java.time.ZoneId; diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/terms_enum/10_basic.yml similarity index 100% rename from x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/term_enum/10_basic.yml rename to x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/terms_enum/10_basic.yml From c40a0dba4994b8146499d0e298eeca90e527ab04 Mon Sep 17 00:00:00 2001 From: markharwood Date: Mon, 26 Apr 2021 16:05:55 +0100 Subject: [PATCH 24/31] Checkstyle fix --- .../xpack/core/termsenum/action/TermsEnumRequestBuilder.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequestBuilder.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequestBuilder.java index 573162b17b758..98c0baf1a5ea8 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequestBuilder.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequestBuilder.java @@ -9,7 +9,10 @@ import org.elasticsearch.action.support.broadcast.BroadcastOperationRequestBuilder; import org.elasticsearch.client.ElasticsearchClient; -public class TermsEnumRequestBuilder extends BroadcastOperationRequestBuilder { +public class TermsEnumRequestBuilder extends BroadcastOperationRequestBuilder< + TermsEnumRequest, + TermsEnumResponse, + TermsEnumRequestBuilder> { public TermsEnumRequestBuilder(ElasticsearchClient client, TermsEnumAction action) { super(client, action, new TermsEnumRequest()); From 6b9f41c3e3c7fb613745361977d485fdbb456a19 Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 27 Apr 2021 11:42:26 +0100 Subject: [PATCH 25/31] Addressing review comments - formatting, thread pool choices and more --- .../index/mapper/KeywordFieldMapper.java | 2 -- .../mapper/flattened/FlattenedFieldMapper.java | 6 +++--- .../mapper/flattened/FlattenedFieldParser.java | 1 + .../termsenum/action/NodeTermsEnumRequest.java | 7 +++---- .../termsenum/action/SimpleTermCountEnum.java | 6 +++--- .../xpack/core/termsenum/action/TermCount.java | 16 ++++------------ .../termsenum/action/TermsEnumResponse.java | 10 ---------- .../action/TransportTermsEnumAction.java | 18 +++++++++++++----- .../termsenum/MultiShardTermsEnumTests.java | 2 +- 9 files changed, 28 insertions(+), 40 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 96a62807ce285..746c9700a8125 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -257,8 +257,6 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) { this.eagerGlobalOrdinals = false; this.scriptValues = null; } - - @Override public TermsEnum getTerms(boolean caseInsensitive, String string, SearchExecutionContext queryShardContext) throws IOException { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 23451868994ff..8e06f72b45d7d 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -306,7 +306,7 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format) // Wraps a raw Lucene TermsEnum to strip values of fieldnames - static class TranslatingTermsEnum extends TermsEnum{ + static class TranslatingTermsEnum extends TermsEnum { TermsEnum delegate; TranslatingTermsEnum(TermsEnum delegate) { @@ -317,7 +317,7 @@ static class TranslatingTermsEnum extends TermsEnum{ public BytesRef next() throws IOException { // Strip the term of the fieldname value BytesRef result = delegate.next(); - if(result != null) { + if (result != null) { result = FlattenedFieldParser.extractValue(result); } return result; @@ -327,7 +327,7 @@ public BytesRef next() throws IOException { public BytesRef term() throws IOException { // Strip the term of the fieldname value BytesRef result = delegate.term(); - if(result != null) { + if (result != null) { result = FlattenedFieldParser.extractValue(result); } return result; diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java index 23420ea47171a..e9aaf7cc9c91b 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java @@ -166,6 +166,7 @@ static BytesRef extractKey(BytesRef keyedValue) { } return new BytesRef(keyedValue.bytes, keyedValue.offset, length); } + static BytesRef extractValue(BytesRef keyedValue) { int length; for (length = 0; length < keyedValue.length; length++){ diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java index 1abb71a1da713..a7e781a466315 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java @@ -84,10 +84,10 @@ public long taskStartedTimeMillis() { } /** - * The time this request was materialized on a shard + * The time this request was materialized on a node * (defaults to "now" if serialization was not used e.g. a local request). */ - public long shardStartedTimeMillis() { + public long nodeStartedTimeMillis() { if (nodeStartedTimeMillis == 0) { nodeStartedTimeMillis = System.currentTimeMillis(); } @@ -122,8 +122,7 @@ public void writeTo(StreamOutput out) throws IOException { out.writeVInt(size); // Adjust the amount of permitted time the shard has remaining to gather terms. long timeSpentSoFarInCoordinatingNode = System.currentTimeMillis() - taskStartedTimeMillis; - assert timeSpentSoFarInCoordinatingNode >= 0; - int remainingTimeForShardToUse = (int) (timeout - timeSpentSoFarInCoordinatingNode); + long remainingTimeForShardToUse = (timeout - timeSpentSoFarInCoordinatingNode); // TODO - if already timed out can we shortcut the trip somehow? Throw exception if remaining time < 0? out.writeVLong(remainingTimeForShardToUse); out.writeVLong(taskStartedTimeMillis); diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/SimpleTermCountEnum.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/SimpleTermCountEnum.java index 8fb2ae1b8081f..0f75dd39a0a1e 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/SimpleTermCountEnum.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/SimpleTermCountEnum.java @@ -23,13 +23,13 @@ * {@link MappedFieldType#getTerms(boolean, String, org.elasticsearch.index.query.SearchExecutionContext)} * but can't return a raw Lucene TermsEnum. */ -public class SimpleTermCountEnum extends TermsEnum{ +public class SimpleTermCountEnum extends TermsEnum { int index =-1; TermCount[] sortedTerms; TermCount current = null; public SimpleTermCountEnum(TermCount[] terms) { - sortedTerms = terms; + sortedTerms = Arrays.copyOf(terms, terms.length); Arrays.sort(sortedTerms, Comparator.comparing(TermCount::getTerm)); } @@ -49,7 +49,7 @@ public BytesRef term() throws IOException { @Override public BytesRef next() throws IOException { index++; - if(index >= sortedTerms.length) { + if (index >= sortedTerms.length) { current = null; } else { current = sortedTerms[index]; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermCount.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermCount.java index 32083c995b519..9e16d3b8fa8d1 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermCount.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermCount.java @@ -18,7 +18,7 @@ import java.io.IOException; import java.util.Objects; -import static org.elasticsearch.common.xcontent.ConstructingObjectParser.optionalConstructorArg; +import static org.elasticsearch.common.xcontent.ConstructingObjectParser.constructorArg; public class TermCount implements Writeable, ToXContentFragment { @@ -31,11 +31,11 @@ public class TermCount implements Writeable, ToXContentFragment { a -> { return new TermCount((String) a[0], (long) a[1]); } ); static { - PARSER.declareString(optionalConstructorArg(), new ParseField(TERM_FIELD)); - PARSER.declareLong(optionalConstructorArg(), new ParseField(DOC_COUNT_FIELD)); + PARSER.declareString(constructorArg(), new ParseField(TERM_FIELD)); + PARSER.declareLong(constructorArg(), new ParseField(DOC_COUNT_FIELD)); } - private String term; + private final String term; private long docCount; @@ -91,14 +91,6 @@ void addToDocCount(long extra) { docCount += extra; } - void setTerm(String term) { - this.term = term; - } - - void setDocCount(long docCount) { - this.docCount = docCount; - } - @Override public String toString() { return term + ":" + docCount; diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumResponse.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumResponse.java index 5bd95a1da8fce..cb7de89ccaea6 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumResponse.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumResponse.java @@ -55,13 +55,11 @@ public class TermsEnumResponse extends BroadcastResponse { private final List terms; private boolean complete; - private int skippedShards; TermsEnumResponse(StreamInput in) throws IOException { super(in); terms = in.readStringList(); complete = in.readBoolean(); - skippedShards = in.readVInt(); } public TermsEnumResponse( @@ -82,20 +80,12 @@ public TermsEnumResponse( public List getTerms() { return terms; } - - /** - * The number of shards skipped by the index filter - */ - public int getSkippedShards() { - return skippedShards; - } @Override public void writeTo(StreamOutput out) throws IOException { super.writeTo(out); out.writeStringCollection(terms); out.writeBoolean(complete); - out.writeVInt(skippedShards); } @Override diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java index 405cb56400336..41b057df871af 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java @@ -30,6 +30,7 @@ import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.inject.Inject; import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.util.concurrent.EsThreadPoolExecutor; import org.elasticsearch.common.util.concurrent.ThreadContext; import org.elasticsearch.core.internal.io.IOUtils; import org.elasticsearch.index.IndexService; @@ -272,7 +273,7 @@ protected NodeTermsEnumResponse dataNodeOperation(NodeTermsEnumRequest request, String error = null; long timeout_millis = request.timeout(); - long scheduledEnd = request.shardStartedTimeMillis() + timeout_millis; + long scheduledEnd = request.nodeStartedTimeMillis() + timeout_millis; ArrayList shardTermsEnums = new ArrayList<>(); ArrayList openedResources = new ArrayList<>(); @@ -291,7 +292,7 @@ protected NodeTermsEnumResponse dataNodeOperation(NodeTermsEnumRequest request, shardId.id(), 0, searcher, - request::shardStartedTimeMillis, + request::nodeStartedTimeMillis, null, Collections.emptyMap() ); @@ -364,7 +365,7 @@ private boolean canAccess( shardId.id(), 0, null, - request::shardStartedTimeMillis, + request::nodeStartedTimeMillis, null, Collections.emptyMap() ); @@ -396,7 +397,7 @@ private boolean canMatchShard(ShardId shardId, NodeTermsEnumRequest req) throws if (req.indexFilter() == null || req.indexFilter() instanceof MatchAllQueryBuilder) { return true; } - ShardSearchRequest searchRequest = new ShardSearchRequest(shardId, req.shardStartedTimeMillis(), AliasFilter.EMPTY); + ShardSearchRequest searchRequest = new ShardSearchRequest(shardId, req.nodeStartedTimeMillis(), AliasFilter.EMPTY); searchRequest.source(new SearchSourceBuilder().query(req.indexFilter())); return searchService.canMatch(searchRequest).canMatch(); } @@ -591,8 +592,15 @@ private void asyncNodeOperation(NodeTermsEnumRequest request, Task task, ActionL if (request.shardIds().size() == 0) { listener.onResponse(new NodeTermsEnumResponse(request.nodeId(), Collections.emptyList(), null, true)); } else { + // Use the search threadpool if its queue is empty + assert transportService.getThreadPool() + .executor( + ThreadPool.Names.SEARCH + ) instanceof EsThreadPoolExecutor : "SEARCH threadpool must be an instance of ThreadPoolExecutor"; + EsThreadPoolExecutor ex = (EsThreadPoolExecutor) transportService.getThreadPool().executor(ThreadPool.Names.SEARCH); + final String executorName = ex.getQueue().size() == 0 ? ThreadPool.Names.SEARCH : shardExecutor; transportService.getThreadPool() - .executor(shardExecutor) + .executor(executorName) .execute(ActionRunnable.supply(listener, () -> dataNodeOperation(request, task))); } } diff --git a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/MultiShardTermsEnumTests.java b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/MultiShardTermsEnumTests.java index f31b9d33c05e6..248bc102c4e64 100644 --- a/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/MultiShardTermsEnumTests.java +++ b/x-pack/plugin/core/src/test/java/org/elasticsearch/xpack/core/termsenum/MultiShardTermsEnumTests.java @@ -86,7 +86,7 @@ public void testRandomIndexFusion() throws Exception { for (DirectoryReader reader : readers) { Terms terms = MultiTerms.getTerms(reader, fieldName); TermsEnum te = automaton.getTermsEnum(terms); - if(randomBoolean()) { + if (randomBoolean()) { // Simulate fields like constant-keyword which use a SimpleTermCountEnum to present results // rather than the raw TermsEnum from Lucene. ArrayList termCounts = new ArrayList<>(); From 814e45e9e0c5e4ae1d94c26aed5fdb091851690d Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 27 Apr 2021 12:32:00 +0100 Subject: [PATCH 26/31] =?UTF-8?q?Oops.=20Thought=20I=E2=80=99d=20resolved?= =?UTF-8?q?=20this=20review=20comment=20but=20hadn=E2=80=99t?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../main/java/org/elasticsearch/threadpool/ThreadPool.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java index 1f99ad82bbd31..bdc011de774bd 100644 --- a/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java +++ b/server/src/main/java/org/elasticsearch/threadpool/ThreadPool.java @@ -178,7 +178,10 @@ public ThreadPool(final Settings settings, final ExecutorBuilder... customBui builders.put(Names.GET, new FixedExecutorBuilder(settings, Names.GET, allocatedProcessors, 1000, false)); builders.put(Names.ANALYZE, new FixedExecutorBuilder(settings, Names.ANALYZE, 1, 16, false)); builders.put(Names.SEARCH, new FixedExecutorBuilder(settings, Names.SEARCH, searchThreadPoolSize(allocatedProcessors), 1000, true)); - builders.put(Names.AUTO_COMPLETE, new FixedExecutorBuilder(settings, Names.AUTO_COMPLETE, 2, 10, true)); + builders.put( + Names.AUTO_COMPLETE, + new FixedExecutorBuilder(settings, Names.AUTO_COMPLETE, Math.max(allocatedProcessors / 4, 1), 100, true) + ); builders.put(Names.SEARCH_THROTTLED, new FixedExecutorBuilder(settings, Names.SEARCH_THROTTLED, 1, 100, true)); builders.put(Names.MANAGEMENT, new ScalingExecutorBuilder(Names.MANAGEMENT, 1, boundedBy(allocatedProcessors, 1, 5), TimeValue.timeValueMinutes(5))); From 989751883f134abeafdef6da3a747ac70203573a Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 27 Apr 2021 16:46:32 +0100 Subject: [PATCH 27/31] Changed timeout setting to a TimeValue --- docs/reference/search/terms-enum.asciidoc | 4 ++-- .../rest-api-spec/api/termsenum.json | 2 +- .../termsenum/action/TermsEnumAction.java | 6 +++++- .../termsenum/action/TermsEnumRequest.java | 20 +++++++++---------- .../test/terms_enum/10_basic.yml | 14 +++++++++++++ 5 files changed, 32 insertions(+), 14 deletions(-) diff --git a/docs/reference/search/terms-enum.asciidoc b/docs/reference/search/terms-enum.asciidoc index c555806679f92..93b2a9b356b7d 100644 --- a/docs/reference/search/terms-enum.asciidoc +++ b/docs/reference/search/terms-enum.asciidoc @@ -79,8 +79,8 @@ How many matching terms to return. Defaults to 10 [[terms-enum-timeout-param]] `timeout`:: -(Optional, integer) -The maximum length of time in milliseconds to spend collecting results. Defaults to 1000. +(Optional, <>) +The maximum length of time to spend collecting results. Defaults to "1s" (one second). If the timeout is exceeded the `complete` flag set to false in the response and the results may be partial or empty. diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/termsenum.json b/rest-api-spec/src/main/resources/rest-api-spec/api/termsenum.json index 0d790c8b1bcad..ea99cdec0ad19 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/api/termsenum.json +++ b/rest-api-spec/src/main/resources/rest-api-spec/api/termsenum.json @@ -29,7 +29,7 @@ }, "params":{}, "body":{ - "description":"field name, string which is the prefix expected in matching terms, timeout in milliseconds and size for max number of results" + "description":"field name, string which is the prefix expected in matching terms, timeout and size for max number of results" } } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumAction.java index d21c1157276f7..acfd46676fb51 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumAction.java @@ -8,6 +8,7 @@ import org.elasticsearch.action.ActionType; import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.unit.TimeValue; import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; @@ -22,6 +23,7 @@ public class TermsEnumAction extends ActionType { static final ParseField INDEX_FILTER = new ParseField("index_filter"); + static final ParseField TIMEOUT = new ParseField("timeout"); private TermsEnumAction() { super(NAME, TermsEnumResponse::new); @@ -39,7 +41,9 @@ public static TermsEnumRequest fromXContent(XContentParser parser, String... ind PARSER.declareString(TermsEnumRequest::string, new ParseField("string")); PARSER.declareInt(TermsEnumRequest::size, new ParseField("size")); PARSER.declareBoolean(TermsEnumRequest::caseInsensitive, new ParseField("case_insensitive")); - PARSER.declareInt(TermsEnumRequest::timeoutInMillis, new ParseField("timeout")); + PARSER.declareField(TermsEnumRequest::timeout, + (p, c) -> TimeValue.parseTimeValue(p.text(), TIMEOUT.getPreferredName()), + TIMEOUT, ObjectParser.ValueType.STRING); PARSER.declareObject(TermsEnumRequest::indexFilter, (p, context) -> parseInnerQueryBuilder(p),INDEX_FILTER); } } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java index 6d4b9de8791e4..f88977419dddb 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java @@ -27,7 +27,7 @@ public class TermsEnumRequest extends BroadcastRequest implements ToXContentObject { public static int DEFAULT_SIZE = 10; - public static int DEFAULT_TIMEOUT_MILLIS = 1000; + public static TimeValue DEFAULT_TIMEOUT = new TimeValue(1000); private String field; private String string; @@ -56,7 +56,7 @@ public TermsEnumRequest(StreamInput in) throws IOException { public TermsEnumRequest(String... indices) { super(indices); indicesOptions(IndicesOptions.fromOptions(false, false, true, false)); - timeout(new TimeValue(DEFAULT_TIMEOUT_MILLIS)); + timeout(DEFAULT_TIMEOUT); } @Override @@ -65,6 +65,13 @@ public ActionRequestValidationException validate() { if (field == null) { validationException = ValidateActions.addValidationError("field cannot be null", validationException); } + if (timeout() == null) { + validationException = ValidateActions.addValidationError("Timeout cannot be null", validationException); + } else { + if (timeout().getSeconds() > 60) { + validationException = ValidateActions.addValidationError("Timeout cannot be > 1 minute", validationException); + } + } return validationException; } @@ -109,14 +116,7 @@ public int size() { public void size(int size) { this.size = size; } - - /** - * TThe max time in milliseconds to spend gathering terms - */ - public void timeoutInMillis(int timeout) { - timeout(new TimeValue(timeout)); - } - + /** * If case insensitive matching is required */ diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/terms_enum/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/terms_enum/10_basic.yml index 6d9ed51f5f9cf..1cfadab8c540e 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/terms_enum/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/terms_enum/10_basic.yml @@ -252,6 +252,20 @@ teardown: body: {"field": "foo", "string":"b", "index_filter":{"range":{"timestamp":{"gte":"2021-01-01T01:01:01.000Z"}}}} - length: {terms: 1} --- +"Test legal timeout": + - do: + termsenum: + index: test_* + body: {"field": "foo", "string":"b", "timeout": "1s"} + - length: {terms: 3} +--- +"Test illegal timeout": + - do: + catch: /Timeout cannot be > 1 minute/ + termsenum: + index: test_* + body: {"field": "foo", "string":"b", "timeout": "2m"} +--- "Test security": - do: From 2cb91dfcc247cd1deb849fd8cf5403f7e7117f34 Mon Sep 17 00:00:00 2001 From: markharwood Date: Tue, 27 Apr 2021 17:02:10 +0100 Subject: [PATCH 28/31] Checkstyle fix --- .../xpack/core/termsenum/action/TermsEnumRequest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java index f88977419dddb..159277349a946 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TermsEnumRequest.java @@ -69,7 +69,8 @@ public ActionRequestValidationException validate() { validationException = ValidateActions.addValidationError("Timeout cannot be null", validationException); } else { if (timeout().getSeconds() > 60) { - validationException = ValidateActions.addValidationError("Timeout cannot be > 1 minute", validationException); + validationException = ValidateActions.addValidationError("Timeout cannot be > 1 minute", + validationException); } } return validationException; From 6d55f99c93e4f61b20a537d6c8ad1c507f95a314 Mon Sep 17 00:00:00 2001 From: markharwood Date: Fri, 30 Apr 2021 10:30:08 +0100 Subject: [PATCH 29/31] In flattened fields make only the value (not the field name) subject to case insensitive search --- .../flattened/FlattenedFieldMapper.java | 13 +++++--- .../test/terms_enum/10_basic.yml | 33 +++++++++++++++++++ 2 files changed, 41 insertions(+), 5 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java index 8e06f72b45d7d..26b4af9300e3f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java @@ -258,16 +258,19 @@ public Query termQueryCaseInsensitive(Object value, SearchExecutionContext conte @Override public TermsEnum getTerms(boolean caseInsensitive, String string, SearchExecutionContext queryShardContext) throws IOException { IndexReader reader = queryShardContext.searcher().getTopReaderContext().reader(); - String searchString = FlattenedFieldParser.createKeyedValue(key, string); Terms terms = MultiTerms.getTerms(reader, name()); if (terms == null) { // Field does not exist on this shard. return null; } - Automaton a = caseInsensitive - ? AutomatonQueries.caseInsensitivePrefix(searchString) - : Automata.makeString(searchString); - a = Operations.concatenate(a, Automata.makeAnyString()); + + Automaton a = Automata.makeString(key + FlattenedFieldParser.SEPARATOR); + if (caseInsensitive) { + a = Operations.concatenate(a, AutomatonQueries.caseInsensitivePrefix(string)); + } else { + a = Operations.concatenate(a, Automata.makeString(string)); + a = Operations.concatenate(a, Automata.makeAnyString()); + } a = MinimizationOperations.minimize(a, Integer.MAX_VALUE); CompiledAutomaton automaton = new CompiledAutomaton(a); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/terms_enum/10_basic.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/terms_enum/10_basic.yml index 1cfadab8c540e..27ca0a82a203a 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/terms_enum/10_basic.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/terms_enum/10_basic.yml @@ -244,6 +244,39 @@ teardown: body: {"field": "foo.bar", "string":"b"} - length: {terms: 1} +--- +"Test case insensitivity": + - do: + termsenum: + index: test_k + body: {"field": "foo", "string":"B"} + - length: {terms: 0} + + - do: + termsenum: + index: test_k + body: {"field": "foo", "string":"B", "case_insensitive": true} + - length: {terms: 1} + + - do: + termsenum: + index: test_f + body: {"field": "foo.bar", "string":"B"} + - length: {terms: 0} + + - do: + termsenum: + index: test_f + body: {"field": "foo.bar", "string":"B", "case_insensitive": true} + - length: {terms: 1} + + - do: + termsenum: + index: test_f + body: {"field": "foo.Bar", "string":"B", "case_insensitive": true} + - length: {terms: 0} + + --- "Test index filtering": - do: From cf7005361a9b3426685eb5911f36c6255c423f2e Mon Sep 17 00:00:00 2001 From: markharwood Date: Fri, 30 Apr 2021 14:30:35 +0100 Subject: [PATCH 30/31] Moved initialisation of data node timing of request from NodeTermsEnumRequest constructor to TransportTermsEnumAction#asyncNodeOperation --- .../core/termsenum/action/NodeTermsEnumRequest.java | 11 ++++++----- .../termsenum/action/TransportTermsEnumAction.java | 5 ++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java index a7e781a466315..2dd1b9776ac00 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/NodeTermsEnumRequest.java @@ -39,9 +39,6 @@ public class NodeTermsEnumRequest extends TransportRequest implements IndicesReq public NodeTermsEnumRequest(StreamInput in) throws IOException { super(in); - // Set the clock running as soon as we appear on a node. - nodeStartedTimeMillis = System.currentTimeMillis(); - field = in.readString(); string = in.readString(); caseInsensitive = in.readBoolean(); @@ -85,15 +82,19 @@ public long taskStartedTimeMillis() { /** * The time this request was materialized on a node - * (defaults to "now" if serialization was not used e.g. a local request). */ - public long nodeStartedTimeMillis() { + long nodeStartedTimeMillis() { + // In case startTimerOnDataNode has not been called (should never happen in normal circumstances?) if (nodeStartedTimeMillis == 0) { nodeStartedTimeMillis = System.currentTimeMillis(); } return this.nodeStartedTimeMillis; } + public void startTimerOnDataNode() { + nodeStartedTimeMillis = System.currentTimeMillis(); + } + public Set shardIds() { return Collections.unmodifiableSet(shardIds); } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java index 41b057df871af..6ad321cc1096a 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java @@ -576,8 +576,11 @@ public void messageReceived(NodeTermsEnumRequest request, TransportChannel chann private void asyncNodeOperation(NodeTermsEnumRequest request, Task task, ActionListener listener) throws IOException { + // Start the clock ticking on the data node using the data node's local current time. + request.startTimerOnDataNode(); + // DLS/FLS check copied from ResizeRequestInterceptor - check permissions and - // any index_filter canMatch checks on network thread before allocating work + // any index_filter canMatch checks on network thread before allocating work ThreadContext threadContext = transportService.getThreadPool().getThreadContext(); final XPackLicenseState frozenLicenseState = licenseState.copyCurrentLicenseState(); for (ShardId shardId : request.shardIds().toArray(new ShardId[0])) { From 22312cf82527d80e4b330d56fec8108c9cdfd513 Mon Sep 17 00:00:00 2001 From: markharwood Date: Thu, 6 May 2021 09:39:47 +0100 Subject: [PATCH 31/31] Remove outdated TODOs --- .../xpack/core/termsenum/action/TransportTermsEnumAction.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java index 6ad321cc1096a..d176a49b512bd 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/termsenum/action/TransportTermsEnumAction.java @@ -481,12 +481,9 @@ boolean checkForEarlyFinish() { protected void performOperation(final String nodeId, final Set shardIds, final int nodeIndex) { if (shardIds.size() == 0) { // no more active shards... (we should not really get here, just safety) - // MH TODO somewhat arbitrarily returining firsy onNoOperation(nodeId); } else { try { - // TODO pass through a reduced timeout (the original time limit, minus whatever we may have - // spent already getting to this point. final NodeTermsEnumRequest nodeRequest = newNodeRequest(nodeId, shardIds, request); nodeRequest.setParentTask(clusterService.localNode().getId(), task.getId()); DiscoveryNode node = nodes.get(nodeId);