From 00db112f32d947e758d7d5f6e4a6c7444a22a7c3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Jul 2022 17:09:37 -0500 Subject: [PATCH 1/6] Bump woodstox-core from 6.2.8 to 6.3.0 in /plugins/repository-hdfs (#3841) Bumps [woodstox-core](https://github.com/FasterXML/woodstox) from 6.2.8 to 6.3.0. - [Release notes](https://github.com/FasterXML/woodstox/releases) - [Commits](https://github.com/FasterXML/woodstox/compare/woodstox-core-6.2.8...woodstox-core-6.3.0) --- updated-dependencies: - dependency-name: com.fasterxml.woodstox:woodstox-core dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- plugins/repository-hdfs/build.gradle | 2 +- plugins/repository-hdfs/licenses/woodstox-core-6.2.8.jar.sha1 | 1 - plugins/repository-hdfs/licenses/woodstox-core-6.3.0.jar.sha1 | 1 + 3 files changed, 2 insertions(+), 2 deletions(-) delete mode 100644 plugins/repository-hdfs/licenses/woodstox-core-6.2.8.jar.sha1 create mode 100644 plugins/repository-hdfs/licenses/woodstox-core-6.3.0.jar.sha1 diff --git a/plugins/repository-hdfs/build.gradle b/plugins/repository-hdfs/build.gradle index 49fdca4ab9c53..d1658279bfe48 100644 --- a/plugins/repository-hdfs/build.gradle +++ b/plugins/repository-hdfs/build.gradle @@ -83,7 +83,7 @@ dependencies { api 'net.minidev:json-smart:2.4.8' api 'org.apache.zookeeper:zookeeper:3.8.0' api "io.netty:netty-all:${versions.netty}" - implementation 'com.fasterxml.woodstox:woodstox-core:6.2.8' + implementation 'com.fasterxml.woodstox:woodstox-core:6.3.0' implementation 'org.codehaus.woodstox:stax2-api:4.2.1' hdfsFixture project(':test:fixtures:hdfs-fixture') diff --git a/plugins/repository-hdfs/licenses/woodstox-core-6.2.8.jar.sha1 b/plugins/repository-hdfs/licenses/woodstox-core-6.2.8.jar.sha1 deleted file mode 100644 index ae65cdebf26de..0000000000000 --- a/plugins/repository-hdfs/licenses/woodstox-core-6.2.8.jar.sha1 +++ /dev/null @@ -1 +0,0 @@ -670748292899c53b1963730d9eb7f8ab71314e90 \ No newline at end of file diff --git a/plugins/repository-hdfs/licenses/woodstox-core-6.3.0.jar.sha1 b/plugins/repository-hdfs/licenses/woodstox-core-6.3.0.jar.sha1 new file mode 100644 index 0000000000000..ebd85df98b39e --- /dev/null +++ b/plugins/repository-hdfs/licenses/woodstox-core-6.3.0.jar.sha1 @@ -0,0 +1 @@ +03c1df4164b107ee22ad4f24bd453ec78a0efd95 \ No newline at end of file From 85bfde171180e1e6ff02074dcac5dfad1c804efc Mon Sep 17 00:00:00 2001 From: Hauck <67768441+hauck-jvsh@users.noreply.github.com> Date: Thu, 28 Jul 2022 16:12:48 -0300 Subject: [PATCH 2/6] Fix the version tests (#4035) Signed-off-by: Hauck --- .../test/search.highlight/30_max_analyzed_offset.yml | 4 ++-- .../fetch/subphase/highlight/AbstractHighlighterBuilder.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml index c38afc96590e9..a18ac45e62175 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.highlight/30_max_analyzed_offset.yml @@ -39,8 +39,8 @@ setup: --- "Unified highlighter on a field WITHOUT OFFSETS using max_analyzer_offset should SUCCEED": - skip: - version: " - 2.99.99" - reason: only starting supporting the parameter max_analyzer_offset on version 3.0 + version: " - 2.1.99" + reason: only starting supporting the parameter max_analyzer_offset on version 2.2 - do: search: rest_total_hits_as_int: true diff --git a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java index db0089fd5f180..94533a5e3dae6 100644 --- a/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java +++ b/server/src/main/java/org/opensearch/search/fetch/subphase/highlight/AbstractHighlighterBuilder.java @@ -189,7 +189,7 @@ protected AbstractHighlighterBuilder(StreamInput in) throws IOException { requireFieldMatch(in.readOptionalBoolean()); - if (in.getVersion().onOrAfter(Version.V_3_0_0)) { + if (in.getVersion().onOrAfter(Version.V_2_2_0)) { maxAnalyzerOffset(in.readOptionalVInt()); } @@ -234,7 +234,7 @@ public final void writeTo(StreamOutput out) throws IOException { out.writeMap(options); } out.writeOptionalBoolean(requireFieldMatch); - if (out.getVersion().onOrAfter(Version.V_3_0_0)) { + if (out.getVersion().onOrAfter(Version.V_2_2_0)) { out.writeOptionalVInt(maxAnalyzerOffset); } doWriteTo(out); From fb7d81a7c00df14223e34511a6b1b5a3eff44e65 Mon Sep 17 00:00:00 2001 From: Petar Dzepina Date: Thu, 28 Jul 2022 21:30:28 +0200 Subject: [PATCH 3/6] Add doc_count field mapper (#3985) Bucket aggregations compute bucket doc_count values by incrementing the doc_count by 1 for every document collected in the bucket. When using summary fields (such as aggregate_metric_double) one field may represent more than one document. To provide this functionality this commit implements a new field mapper (named doc_count field mapper). This field is a positive integer representing the number of documents aggregated in a single summary field. Bucket aggregations check if a field of type doc_count exists in a document and take this value into consideration when computing doc counts. Note: This originated from upstream PR 64503. Signed-off-by: Petar Dzepina --- .../380_doc_count_field.yml | 146 ++++++++++++++++ .../index/mapper/DocCountFieldMapper.java | 164 ++++++++++++++++++ .../org/opensearch/indices/IndicesModule.java | 2 + .../search/aggregations/AggregatorBase.java | 4 +- .../bucket/BucketsAggregator.java | 37 ++-- .../aggregations/bucket/DocCountProvider.java | 62 +++++++ .../adjacency/AdjacencyMatrixAggregator.java | 4 +- .../bucket/composite/CompositeAggregator.java | 5 +- .../CompositeValuesCollectorQueue.java | 22 +-- .../bucket/composite/SortedDocsProducer.java | 6 +- .../bucket/nested/NestedAggregator.java | 3 +- .../GlobalOrdinalsStringTermsAggregator.java | 13 +- .../mapper/DocCountFieldMapperTests.java | 83 +++++++++ .../index/mapper/DocCountFieldTypeTests.java | 69 ++++++++ .../indices/IndicesModuleTests.java | 2 + .../bucket/DocCountProviderTests.java | 107 ++++++++++++ .../CompositeValuesCollectorQueueTests.java | 2 +- 17 files changed, 692 insertions(+), 39 deletions(-) create mode 100644 rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/380_doc_count_field.yml create mode 100644 server/src/main/java/org/opensearch/index/mapper/DocCountFieldMapper.java create mode 100644 server/src/main/java/org/opensearch/search/aggregations/bucket/DocCountProvider.java create mode 100644 server/src/test/java/org/opensearch/index/mapper/DocCountFieldMapperTests.java create mode 100644 server/src/test/java/org/opensearch/index/mapper/DocCountFieldTypeTests.java create mode 100644 server/src/test/java/org/opensearch/search/aggregations/bucket/DocCountProviderTests.java diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/380_doc_count_field.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/380_doc_count_field.yml new file mode 100644 index 0000000000000..a7a796eceeb39 --- /dev/null +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search.aggregation/380_doc_count_field.yml @@ -0,0 +1,146 @@ +setup: + - do: + indices.create: + index: test_1 + body: + settings: + number_of_replicas: 0 + mappings: + properties: + str: + type: keyword + number: + type: integer + + - do: + bulk: + index: test_1 + refresh: true + body: + - '{"index": {}}' + - '{"_doc_count": 10, "str": "abc", "number" : 500, "unmapped": "abc" }' + - '{"index": {}}' + - '{"_doc_count": 5, "str": "xyz", "number" : 100, "unmapped": "xyz" }' + - '{"index": {}}' + - '{"_doc_count": 7, "str": "foo", "number" : 100, "unmapped": "foo" }' + - '{"index": {}}' + - '{"_doc_count": 1, "str": "foo", "number" : 200, "unmapped": "foo" }' + - '{"index": {}}' + - '{"str": "abc", "number" : 500, "unmapped": "abc" }' + +--- +"Test numeric terms agg with doc_count": + - skip: + version: "1.0.0 - " + reason: "until this is released and we know exact version" + + - do: + search: + rest_total_hits_as_int: true + body: { "size" : 0, "aggs" : { "num_terms" : { "terms" : { "field" : "number" } } } } + + - match: { hits.total: 5 } + - length: { aggregations.num_terms.buckets: 3 } + - match: { aggregations.num_terms.buckets.0.key: 100 } + - match: { aggregations.num_terms.buckets.0.doc_count: 12 } + - match: { aggregations.num_terms.buckets.1.key: 500 } + - match: { aggregations.num_terms.buckets.1.doc_count: 11 } + - match: { aggregations.num_terms.buckets.2.key: 200 } + - match: { aggregations.num_terms.buckets.2.doc_count: 1 } + +--- +"Test keyword terms agg with doc_count": + - skip: + version: "1.0.0 - " + reason: "until this is released and we know exact version" + - do: + search: + rest_total_hits_as_int: true + body: { "size" : 0, "aggs" : { "str_terms" : { "terms" : { "field" : "str" } } } } + + - match: { hits.total: 5 } + - length: { aggregations.str_terms.buckets: 3 } + - match: { aggregations.str_terms.buckets.0.key: "abc" } + - match: { aggregations.str_terms.buckets.0.doc_count: 11 } + - match: { aggregations.str_terms.buckets.1.key: "foo" } + - match: { aggregations.str_terms.buckets.1.doc_count: 8 } + - match: { aggregations.str_terms.buckets.2.key: "xyz" } + - match: { aggregations.str_terms.buckets.2.doc_count: 5 } + +--- +"Test unmapped string terms agg with doc_count": + - skip: + version: "1.0.0 - " + reason: "until this is released and we know exact version" + - do: + bulk: + index: test_2 + refresh: true + body: + - '{"index": {}}' + - '{"_doc_count": 10, "str": "abc" }' + - '{"index": {}}' + - '{"str": "abc" }' + - do: + search: + index: test_2 + rest_total_hits_as_int: true + body: { "size" : 0, "aggs" : { "str_terms" : { "terms" : { "field" : "str.keyword" } } } } + + - match: { hits.total: 2 } + - length: { aggregations.str_terms.buckets: 1 } + - match: { aggregations.str_terms.buckets.0.key: "abc" } + - match: { aggregations.str_terms.buckets.0.doc_count: 11 } + +--- +"Test composite str_terms agg with doc_count": + - skip: + version: "1.0.0 - " + reason: "until this is released and we know exact version" + - do: + search: + rest_total_hits_as_int: true + body: { "size" : 0, "aggs" : + { "composite_agg" : { "composite" : + { + "sources": ["str_terms": { "terms": { "field": "str" } }] + } + } + } + } + + - match: { hits.total: 5 } + - length: { aggregations.composite_agg.buckets: 3 } + - match: { aggregations.composite_agg.buckets.0.key.str_terms: "abc" } + - match: { aggregations.composite_agg.buckets.0.doc_count: 11 } + - match: { aggregations.composite_agg.buckets.1.key.str_terms: "foo" } + - match: { aggregations.composite_agg.buckets.1.doc_count: 8 } + - match: { aggregations.composite_agg.buckets.2.key.str_terms: "xyz" } + - match: { aggregations.composite_agg.buckets.2.doc_count: 5 } + +--- +"Test composite num_terms agg with doc_count": + - skip: + version: "1.0.0 - " + reason: "until this is released and we know exact version" + - do: + search: + rest_total_hits_as_int: true + body: { "size" : 0, "aggs" : + { "composite_agg" : + { "composite" : + { + "sources": ["num_terms" : { "terms" : { "field" : "number" } }] + } + } + } + } + + - match: { hits.total: 5 } + - length: { aggregations.composite_agg.buckets: 3 } + - match: { aggregations.composite_agg.buckets.0.key.num_terms: 100 } + - match: { aggregations.composite_agg.buckets.0.doc_count: 12 } + - match: { aggregations.composite_agg.buckets.1.key.num_terms: 200 } + - match: { aggregations.composite_agg.buckets.1.doc_count: 1 } + - match: { aggregations.composite_agg.buckets.2.key.num_terms: 500 } + - match: { aggregations.composite_agg.buckets.2.doc_count: 11 } diff --git a/server/src/main/java/org/opensearch/index/mapper/DocCountFieldMapper.java b/server/src/main/java/org/opensearch/index/mapper/DocCountFieldMapper.java new file mode 100644 index 0000000000000..e3f6f5ef130e9 --- /dev/null +++ b/server/src/main/java/org/opensearch/index/mapper/DocCountFieldMapper.java @@ -0,0 +1,164 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.index.mapper; + +import org.apache.lucene.document.Field; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.apache.lucene.search.Query; +import org.opensearch.common.xcontent.XContentParser; +import org.opensearch.common.xcontent.XContentParserUtils; +import org.opensearch.index.query.QueryShardContext; +import org.opensearch.index.query.QueryShardException; +import org.opensearch.search.lookup.SearchLookup; + +import java.io.IOException; +import java.util.Collections; +import java.util.List; + +/** + * Mapper for the doc_count field. + * + * @opensearch.internal + */ +public class DocCountFieldMapper extends MetadataFieldMapper { + + public static final String NAME = "_doc_count"; + public static final String CONTENT_TYPE = "_doc_count"; + + public static final TypeParser PARSER = new ConfigurableTypeParser( + c -> new DocCountFieldMapper(), + c -> new DocCountFieldMapper.Builder() + ); + + static class Builder extends MetadataFieldMapper.Builder { + + Builder() { + super(NAME); + } + + @Override + protected List> getParameters() { + return Collections.emptyList(); + } + + @Override + public DocCountFieldMapper build(BuilderContext context) { + return new DocCountFieldMapper(); + } + } + + /** + * Field type for DocCount Field Mapper + * + * @opensearch.internal + */ + public static final class DocCountFieldType extends MappedFieldType { + + public static final DocCountFieldType INSTANCE = new DocCountFieldType(); + + private static final Long defaultValue = 1L; + + public DocCountFieldType() { + super(NAME, false, false, true, TextSearchInfo.NONE, Collections.emptyMap()); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public String familyTypeName() { + return NumberFieldMapper.NumberType.LONG.typeName(); + } + + @Override + public Query existsQuery(QueryShardContext context) { + return new DocValuesFieldExistsQuery(NAME); + } + + @Override + public Query termQuery(Object value, QueryShardContext context) { + throw new QueryShardException(context, "Field [" + name() + "] of type [" + typeName() + "] is not searchable"); + } + + @Override + public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) { + if (format != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats."); + } + + return new SourceValueFetcher(name(), context, defaultValue) { + @Override + protected Object parseSourceValue(Object value) { + if ("".equals(value)) { + return defaultValue; + } else { + return NumberFieldMapper.NumberType.objectToLong(value, false); + } + } + }; + } + } + + private DocCountFieldMapper() { + super(DocCountFieldType.INSTANCE); + } + + @Override + protected void parseCreateField(ParseContext context) throws IOException { + XContentParser parser = context.parser(); + XContentParserUtils.ensureExpectedToken(XContentParser.Token.VALUE_NUMBER, parser.currentToken(), parser); + + long value = parser.longValue(false); + if (value <= 0) { + throw new IllegalArgumentException("Field [" + fieldType().name() + "] must be a positive integer."); + } + final Field docCount = new NumericDocValuesField(NAME, value); + context.doc().add(docCount); + } + + @Override + public void preParse(ParseContext context) {} + + @Override + public DocCountFieldType fieldType() { + return (DocCountFieldType) super.fieldType(); + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + +} diff --git a/server/src/main/java/org/opensearch/indices/IndicesModule.java b/server/src/main/java/org/opensearch/indices/IndicesModule.java index 29ff507ad9fcf..e413a73940011 100644 --- a/server/src/main/java/org/opensearch/indices/IndicesModule.java +++ b/server/src/main/java/org/opensearch/indices/IndicesModule.java @@ -48,6 +48,7 @@ import org.opensearch.index.mapper.CompletionFieldMapper; import org.opensearch.index.mapper.DataStreamFieldMapper; import org.opensearch.index.mapper.DateFieldMapper; +import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.FieldAliasMapper; import org.opensearch.index.mapper.FieldNamesFieldMapper; import org.opensearch.index.mapper.GeoPointFieldMapper; @@ -192,6 +193,7 @@ private static Map initBuiltInMetadataMa builtInMetadataMappers.put(NestedPathFieldMapper.NAME, NestedPathFieldMapper.PARSER); builtInMetadataMappers.put(VersionFieldMapper.NAME, VersionFieldMapper.PARSER); builtInMetadataMappers.put(SeqNoFieldMapper.NAME, SeqNoFieldMapper.PARSER); + builtInMetadataMappers.put(DocCountFieldMapper.NAME, DocCountFieldMapper.PARSER); // _field_names must be added last so that it has a chance to see all the other mappers builtInMetadataMappers.put(FieldNamesFieldMapper.NAME, FieldNamesFieldMapper.PARSER); return Collections.unmodifiableMap(builtInMetadataMappers); diff --git a/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java b/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java index db51f00056f72..1d315980512b4 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java +++ b/server/src/main/java/org/opensearch/search/aggregations/AggregatorBase.java @@ -200,7 +200,7 @@ public Map metadata() { @Override public final LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws IOException { - preGetSubLeafCollectors(); + preGetSubLeafCollectors(ctx); final LeafBucketCollector sub = collectableSubAggregators.getLeafCollector(ctx); return getLeafCollector(ctx, sub); } @@ -209,7 +209,7 @@ public final LeafBucketCollector getLeafCollector(LeafReaderContext ctx) throws * Can be overridden by aggregator implementations that like the perform an operation before the leaf collectors * of children aggregators are instantiated for the next segment. */ - protected void preGetSubLeafCollectors() throws IOException {} + protected void preGetSubLeafCollectors(LeafReaderContext ctx) throws IOException {} /** * Can be overridden by aggregator implementation to be called back when the collection phase starts. diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java index 79512586d06de..67af0b13eed3b 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/BucketsAggregator.java @@ -31,9 +31,10 @@ package org.opensearch.search.aggregations.bucket; +import org.apache.lucene.index.LeafReaderContext; import org.opensearch.common.lease.Releasable; import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.IntArray; +import org.opensearch.common.util.LongArray; import org.opensearch.search.aggregations.AggregationExecutionException; import org.opensearch.search.aggregations.Aggregator; import org.opensearch.search.aggregations.AggregatorBase; @@ -70,7 +71,8 @@ public abstract class BucketsAggregator extends AggregatorBase { private final BigArrays bigArrays; private final IntConsumer multiBucketConsumer; - private IntArray docCounts; + private LongArray docCounts; + protected final DocCountProvider docCountProvider; public BucketsAggregator( String name, @@ -87,7 +89,8 @@ public BucketsAggregator( } else { multiBucketConsumer = (count) -> {}; } - docCounts = bigArrays.newIntArray(1, true); + docCounts = bigArrays.newLongArray(1, true); + docCountProvider = new DocCountProvider(); } /** @@ -116,7 +119,8 @@ public final void collectBucket(LeafBucketCollector subCollector, int doc, long * Same as {@link #collectBucket(LeafBucketCollector, int, long)}, but doesn't check if the docCounts needs to be re-sized. */ public final void collectExistingBucket(LeafBucketCollector subCollector, int doc, long bucketOrd) throws IOException { - if (docCounts.increment(bucketOrd, 1) == 1) { + long docCount = docCountProvider.getDocCount(doc); + if (docCounts.increment(bucketOrd, docCount) == docCount) { // We calculate the final number of buckets only during the reduce phase. But we still need to // trigger bucket consumer from time to time in order to give it a chance to check available memory and break // the execution if we are running out. To achieve that we are passing 0 as a bucket count. @@ -147,11 +151,11 @@ public final void mergeBuckets(long[] mergeMap, long newNumBuckets) { * merge the actual ordinals and doc ID deltas. */ public final void mergeBuckets(long newNumBuckets, LongUnaryOperator mergeMap) { - try (IntArray oldDocCounts = docCounts) { - docCounts = bigArrays.newIntArray(newNumBuckets, true); + try (LongArray oldDocCounts = docCounts) { + docCounts = bigArrays.newLongArray(newNumBuckets, true); docCounts.fill(0, newNumBuckets, 0); for (long i = 0; i < oldDocCounts.size(); i++) { - int docCount = oldDocCounts.get(i); + long docCount = oldDocCounts.get(i); if (docCount == 0) continue; @@ -164,14 +168,14 @@ public final void mergeBuckets(long newNumBuckets, LongUnaryOperator mergeMap) { } } - public IntArray getDocCounts() { + public LongArray getDocCounts() { return docCounts; } /** * Utility method to increment the doc counts of the given bucket (identified by the bucket ordinal) */ - public final void incrementBucketDocCount(long bucketOrd, int inc) { + public final void incrementBucketDocCount(long bucketOrd, long inc) { docCounts = bigArrays.grow(docCounts, bucketOrd + 1); docCounts.increment(bucketOrd, inc); } @@ -179,7 +183,7 @@ public final void incrementBucketDocCount(long bucketOrd, int inc) { /** * Utility method to return the number of documents that fell in the given bucket (identified by the bucket ordinal) */ - public final int bucketDocCount(long bucketOrd) { + public final long bucketDocCount(long bucketOrd) { if (bucketOrd >= docCounts.size()) { // This may happen eg. if no document in the highest buckets is accepted by a sub aggregator. // For example, if there is a long terms agg on 3 terms 1,2,3 with a sub filter aggregator and if no document with 3 as a value @@ -337,7 +341,7 @@ protected final InternalAggregation[] buildAggregationsForFixedBucketCount( */ @FunctionalInterface protected interface BucketBuilderForFixedCount { - B build(int offsetInOwningOrd, int docCount, InternalAggregations subAggregationResults); + B build(int offsetInOwningOrd, long docCount, InternalAggregations subAggregationResults); } /** @@ -432,7 +436,7 @@ protected final InternalAggregation[] buildAggregationsForVariableBuckets( */ @FunctionalInterface protected interface BucketBuilderForVariable { - B build(long bucketValue, int docCount, InternalAggregations subAggregationResults); + B build(long bucketValue, long docCount, InternalAggregations subAggregationResults); } /** @@ -466,7 +470,7 @@ public BucketComparator bucketComparator(String key, SortOrder order) { return super.bucketComparator(key, order); } if (key == null || "doc_count".equals(key)) { - return (lhs, rhs) -> order.reverseMul() * Integer.compare(bucketDocCount(lhs), bucketDocCount(rhs)); + return (lhs, rhs) -> order.reverseMul() * Long.compare(bucketDocCount(lhs), bucketDocCount(rhs)); } throw new IllegalArgumentException( "Ordering on a single-bucket aggregation can only be done on its doc_count. " @@ -478,6 +482,13 @@ public BucketComparator bucketComparator(String key, SortOrder order) { ); } + @Override + protected void preGetSubLeafCollectors(LeafReaderContext ctx) throws IOException { + super.preGetSubLeafCollectors(ctx); + // Set LeafReaderContext to the doc_count provider + docCountProvider.setLeafReaderContext(ctx); + } + public static boolean descendsFromGlobalAggregator(Aggregator parent) { while (parent != null) { if (parent.getClass() == GlobalAggregator.class) { diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/DocCountProvider.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/DocCountProvider.java new file mode 100644 index 0000000000000..30fe16ec4eb0e --- /dev/null +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/DocCountProvider.java @@ -0,0 +1,62 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.aggregations.bucket; + +import org.apache.lucene.index.DocValues; +import org.apache.lucene.index.LeafReaderContext; +import org.apache.lucene.index.NumericDocValues; +import org.opensearch.index.mapper.DocCountFieldMapper; + +import java.io.IOException; + +/** + * An implementation of a doc_count provider that reads the value + * of the _doc_count field in the document. If a document does not have a + * _doc_count field the implementation will return 1 as the default value. + */ +public class DocCountProvider { + + private NumericDocValues docCountValues; + + public long getDocCount(int doc) throws IOException { + if (docCountValues != null && docCountValues.advanceExact(doc)) { + return docCountValues.longValue(); + } else { + return 1L; + } + } + + public void setLeafReaderContext(LeafReaderContext ctx) throws IOException { + docCountValues = DocValues.getNumeric(ctx.reader(), DocCountFieldMapper.NAME); + } +} diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java index c33887878dbd6..c9efbf7d34348 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/adjacency/AdjacencyMatrixAggregator.java @@ -228,7 +228,7 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I List buckets = new ArrayList<>(filters.length); for (int i = 0; i < keys.length; i++) { long bucketOrd = bucketOrd(owningBucketOrds[owningBucketOrdIdx], i); - int docCount = bucketDocCount(bucketOrd); + long docCount = bucketDocCount(bucketOrd); // Empty buckets are not returned because this aggregation will commonly be used under a // a date-histogram where we will look for transactions over time and can expect many // empty buckets. @@ -245,7 +245,7 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I for (int i = 0; i < keys.length; i++) { for (int j = i + 1; j < keys.length; j++) { long bucketOrd = bucketOrd(owningBucketOrds[owningBucketOrdIdx], pos); - int docCount = bucketDocCount(bucketOrd); + long docCount = bucketDocCount(bucketOrd); // Empty buckets are not returned due to potential for very sparse matrices if (docCount > 0) { String intersectKey = keys[i] + separator + keys[j]; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java index a907d17a731fe..8b487fc499602 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeAggregator.java @@ -196,7 +196,7 @@ public InternalAggregation[] buildAggregations(long[] owningBucketOrds) throws I int slot = queue.pop(); CompositeKey key = queue.toCompositeKey(slot); InternalAggregations aggs = subAggsForBuckets[slot]; - int docCount = queue.getDocCount(slot); + long docCount = queue.getDocCount(slot); buckets[queue.size()] = new InternalComposite.InternalBucket( sourceNames, formats, @@ -504,7 +504,8 @@ private LeafBucketCollector getFirstPassCollector(RoaringDocIdSet.Builder builde @Override public void collect(int doc, long bucket) throws IOException { try { - if (queue.addIfCompetitive(indexSortPrefix)) { + long docCount = docCountProvider.getDocCount(doc); + if (queue.addIfCompetitive(indexSortPrefix, docCount)) { if (builder != null && lastDoc != doc) { builder.add(doc); lastDoc = doc; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java index ac50869b5d79b..6ee1682a7b196 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueue.java @@ -38,7 +38,7 @@ import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; import org.opensearch.common.util.BigArrays; -import org.opensearch.common.util.IntArray; +import org.opensearch.common.util.LongArray; import org.opensearch.search.aggregations.LeafBucketCollector; import java.io.IOException; @@ -80,7 +80,7 @@ public int hashCode() { private final Map map; private final SingleDimensionValuesSource[] arrays; - private IntArray docCounts; + private LongArray docCounts; private boolean afterKeyIsSet = false; /** @@ -103,7 +103,7 @@ public int hashCode() { sources[i].setAfter(afterKey.get(i)); } } - this.docCounts = bigArrays.newIntArray(1, false); + this.docCounts = bigArrays.newLongArray(1, false); } @Override @@ -143,19 +143,19 @@ Comparable getUpperValueLeadSource() throws IOException { /** * Returns the document count in slot. */ - int getDocCount(int slot) { + long getDocCount(int slot) { return docCounts.get(slot); } /** * Copies the current value in slot. */ - private void copyCurrent(int slot) { + private void copyCurrent(int slot, long value) { for (int i = 0; i < arrays.length; i++) { arrays[i].copyCurrent(slot); } docCounts = bigArrays.grow(docCounts, slot + 1); - docCounts.set(slot, 1); + docCounts.set(slot, value); } /** @@ -265,8 +265,8 @@ LeafBucketCollector getLeafCollector(Comparable forceLeadSourceValue, LeafReader * Check if the current candidate should be added in the queue. * @return true if the candidate is competitive (added or already in the queue). */ - boolean addIfCompetitive() { - return addIfCompetitive(0); + boolean addIfCompetitive(long inc) { + return addIfCompetitive(0, inc); } /** @@ -279,12 +279,12 @@ boolean addIfCompetitive() { * * @throws CollectionTerminatedException if the current collection can be terminated early due to index sorting. */ - boolean addIfCompetitive(int indexSortSourcePrefix) { + boolean addIfCompetitive(int indexSortSourcePrefix, long inc) { // checks if the candidate key is competitive Integer topSlot = compareCurrent(); if (topSlot != null) { // this key is already in the top N, skip it - docCounts.increment(topSlot, 1); + docCounts.increment(topSlot, inc); return true; } if (afterKeyIsSet) { @@ -325,7 +325,7 @@ boolean addIfCompetitive(int indexSortSourcePrefix) { newSlot = size(); } // move the candidate key to its new slot - copyCurrent(newSlot); + copyCurrent(newSlot, inc); map.put(new Slot(newSlot), newSlot); add(newSlot); return true; diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SortedDocsProducer.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SortedDocsProducer.java index 3cdbee6b119ec..bd0a4f13ddf08 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SortedDocsProducer.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/composite/SortedDocsProducer.java @@ -40,6 +40,7 @@ import org.apache.lucene.util.DocIdSetBuilder; import org.opensearch.common.Nullable; import org.opensearch.search.aggregations.LeafBucketCollector; +import org.opensearch.search.aggregations.bucket.DocCountProvider; import java.io.IOException; @@ -74,6 +75,8 @@ protected boolean processBucket( ) throws IOException { final int[] topCompositeCollected = new int[1]; final boolean[] hasCollected = new boolean[1]; + final DocCountProvider docCountProvider = new DocCountProvider(); + docCountProvider.setLeafReaderContext(context); final LeafBucketCollector queueCollector = new LeafBucketCollector() { int lastDoc = -1; @@ -86,7 +89,8 @@ protected boolean processBucket( @Override public void collect(int doc, long bucket) throws IOException { hasCollected[0] = true; - if (queue.addIfCompetitive()) { + long docCount = docCountProvider.getDocCount(doc); + if (queue.addIfCompetitive(docCount)) { topCompositeCollected[0]++; if (adder != null && doc != lastDoc) { if (remainingBits == 0) { diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java index 5dc4d1c780350..02507f8569ffa 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/nested/NestedAggregator.java @@ -130,7 +130,8 @@ public void collect(int parentDoc, long bucket) throws IOException { } @Override - protected void preGetSubLeafCollectors() throws IOException { + protected void preGetSubLeafCollectors(LeafReaderContext ctx) throws IOException { + super.preGetSubLeafCollectors(ctx); processBufferedDocs(); } diff --git a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java index 4fcc42b6211b2..2475ade68bc07 100644 --- a/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java +++ b/server/src/main/java/org/opensearch/search/aggregations/bucket/terms/GlobalOrdinalsStringTermsAggregator.java @@ -43,7 +43,6 @@ import org.opensearch.common.io.stream.StreamOutput; import org.opensearch.common.lease.Releasable; import org.opensearch.common.lease.Releasables; -import org.opensearch.common.util.IntArray; import org.opensearch.common.util.LongArray; import org.opensearch.common.util.LongHash; import org.opensearch.common.xcontent.XContentBuilder; @@ -297,7 +296,7 @@ protected void doClose() { static class LowCardinality extends GlobalOrdinalsStringTermsAggregator { private LongUnaryOperator mapping; - private IntArray segmentDocCounts; + private LongArray segmentDocCounts; LowCardinality( String name, @@ -332,7 +331,7 @@ static class LowCardinality extends GlobalOrdinalsStringTermsAggregator { metadata ); assert factories == null || factories.countAggregators() == 0; - this.segmentDocCounts = context.bigArrays().newIntArray(1, true); + this.segmentDocCounts = context.bigArrays().newLongArray(1, true); } @Override @@ -356,7 +355,8 @@ public void collect(int doc, long owningBucketOrd) throws IOException { return; } int ord = singleValues.ordValue(); - segmentDocCounts.increment(ord + 1, 1); + long docCount = docCountProvider.getDocCount(doc); + segmentDocCounts.increment(ord + 1, docCount); } }); } @@ -369,7 +369,8 @@ public void collect(int doc, long owningBucketOrd) throws IOException { return; } for (long segmentOrd = segmentOrds.nextOrd(); segmentOrd != NO_MORE_ORDS; segmentOrd = segmentOrds.nextOrd()) { - segmentDocCounts.increment(segmentOrd + 1, 1); + long docCount = docCountProvider.getDocCount(doc); + segmentDocCounts.increment(segmentOrd + 1, docCount); } } }); @@ -392,7 +393,7 @@ private void mapSegmentCountsToGlobalCounts(LongUnaryOperator mapping) throws IO for (long i = 1; i < segmentDocCounts.size(); i++) { // We use set(...) here, because we need to reset the slow to 0. // segmentDocCounts get reused over the segments and otherwise counts would be too high. - int inc = segmentDocCounts.set(i, 0); + long inc = segmentDocCounts.set(i, 0); if (inc == 0) { continue; } diff --git a/server/src/test/java/org/opensearch/index/mapper/DocCountFieldMapperTests.java b/server/src/test/java/org/opensearch/index/mapper/DocCountFieldMapperTests.java new file mode 100644 index 0000000000000..634276a34b9af --- /dev/null +++ b/server/src/test/java/org/opensearch/index/mapper/DocCountFieldMapperTests.java @@ -0,0 +1,83 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.index.mapper; + +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexableField; + +import static org.hamcrest.Matchers.containsString; + +public class DocCountFieldMapperTests extends MapperServiceTestCase { + + private static final String CONTENT_TYPE = DocCountFieldMapper.CONTENT_TYPE; + private static final String DOC_COUNT_FIELD = DocCountFieldMapper.NAME; + + /** + * Test parsing field mapping and adding simple field + */ + public void testParseValue() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + ParsedDocument doc = mapper.parse(source(b -> b.field("foo", 500).field(CONTENT_TYPE, 100))); + + IndexableField field = doc.rootDoc().getField(DOC_COUNT_FIELD); + assertEquals(100L, field.numericValue()); + assertEquals(DocValuesType.NUMERIC, field.fieldType().docValuesType()); + assertEquals(1, doc.rootDoc().getFields(DOC_COUNT_FIELD).length); + } + + public void testInvalidDocument_NegativeDocCount() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field(CONTENT_TYPE, -100)))); + assertThat(e.getCause().getMessage(), containsString("Field [_doc_count] must be a positive integer")); + } + + public void testInvalidDocument_ZeroDocCount() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field(CONTENT_TYPE, 0)))); + assertThat(e.getCause().getMessage(), containsString("Field [_doc_count] must be a positive integer")); + } + + public void testInvalidDocument_NonNumericDocCount() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field(CONTENT_TYPE, "foo")))); + assertThat( + e.getCause().getMessage(), + containsString("Failed to parse object: expecting token of type [VALUE_NUMBER] but found [VALUE_STRING]") + ); + } + + public void testInvalidDocument_FractionalDocCount() throws Exception { + DocumentMapper mapper = createDocumentMapper(mapping(b -> {})); + Exception e = expectThrows(MapperParsingException.class, () -> mapper.parse(source(b -> b.field(CONTENT_TYPE, 100.23)))); + assertThat(e.getCause().getMessage(), containsString("100.23 cannot be converted to Long without data loss")); + } +} diff --git a/server/src/test/java/org/opensearch/index/mapper/DocCountFieldTypeTests.java b/server/src/test/java/org/opensearch/index/mapper/DocCountFieldTypeTests.java new file mode 100644 index 0000000000000..4961736ea933c --- /dev/null +++ b/server/src/test/java/org/opensearch/index/mapper/DocCountFieldTypeTests.java @@ -0,0 +1,69 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ +package org.opensearch.index.mapper; + +import org.apache.lucene.search.DocValuesFieldExistsQuery; +import org.opensearch.index.query.QueryShardException; + +import java.io.IOException; +import java.util.Arrays; + +public class DocCountFieldTypeTests extends FieldTypeTestCase { + + public void testTermQuery() { + MappedFieldType ft = new DocCountFieldMapper.DocCountFieldType(); + QueryShardException e = expectThrows(QueryShardException.class, () -> ft.termQuery(10L, randomMockShardContext())); + assertEquals("Field [_doc_count] of type [_doc_count] is not searchable", e.getMessage()); + } + + public void testRangeQuery() { + MappedFieldType ft = new DocCountFieldMapper.DocCountFieldType(); + IllegalArgumentException e = expectThrows( + IllegalArgumentException.class, + () -> ft.rangeQuery(null, null, randomBoolean(), randomBoolean(), null, null, null, null) + ); + assertEquals("Field [_doc_count] of type [_doc_count] does not support range queries", e.getMessage()); + } + + public void testExistsQuery() { + MappedFieldType ft = new DocCountFieldMapper.DocCountFieldType(); + assertTrue(ft.existsQuery(randomMockShardContext()) instanceof DocValuesFieldExistsQuery); + } + + public void testFetchSourceValue() throws IOException { + MappedFieldType fieldType = new DocCountFieldMapper.DocCountFieldType(); + assertEquals(Arrays.asList(14L), fetchSourceValue(fieldType, 14)); + assertEquals(Arrays.asList(14L), fetchSourceValue(fieldType, "14")); + assertEquals(Arrays.asList(1L), fetchSourceValue(fieldType, "")); + assertEquals(Arrays.asList(1L), fetchSourceValue(fieldType, null)); + } +} diff --git a/server/src/test/java/org/opensearch/indices/IndicesModuleTests.java b/server/src/test/java/org/opensearch/indices/IndicesModuleTests.java index afcc6aa006500..ec7ab06ac86a6 100644 --- a/server/src/test/java/org/opensearch/indices/IndicesModuleTests.java +++ b/server/src/test/java/org/opensearch/indices/IndicesModuleTests.java @@ -33,6 +33,7 @@ package org.opensearch.indices; import org.opensearch.Version; +import org.opensearch.index.mapper.DocCountFieldMapper; import org.opensearch.index.mapper.DataStreamFieldMapper; import org.opensearch.index.mapper.FieldNamesFieldMapper; import org.opensearch.index.mapper.IdFieldMapper; @@ -98,6 +99,7 @@ public Map getMetadataMappers() { NestedPathFieldMapper.NAME, VersionFieldMapper.NAME, SeqNoFieldMapper.NAME, + DocCountFieldMapper.NAME, FieldNamesFieldMapper.NAME }; public void testBuiltinMappers() { diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/DocCountProviderTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/DocCountProviderTests.java new file mode 100644 index 0000000000000..708d683fe2484 --- /dev/null +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/DocCountProviderTests.java @@ -0,0 +1,107 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/* + * Modifications Copyright OpenSearch Contributors. See + * GitHub history for details. + */ + +package org.opensearch.search.aggregations.bucket; + +import org.apache.lucene.document.IntPoint; +import org.apache.lucene.document.NumericDocValuesField; +import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.tests.index.RandomIndexWriter; +import org.opensearch.common.CheckedConsumer; +import org.opensearch.index.mapper.DocCountFieldMapper; +import org.opensearch.index.mapper.MappedFieldType; +import org.opensearch.index.mapper.NumberFieldMapper; +import org.opensearch.search.aggregations.AggregatorTestCase; +import org.opensearch.search.aggregations.bucket.global.GlobalAggregationBuilder; +import org.opensearch.search.aggregations.bucket.global.InternalGlobal; + +import java.io.IOException; +import java.util.List; +import java.util.function.Consumer; + +import static java.util.Collections.singleton; + +public class DocCountProviderTests extends AggregatorTestCase { + + private static final String DOC_COUNT_FIELD = DocCountFieldMapper.NAME; + private static final String NUMBER_FIELD = "number"; + + public void testDocsWithDocCount() throws IOException { + testAggregation(new MatchAllDocsQuery(), iw -> { + iw.addDocument(List.of(new NumericDocValuesField(DOC_COUNT_FIELD, 4), new SortedNumericDocValuesField(NUMBER_FIELD, 1))); + iw.addDocument(List.of(new NumericDocValuesField(DOC_COUNT_FIELD, 5), new SortedNumericDocValuesField(NUMBER_FIELD, 7))); + iw.addDocument( + List.of( + // Intentionally omit doc_count field + new SortedNumericDocValuesField(NUMBER_FIELD, 1) + ) + ); + }, global -> { assertEquals(10, global.getDocCount()); }); + } + + public void testDocsWithoutDocCount() throws IOException { + testAggregation(new MatchAllDocsQuery(), iw -> { + iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD, 1))); + iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD, 7))); + iw.addDocument(singleton(new SortedNumericDocValuesField(NUMBER_FIELD, 1))); + }, global -> { assertEquals(3, global.getDocCount()); }); + } + + public void testQueryFiltering() throws IOException { + testAggregation(IntPoint.newRangeQuery(NUMBER_FIELD, 4, 5), iw -> { + iw.addDocument(List.of(new NumericDocValuesField(DOC_COUNT_FIELD, 4), new IntPoint(NUMBER_FIELD, 6))); + iw.addDocument(List.of(new NumericDocValuesField(DOC_COUNT_FIELD, 2), new IntPoint(NUMBER_FIELD, 5))); + iw.addDocument( + List.of( + // Intentionally omit doc_count field + new IntPoint(NUMBER_FIELD, 1) + ) + ); + iw.addDocument( + List.of( + // Intentionally omit doc_count field + new IntPoint(NUMBER_FIELD, 5) + ) + ); + }, global -> { assertEquals(3, global.getDocCount()); }); + } + + private void testAggregation(Query query, CheckedConsumer indexer, Consumer verify) + throws IOException { + GlobalAggregationBuilder aggregationBuilder = new GlobalAggregationBuilder("_name"); + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NUMBER_FIELD, NumberFieldMapper.NumberType.LONG); + MappedFieldType docCountFieldType = new DocCountFieldMapper.DocCountFieldType(); + testCase(aggregationBuilder, query, indexer, verify, fieldType, docCountFieldType); + } +} diff --git a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java index 3dccdf8dab95e..4571ae7496ae1 100644 --- a/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java +++ b/server/src/test/java/org/opensearch/search/aggregations/bucket/composite/CompositeValuesCollectorQueueTests.java @@ -347,7 +347,7 @@ private void testRandomCase(boolean forceMerge, boolean missingBucket, int index final LeafBucketCollector leafCollector = new LeafBucketCollector() { @Override public void collect(int doc, long bucket) throws IOException { - queue.addIfCompetitive(indexSortSourcePrefix); + queue.addIfCompetitive(indexSortSourcePrefix, 1); } }; final LeafBucketCollector queueCollector = queue.getLeafCollector(leafReaderContext, leafCollector); From e65aacaf29e84f6ae4468499f738bd6b1fb88f09 Mon Sep 17 00:00:00 2001 From: Tianli Feng Date: Thu, 28 Jul 2022 12:36:43 -0700 Subject: [PATCH 4/6] Deprecate public methods and variables that contain 'master' terminology in class 'NoMasterBlockService' and 'MasterService' (#4006) Deprecates public methods and variables that contain 'master' terminology in class NoMasterBlockService and MasterService. Unit tests are also added to ensure consistency with the new naming. Signed-off-by: Tianli Feng --- .../index/rankeval/RankEvalResponseTests.java | 2 +- .../cluster/ClusterStateDiffIT.java | 4 +- .../cluster/MinimumClusterManagerNodesIT.java | 24 +++++------ .../cluster/NoClusterManagerNodeIT.java | 6 +-- .../UnsafeBootstrapAndDetachCommandIT.java | 8 ++-- .../discovery/ClusterManagerDisruptionIT.java | 8 +++- .../cluster/coordination/Coordinator.java | 10 ++--- .../coordination/JoinTaskExecutor.java | 4 +- .../NoClusterManagerBlockService.java | 43 +++++++++++++------ .../cluster/service/ClusterService.java | 2 +- .../cluster/service/MasterService.java | 37 +++++++++++----- .../common/util/concurrent/BaseFuture.java | 2 +- .../ExceptionSerializationTests.java | 6 ++- .../opensearch/OpenSearchExceptionTests.java | 7 ++- ...TransportResyncReplicationActionTests.java | 2 +- ...rnalClusterInfoServiceSchedulingTests.java | 4 +- .../coordination/CoordinatorTests.java | 14 +++--- ...anagerBlockServiceRenamedSettingTests.java | 8 ++-- .../NoClusterManagerBlockServiceTests.java | 20 ++++----- .../service/ClusterApplierServiceTests.java | 2 +- .../cluster/service/MasterServiceTests.java | 13 ++++++ ...ClusterStateServiceRandomUpdatesTests.java | 16 ++++--- .../AbstractCoordinatorTestCase.java | 8 ++-- .../org/opensearch/test/RandomObjects.java | 2 +- 24 files changed, 159 insertions(+), 93 deletions(-) diff --git a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java index 463df690be763..c8627932f2a9a 100644 --- a/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java +++ b/modules/rank-eval/src/test/java/org/opensearch/index/rankeval/RankEvalResponseTests.java @@ -76,7 +76,7 @@ public class RankEvalResponseTests extends OpenSearchTestCase { private static final Exception[] RANDOM_EXCEPTIONS = new Exception[] { - new ClusterBlockException(singleton(NoClusterManagerBlockService.NO_MASTER_BLOCK_WRITES)), + new ClusterBlockException(singleton(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_WRITES)), new CircuitBreakingException("Data too large", 123, 456, CircuitBreaker.Durability.PERMANENT), new SearchParseException(SHARD_TARGET, "Parse failure", new XContentLocation(12, 98)), new IllegalArgumentException("Closed resource", new RuntimeException("Resource")), diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterStateDiffIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterStateDiffIT.java index 52c2cf9bfdcd0..7d26a0a31833b 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterStateDiffIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/ClusterStateDiffIT.java @@ -396,9 +396,9 @@ private ClusterState.Builder randomBlocks(ClusterState clusterState) { private ClusterBlock randomGlobalBlock() { switch (randomInt(2)) { case 0: - return NoClusterManagerBlockService.NO_MASTER_BLOCK_ALL; + return NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ALL; case 1: - return NoClusterManagerBlockService.NO_MASTER_BLOCK_WRITES; + return NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_WRITES; default: return GatewayService.STATE_NOT_RECOVERED_BLOCK; } diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/MinimumClusterManagerNodesIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/MinimumClusterManagerNodesIT.java index 0bed559085b27..3e2a1f1452628 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/MinimumClusterManagerNodesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/MinimumClusterManagerNodesIT.java @@ -93,7 +93,7 @@ public void testTwoNodesNoClusterManagerBlock() throws Exception { logger.info("--> should be blocked, no cluster-manager..."); ClusterState state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID), equalTo(true)); + assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(true)); assertThat(state.nodes().getSize(), equalTo(1)); // verify that we still see the local node in the cluster state logger.info("--> start second node, cluster should be formed"); @@ -109,9 +109,9 @@ public void testTwoNodesNoClusterManagerBlock() throws Exception { assertThat(clusterHealthResponse.isTimedOut(), equalTo(false)); state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID), equalTo(false)); + assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(false)); state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID), equalTo(false)); + assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(false)); state = client().admin().cluster().prepareState().execute().actionGet().getState(); assertThat(state.nodes().getSize(), equalTo(2)); @@ -161,11 +161,11 @@ public void testTwoNodesNoClusterManagerBlock() throws Exception { assertBusy(() -> { ClusterState clusterState = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertTrue(clusterState.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID)); + assertTrue(clusterState.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID)); }); state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID), equalTo(true)); + assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(true)); // verify that both nodes are still in the cluster state but there is no cluster-manager assertThat(state.nodes().getSize(), equalTo(2)); assertThat(state.nodes().getClusterManagerNode(), equalTo(null)); @@ -184,9 +184,9 @@ public void testTwoNodesNoClusterManagerBlock() throws Exception { assertThat(clusterHealthResponse.isTimedOut(), equalTo(false)); state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID), equalTo(false)); + assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(false)); state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID), equalTo(false)); + assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(false)); state = client().admin().cluster().prepareState().execute().actionGet().getState(); assertThat(state.nodes().getSize(), equalTo(2)); @@ -214,7 +214,7 @@ public void testTwoNodesNoClusterManagerBlock() throws Exception { assertBusy(() -> { ClusterState state1 = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertThat(state1.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID), equalTo(true)); + assertThat(state1.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(true)); }); logger.info("--> starting the previous cluster-manager node again..."); @@ -232,9 +232,9 @@ public void testTwoNodesNoClusterManagerBlock() throws Exception { assertThat(clusterHealthResponse.isTimedOut(), equalTo(false)); state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID), equalTo(false)); + assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(false)); state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID), equalTo(false)); + assertThat(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(false)); state = client().admin().cluster().prepareState().execute().actionGet().getState(); assertThat(state.nodes().getSize(), equalTo(2)); @@ -262,7 +262,7 @@ public void testThreeNodesNoClusterManagerBlock() throws Exception { assertBusy(() -> { for (Client client : clients()) { ClusterState state1 = client.admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertThat(state1.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID), equalTo(true)); + assertThat(state1.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(true)); } }); @@ -324,7 +324,7 @@ public void testThreeNodesNoClusterManagerBlock() throws Exception { // spin here to wait till the state is set assertBusy(() -> { ClusterState st = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertThat(st.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID), equalTo(true)); + assertThat(st.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(true)); }); logger.info("--> start back the 2 nodes "); diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/NoClusterManagerNodeIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/NoClusterManagerNodeIT.java index 0a4d9f8564b61..313f9e0da17aa 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/NoClusterManagerNodeIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/NoClusterManagerNodeIT.java @@ -117,7 +117,7 @@ public void testNoClusterManagerActions() throws Exception { .execute() .actionGet() .getState(); - assertTrue(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID)); + assertTrue(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID)); }); assertRequestBuilderThrows( @@ -288,7 +288,7 @@ public void testNoClusterManagerActionsWriteClusterManagerBlock() throws Excepti assertBusy(() -> { ClusterState state = clientToClusterManagerlessNode.admin().cluster().prepareState().setLocal(true).get().getState(); - assertTrue(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID)); + assertTrue(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID)); }); GetResponse getResponse = clientToClusterManagerlessNode.prepareGet("test1", "1").get(); @@ -387,7 +387,7 @@ public void testNoClusterManagerActionsMetadataWriteClusterManagerBlock() throws assertBusy(() -> { for (String node : nodesWithShards) { ClusterState state = client(node).admin().cluster().prepareState().setLocal(true).get().getState(); - assertTrue(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID)); + assertTrue(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID)); } }); diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java index 0bbeeea9e27a2..ebe65ad48f47e 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/coordination/UnsafeBootstrapAndDetachCommandIT.java @@ -220,7 +220,7 @@ public void testBootstrapNotBootstrappedCluster() throws Exception { ); assertBusy(() -> { ClusterState state = client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertTrue(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID)); + assertTrue(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID)); }); Settings dataPathSettings = internalCluster().dataPathSettings(node); @@ -338,7 +338,7 @@ public void test3ClusterManagerNodes2Failed() throws Exception { .execute() .actionGet() .getState(); - assertTrue(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID)); + assertTrue(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID)); }); logger.info("--> try to unsafely bootstrap 1st cluster-manager-eligible node, while node lock is held"); @@ -394,7 +394,7 @@ public void test3ClusterManagerNodes2Failed() throws Exception { .execute() .actionGet() .getState(); - assertFalse(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID)); + assertFalse(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID)); assertTrue( state.metadata().persistentSettings().getAsBoolean(UnsafeBootstrapClusterManagerCommand.UNSAFE_BOOTSTRAP.getKey(), false) ); @@ -508,7 +508,7 @@ public void testNoInitialBootstrapAfterDetach() throws Exception { ); ClusterState state = internalCluster().client().admin().cluster().prepareState().setLocal(true).execute().actionGet().getState(); - assertTrue(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID)); + assertTrue(state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID)); internalCluster().stopRandomNode(InternalTestCluster.nameFilter(node)); } diff --git a/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java b/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java index 412b325b14e73..f548cfdb0eda2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/discovery/ClusterManagerDisruptionIT.java @@ -234,7 +234,11 @@ public void testVerifyApiBlocksDuringPartition() throws Exception { // continuously ping until network failures have been resolved. However // It may a take a bit before the node detects it has been cut off from the elected cluster-manager logger.info("waiting for isolated node [{}] to have no cluster-manager", isolatedNode); - assertNoClusterManager(isolatedNode, NoClusterManagerBlockService.NO_MASTER_BLOCK_METADATA_WRITES, TimeValue.timeValueSeconds(30)); + assertNoClusterManager( + isolatedNode, + NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_METADATA_WRITES, + TimeValue.timeValueSeconds(30) + ); logger.info("wait until elected cluster-manager has been removed and a new 2 node cluster was from (via [{}])", isolatedNode); ensureStableCluster(2, nonIsolatedNode); @@ -280,7 +284,7 @@ public void testVerifyApiBlocksDuringPartition() throws Exception { // continuously ping until network failures have been resolved. However // It may a take a bit before the node detects it has been cut off from the elected cluster-manager logger.info("waiting for isolated node [{}] to have no cluster-manager", isolatedNode); - assertNoClusterManager(isolatedNode, NoClusterManagerBlockService.NO_MASTER_BLOCK_ALL, TimeValue.timeValueSeconds(30)); + assertNoClusterManager(isolatedNode, NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ALL, TimeValue.timeValueSeconds(30)); // make sure we have stable cluster & cross partition recoveries are canceled by the removal of the missing node // the unresponsive partition causes recoveries to only time out after 15m (default) and these will cause diff --git a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java index 14f9cca3630fe..36dd4aba104b2 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/Coordinator.java @@ -105,7 +105,7 @@ import java.util.stream.Stream; import java.util.stream.StreamSupport; -import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_MASTER_BLOCK_ID; +import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID; import static org.opensearch.gateway.ClusterStateUpdaters.hideStateIfNotRecovered; import static org.opensearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; import static org.opensearch.monitor.StatusInfo.Status.UNHEALTHY; @@ -839,7 +839,7 @@ protected void doStart() { .blocks( ClusterBlocks.builder() .addGlobalBlock(STATE_NOT_RECOVERED_BLOCK) - .addGlobalBlock(noClusterManagerBlockService.getNoMasterBlock()) + .addGlobalBlock(noClusterManagerBlockService.getNoClusterManagerBlock()) ) .nodes(DiscoveryNodes.builder().add(getLocalNode()).localNodeId(getLocalNode().getId())) .build(); @@ -886,7 +886,7 @@ public void invariant() { assert followersChecker.getFastResponseState().term == getCurrentTerm() : followersChecker.getFastResponseState(); assert followersChecker.getFastResponseState().mode == getMode() : followersChecker.getFastResponseState(); assert (applierState.nodes().getClusterManagerNodeId() == null) == applierState.blocks() - .hasGlobalBlockWithId(NO_MASTER_BLOCK_ID); + .hasGlobalBlockWithId(NO_CLUSTER_MANAGER_BLOCK_ID); assert preVoteCollector.getPreVoteResponse().equals(getPreVoteResponse()) : preVoteCollector + " vs " + getPreVoteResponse(); assert lagDetector.getTrackedNodes().contains(getLocalNode()) == false : lagDetector.getTrackedNodes(); @@ -1221,11 +1221,11 @@ ClusterState getStateForClusterManagerService() { private ClusterState clusterStateWithNoClusterManagerBlock(ClusterState clusterState) { if (clusterState.nodes().getClusterManagerNodeId() != null) { // remove block if it already exists before adding new one - assert clusterState.blocks().hasGlobalBlockWithId(NO_MASTER_BLOCK_ID) == false + assert clusterState.blocks().hasGlobalBlockWithId(NO_CLUSTER_MANAGER_BLOCK_ID) == false : "NO_MASTER_BLOCK should only be added by Coordinator"; final ClusterBlocks clusterBlocks = ClusterBlocks.builder() .blocks(clusterState.blocks()) - .addGlobalBlock(noClusterManagerBlockService.getNoMasterBlock()) + .addGlobalBlock(noClusterManagerBlockService.getNoClusterManagerBlock()) .build(); final DiscoveryNodes discoveryNodes = new DiscoveryNodes.Builder(clusterState.nodes()).clusterManagerNodeId(null).build(); return ClusterState.builder(clusterState).blocks(clusterBlocks).nodes(discoveryNodes).build(); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java index 7582946bb8f3f..5afdb5b12db23 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/JoinTaskExecutor.java @@ -285,7 +285,9 @@ protected ClusterState.Builder becomeClusterManagerAndTrimConflictingNodes(Clust ClusterState tmpState = ClusterState.builder(currentState) .nodes(nodesBuilder) .blocks( - ClusterBlocks.builder().blocks(currentState.blocks()).removeGlobalBlock(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID) + ClusterBlocks.builder() + .blocks(currentState.blocks()) + .removeGlobalBlock(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID) ) .build(); logger.trace("becomeClusterManagerAndTrimConflictingNodes: {}", tmpState.nodes()); diff --git a/server/src/main/java/org/opensearch/cluster/coordination/NoClusterManagerBlockService.java b/server/src/main/java/org/opensearch/cluster/coordination/NoClusterManagerBlockService.java index 44ced46048736..71e9a87cdffae 100644 --- a/server/src/main/java/org/opensearch/cluster/coordination/NoClusterManagerBlockService.java +++ b/server/src/main/java/org/opensearch/cluster/coordination/NoClusterManagerBlockService.java @@ -47,9 +47,9 @@ * @opensearch.internal */ public class NoClusterManagerBlockService { - public static final int NO_MASTER_BLOCK_ID = 2; - public static final ClusterBlock NO_MASTER_BLOCK_WRITES = new ClusterBlock( - NO_MASTER_BLOCK_ID, + public static final int NO_CLUSTER_MANAGER_BLOCK_ID = 2; + public static final ClusterBlock NO_CLUSTER_MANAGER_BLOCK_WRITES = new ClusterBlock( + NO_CLUSTER_MANAGER_BLOCK_ID, "no cluster-manager", true, false, @@ -57,8 +57,8 @@ public class NoClusterManagerBlockService { RestStatus.SERVICE_UNAVAILABLE, EnumSet.of(ClusterBlockLevel.WRITE, ClusterBlockLevel.METADATA_WRITE) ); - public static final ClusterBlock NO_MASTER_BLOCK_ALL = new ClusterBlock( - NO_MASTER_BLOCK_ID, + public static final ClusterBlock NO_CLUSTER_MANAGER_BLOCK_ALL = new ClusterBlock( + NO_CLUSTER_MANAGER_BLOCK_ID, "no cluster-manager", true, true, @@ -66,8 +66,8 @@ public class NoClusterManagerBlockService { RestStatus.SERVICE_UNAVAILABLE, ClusterBlockLevel.ALL ); - public static final ClusterBlock NO_MASTER_BLOCK_METADATA_WRITES = new ClusterBlock( - NO_MASTER_BLOCK_ID, + public static final ClusterBlock NO_CLUSTER_MANAGER_BLOCK_METADATA_WRITES = new ClusterBlock( + NO_CLUSTER_MANAGER_BLOCK_ID, "no cluster-manager", true, false, @@ -76,6 +76,19 @@ public class NoClusterManagerBlockService { EnumSet.of(ClusterBlockLevel.METADATA_WRITE) ); + /** @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #NO_CLUSTER_MANAGER_BLOCK_ID} */ + @Deprecated + public static final int NO_MASTER_BLOCK_ID = NO_CLUSTER_MANAGER_BLOCK_ID; + /** @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #NO_CLUSTER_MANAGER_BLOCK_WRITES} */ + @Deprecated + public static final ClusterBlock NO_MASTER_BLOCK_WRITES = NO_CLUSTER_MANAGER_BLOCK_WRITES; + /** @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #NO_CLUSTER_MANAGER_BLOCK_ALL} */ + @Deprecated + public static final ClusterBlock NO_MASTER_BLOCK_ALL = NO_CLUSTER_MANAGER_BLOCK_ALL; + /** @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #NO_CLUSTER_MANAGER_BLOCK_METADATA_WRITES} */ + @Deprecated + public static final ClusterBlock NO_MASTER_BLOCK_METADATA_WRITES = NO_CLUSTER_MANAGER_BLOCK_METADATA_WRITES; + public static final Setting NO_MASTER_BLOCK_SETTING = new Setting<>( "cluster.no_master_block", "metadata_write", @@ -98,17 +111,17 @@ public class NoClusterManagerBlockService { public NoClusterManagerBlockService(Settings settings, ClusterSettings clusterSettings) { this.noClusterManagerBlock = NO_CLUSTER_MANAGER_BLOCK_SETTING.get(settings); - clusterSettings.addSettingsUpdateConsumer(NO_CLUSTER_MANAGER_BLOCK_SETTING, this::setNoMasterBlock); + clusterSettings.addSettingsUpdateConsumer(NO_CLUSTER_MANAGER_BLOCK_SETTING, this::setNoClusterManagerBlock); } private static ClusterBlock parseNoClusterManagerBlock(String value) { switch (value) { case "all": - return NO_MASTER_BLOCK_ALL; + return NO_CLUSTER_MANAGER_BLOCK_ALL; case "write": - return NO_MASTER_BLOCK_WRITES; + return NO_CLUSTER_MANAGER_BLOCK_WRITES; case "metadata_write": - return NO_MASTER_BLOCK_METADATA_WRITES; + return NO_CLUSTER_MANAGER_BLOCK_METADATA_WRITES; default: throw new IllegalArgumentException( "invalid no-cluster-manager block [" + value + "], must be one of [all, write, metadata_write]" @@ -116,11 +129,17 @@ private static ClusterBlock parseNoClusterManagerBlock(String value) { } } + public ClusterBlock getNoClusterManagerBlock() { + return noClusterManagerBlock; + } + + /** @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #getNoClusterManagerBlock()} */ + @Deprecated public ClusterBlock getNoMasterBlock() { return noClusterManagerBlock; } - private void setNoMasterBlock(ClusterBlock noClusterManagerBlock) { + private void setNoClusterManagerBlock(ClusterBlock noClusterManagerBlock) { this.noClusterManagerBlock = noClusterManagerBlock; } } diff --git a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java index c58bb4d9a947c..17ad8412de525 100644 --- a/server/src/main/java/org/opensearch/cluster/service/ClusterService.java +++ b/server/src/main/java/org/opensearch/cluster/service/ClusterService.java @@ -252,7 +252,7 @@ public ClusterApplierService getClusterApplierService() { public static boolean assertClusterOrClusterManagerStateThread() { assert Thread.currentThread().getName().contains(ClusterApplierService.CLUSTER_UPDATE_THREAD_NAME) - || Thread.currentThread().getName().contains(MasterService.MASTER_UPDATE_THREAD_NAME) + || Thread.currentThread().getName().contains(MasterService.CLUSTER_MANAGER_UPDATE_THREAD_NAME) : "not called from the master/cluster state update thread"; return true; } diff --git a/server/src/main/java/org/opensearch/cluster/service/MasterService.java b/server/src/main/java/org/opensearch/cluster/service/MasterService.java index cc58c56e00dd5..e20d93e76ef01 100644 --- a/server/src/main/java/org/opensearch/cluster/service/MasterService.java +++ b/server/src/main/java/org/opensearch/cluster/service/MasterService.java @@ -107,6 +107,10 @@ public class MasterService extends AbstractLifecycleComponent { Setting.Property.NodeScope ); + static final String CLUSTER_MANAGER_UPDATE_THREAD_NAME = "clusterManagerService#updateTask"; + + /** @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #CLUSTER_MANAGER_UPDATE_THREAD_NAME} */ + @Deprecated static final String MASTER_UPDATE_THREAD_NAME = "masterService#updateTask"; ClusterStatePublisher clusterStatePublisher; @@ -156,8 +160,8 @@ protected synchronized void doStart() { protected PrioritizedOpenSearchThreadPoolExecutor createThreadPoolExecutor() { return OpenSearchExecutors.newSinglePrioritizing( - nodeName + "/" + MASTER_UPDATE_THREAD_NAME, - daemonThreadFactory(nodeName, MASTER_UPDATE_THREAD_NAME), + nodeName + "/" + CLUSTER_MANAGER_UPDATE_THREAD_NAME, + daemonThreadFactory(nodeName, CLUSTER_MANAGER_UPDATE_THREAD_NAME), threadPool.getThreadContext(), threadPool.scheduler() ); @@ -228,24 +232,37 @@ ClusterState state() { return clusterStateSupplier.get(); } - private static boolean isMasterUpdateThread() { - return Thread.currentThread().getName().contains(MASTER_UPDATE_THREAD_NAME); + private static boolean isClusterManagerUpdateThread() { + return Thread.currentThread().getName().contains(CLUSTER_MANAGER_UPDATE_THREAD_NAME) + || Thread.currentThread().getName().contains(MASTER_UPDATE_THREAD_NAME); } - public static boolean assertMasterUpdateThread() { - assert isMasterUpdateThread() : "not called from the cluster-manager service thread"; + public static boolean assertClusterManagerUpdateThread() { + assert isClusterManagerUpdateThread() : "not called from the cluster-manager service thread"; return true; } - public static boolean assertNotMasterUpdateThread(String reason) { - assert isMasterUpdateThread() == false : "Expected current thread [" + public static boolean assertNotClusterManagerUpdateThread(String reason) { + assert isClusterManagerUpdateThread() == false : "Expected current thread [" + Thread.currentThread() - + "] to not be the cluster-maanger service thread. Reason: [" + + "] to not be the cluster-manager service thread. Reason: [" + reason + "]"; return true; } + /** @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #assertClusterManagerUpdateThread()} */ + @Deprecated + public static boolean assertMasterUpdateThread() { + return assertClusterManagerUpdateThread(); + } + + /** @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #assertNotClusterManagerUpdateThread(String)} */ + @Deprecated + public static boolean assertNotMasterUpdateThread(String reason) { + return assertNotClusterManagerUpdateThread(reason); + } + private void runTasks(TaskInputs taskInputs) { final String summary = taskInputs.summary; if (!lifecycle.started()) { @@ -314,7 +331,7 @@ protected void publish(ClusterChangedEvent clusterChangedEvent, TaskOutputs task final PlainActionFuture fut = new PlainActionFuture() { @Override protected boolean blockingAllowed() { - return isMasterUpdateThread() || super.blockingAllowed(); + return isClusterManagerUpdateThread() || super.blockingAllowed(); } }; clusterStatePublisher.publish(clusterChangedEvent, fut, taskOutputs.createAckListener(threadPool, clusterChangedEvent.state())); diff --git a/server/src/main/java/org/opensearch/common/util/concurrent/BaseFuture.java b/server/src/main/java/org/opensearch/common/util/concurrent/BaseFuture.java index fef37299b349d..f394809aaacf7 100644 --- a/server/src/main/java/org/opensearch/common/util/concurrent/BaseFuture.java +++ b/server/src/main/java/org/opensearch/common/util/concurrent/BaseFuture.java @@ -109,7 +109,7 @@ protected boolean blockingAllowed() { return Transports.assertNotTransportThread(BLOCKING_OP_REASON) && ThreadPool.assertNotScheduleThread(BLOCKING_OP_REASON) && ClusterApplierService.assertNotClusterStateUpdateThread(BLOCKING_OP_REASON) - && MasterService.assertNotMasterUpdateThread(BLOCKING_OP_REASON); + && MasterService.assertNotClusterManagerUpdateThread(BLOCKING_OP_REASON); } @Override diff --git a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java index c0d30a92e2d4d..5a93d7c0bd86e 100644 --- a/server/src/test/java/org/opensearch/ExceptionSerializationTests.java +++ b/server/src/test/java/org/opensearch/ExceptionSerializationTests.java @@ -516,9 +516,11 @@ public void testFailedNodeException() throws IOException { } public void testClusterBlockException() throws IOException { - ClusterBlockException ex = serialize(new ClusterBlockException(singleton(NoClusterManagerBlockService.NO_MASTER_BLOCK_WRITES))); + ClusterBlockException ex = serialize( + new ClusterBlockException(singleton(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_WRITES)) + ); assertEquals("blocked by: [SERVICE_UNAVAILABLE/2/no cluster-manager];", ex.getMessage()); - assertTrue(ex.blocks().contains(NoClusterManagerBlockService.NO_MASTER_BLOCK_WRITES)); + assertTrue(ex.blocks().contains(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_WRITES)); assertEquals(1, ex.blocks().size()); } diff --git a/server/src/test/java/org/opensearch/OpenSearchExceptionTests.java b/server/src/test/java/org/opensearch/OpenSearchExceptionTests.java index ca2940c742ec0..bd2695508dfcb 100644 --- a/server/src/test/java/org/opensearch/OpenSearchExceptionTests.java +++ b/server/src/test/java/org/opensearch/OpenSearchExceptionTests.java @@ -478,7 +478,10 @@ public void testToXContentWithHeadersAndMetadata() throws IOException { "foo", new OpenSearchException( "bar", - new OpenSearchException("baz", new ClusterBlockException(singleton(NoClusterManagerBlockService.NO_MASTER_BLOCK_WRITES))) + new OpenSearchException( + "baz", + new ClusterBlockException(singleton(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_WRITES)) + ) ) ); e.addHeader("foo_0", "0"); @@ -1032,7 +1035,7 @@ public static Tuple randomExceptions() { int type = randomIntBetween(0, 5); switch (type) { case 0: - actual = new ClusterBlockException(singleton(NoClusterManagerBlockService.NO_MASTER_BLOCK_WRITES)); + actual = new ClusterBlockException(singleton(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_WRITES)); expected = new OpenSearchException( "OpenSearch exception [type=cluster_block_exception, " + "reason=blocked by: [SERVICE_UNAVAILABLE/2/no cluster-manager];]" diff --git a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java index 91d780e218e71..2ebca16519258 100644 --- a/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java +++ b/server/src/test/java/org/opensearch/action/resync/TransportResyncReplicationActionTests.java @@ -116,7 +116,7 @@ public void testResyncDoesNotBlockOnPrimaryAction() throws Exception { ClusterState.builder(clusterService.state()) .blocks( ClusterBlocks.builder() - .addGlobalBlock(NoClusterManagerBlockService.NO_MASTER_BLOCK_ALL) + .addGlobalBlock(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ALL) .addIndexBlock(indexName, IndexMetadata.INDEX_WRITE_BLOCK) ) ); diff --git a/server/src/test/java/org/opensearch/cluster/InternalClusterInfoServiceSchedulingTests.java b/server/src/test/java/org/opensearch/cluster/InternalClusterInfoServiceSchedulingTests.java index a989bacbf47bb..7a073456af1ba 100644 --- a/server/src/test/java/org/opensearch/cluster/InternalClusterInfoServiceSchedulingTests.java +++ b/server/src/test/java/org/opensearch/cluster/InternalClusterInfoServiceSchedulingTests.java @@ -210,7 +210,9 @@ protected void requestCount++; // ClusterInfoService handles ClusterBlockExceptions quietly, so we invent such an exception to avoid excess logging listener.onFailure( - new ClusterBlockException(org.opensearch.common.collect.Set.of(NoClusterManagerBlockService.NO_MASTER_BLOCK_ALL)) + new ClusterBlockException( + org.opensearch.common.collect.Set.of(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ALL) + ) ); } else { fail("unexpected action: " + action.name()); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/CoordinatorTests.java b/server/src/test/java/org/opensearch/cluster/coordination/CoordinatorTests.java index 9f9de698296b8..d96c972bc6021 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/CoordinatorTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/CoordinatorTests.java @@ -82,10 +82,10 @@ import static org.opensearch.cluster.coordination.LeaderChecker.LEADER_CHECK_INTERVAL_SETTING; import static org.opensearch.cluster.coordination.LeaderChecker.LEADER_CHECK_RETRY_COUNT_SETTING; import static org.opensearch.cluster.coordination.LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING; -import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_MASTER_BLOCK_ALL; -import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_MASTER_BLOCK_METADATA_WRITES; +import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ALL; +import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_METADATA_WRITES; import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_SETTING; -import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_MASTER_BLOCK_WRITES; +import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_WRITES; import static org.opensearch.cluster.coordination.Reconfigurator.CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION; import static org.opensearch.discovery.PeerFinder.DISCOVERY_FIND_PEERS_INTERVAL_SETTING; import static org.opensearch.monitor.StatusInfo.Status.HEALTHY; @@ -1122,19 +1122,19 @@ public void testStayCandidateAfterReceivingFollowerCheckFromKnownClusterManager( } public void testAppliesNoClusterManagerBlockWritesByDefault() { - testAppliesNoClusterManagerBlock(null, NO_MASTER_BLOCK_WRITES); + testAppliesNoClusterManagerBlock(null, NO_CLUSTER_MANAGER_BLOCK_WRITES); } public void testAppliesNoClusterManagerBlockWritesIfConfigured() { - testAppliesNoClusterManagerBlock("write", NO_MASTER_BLOCK_WRITES); + testAppliesNoClusterManagerBlock("write", NO_CLUSTER_MANAGER_BLOCK_WRITES); } public void testAppliesNoClusterManagerBlockAllIfConfigured() { - testAppliesNoClusterManagerBlock("all", NO_MASTER_BLOCK_ALL); + testAppliesNoClusterManagerBlock("all", NO_CLUSTER_MANAGER_BLOCK_ALL); } public void testAppliesNoClusterManagerBlockMetadataWritesIfConfigured() { - testAppliesNoClusterManagerBlock("metadata_write", NO_MASTER_BLOCK_METADATA_WRITES); + testAppliesNoClusterManagerBlock("metadata_write", NO_CLUSTER_MANAGER_BLOCK_METADATA_WRITES); } private void testAppliesNoClusterManagerBlock(String noClusterManagerBlockSetting, ClusterBlock expectedBlock) { diff --git a/server/src/test/java/org/opensearch/cluster/coordination/NoClusterManagerBlockServiceRenamedSettingTests.java b/server/src/test/java/org/opensearch/cluster/coordination/NoClusterManagerBlockServiceRenamedSettingTests.java index 42a230d05cbd4..8a85544299618 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/NoClusterManagerBlockServiceRenamedSettingTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/NoClusterManagerBlockServiceRenamedSettingTests.java @@ -56,7 +56,7 @@ public void testSettingFallback() { public void testSettingGetValue() { Settings settings = Settings.builder().put("cluster.no_cluster_manager_block", "all").build(); assertEquals( - NoClusterManagerBlockService.NO_MASTER_BLOCK_ALL, + NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ALL, NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_SETTING.get(settings) ); assertEquals( @@ -71,7 +71,7 @@ public void testSettingGetValue() { public void testSettingGetValueWithFallback() { Settings settings = Settings.builder().put("cluster.no_master_block", "metadata_write").build(); assertEquals( - NoClusterManagerBlockService.NO_MASTER_BLOCK_METADATA_WRITES, + NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_METADATA_WRITES, NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_SETTING.get(settings) ); assertSettingDeprecationsAndWarnings(new Setting[] { NoClusterManagerBlockService.NO_MASTER_BLOCK_SETTING }); @@ -86,11 +86,11 @@ public void testSettingGetValueWhenBothAreConfigured() { .put("cluster.no_master_block", "metadata_write") .build(); assertEquals( - NoClusterManagerBlockService.NO_MASTER_BLOCK_ALL, + NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ALL, NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_SETTING.get(settings) ); assertEquals( - NoClusterManagerBlockService.NO_MASTER_BLOCK_METADATA_WRITES, + NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_METADATA_WRITES, NoClusterManagerBlockService.NO_MASTER_BLOCK_SETTING.get(settings) ); assertSettingDeprecationsAndWarnings(new Setting[] { NoClusterManagerBlockService.NO_MASTER_BLOCK_SETTING }); diff --git a/server/src/test/java/org/opensearch/cluster/coordination/NoClusterManagerBlockServiceTests.java b/server/src/test/java/org/opensearch/cluster/coordination/NoClusterManagerBlockServiceTests.java index b9a72e00aebb0..d9ff21204387d 100644 --- a/server/src/test/java/org/opensearch/cluster/coordination/NoClusterManagerBlockServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/coordination/NoClusterManagerBlockServiceTests.java @@ -35,10 +35,10 @@ import org.opensearch.common.settings.Settings; import org.opensearch.test.OpenSearchTestCase; -import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_MASTER_BLOCK_ALL; -import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_MASTER_BLOCK_METADATA_WRITES; +import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ALL; +import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_METADATA_WRITES; import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_SETTING; -import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_MASTER_BLOCK_WRITES; +import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_WRITES; import static org.opensearch.common.settings.ClusterSettings.BUILT_IN_CLUSTER_SETTINGS; import static org.hamcrest.Matchers.sameInstance; @@ -61,22 +61,22 @@ private void assertDeprecatedWarningEmitted() { public void testBlocksWritesByDefault() { createService(Settings.EMPTY); - assertThat(noClusterManagerBlockService.getNoMasterBlock(), sameInstance(NO_MASTER_BLOCK_METADATA_WRITES)); + assertThat(noClusterManagerBlockService.getNoClusterManagerBlock(), sameInstance(NO_CLUSTER_MANAGER_BLOCK_METADATA_WRITES)); } public void testBlocksWritesIfConfiguredBySetting() { createService(Settings.builder().put(NO_CLUSTER_MANAGER_BLOCK_SETTING.getKey(), "write").build()); - assertThat(noClusterManagerBlockService.getNoMasterBlock(), sameInstance(NO_MASTER_BLOCK_WRITES)); + assertThat(noClusterManagerBlockService.getNoClusterManagerBlock(), sameInstance(NO_CLUSTER_MANAGER_BLOCK_WRITES)); } public void testBlocksAllIfConfiguredBySetting() { createService(Settings.builder().put(NO_CLUSTER_MANAGER_BLOCK_SETTING.getKey(), "all").build()); - assertThat(noClusterManagerBlockService.getNoMasterBlock(), sameInstance(NO_MASTER_BLOCK_ALL)); + assertThat(noClusterManagerBlockService.getNoClusterManagerBlock(), sameInstance(NO_CLUSTER_MANAGER_BLOCK_ALL)); } public void testBlocksMetadataWritesIfConfiguredBySetting() { createService(Settings.builder().put(NO_CLUSTER_MANAGER_BLOCK_SETTING.getKey(), "metadata_write").build()); - assertThat(noClusterManagerBlockService.getNoMasterBlock(), sameInstance(NO_MASTER_BLOCK_METADATA_WRITES)); + assertThat(noClusterManagerBlockService.getNoClusterManagerBlock(), sameInstance(NO_CLUSTER_MANAGER_BLOCK_METADATA_WRITES)); } public void testRejectsInvalidSetting() { @@ -88,12 +88,12 @@ public void testRejectsInvalidSetting() { public void testSettingCanBeUpdated() { createService(Settings.builder().put(NO_CLUSTER_MANAGER_BLOCK_SETTING.getKey(), "all").build()); - assertThat(noClusterManagerBlockService.getNoMasterBlock(), sameInstance(NO_MASTER_BLOCK_ALL)); + assertThat(noClusterManagerBlockService.getNoClusterManagerBlock(), sameInstance(NO_CLUSTER_MANAGER_BLOCK_ALL)); clusterSettings.applySettings(Settings.builder().put(NO_CLUSTER_MANAGER_BLOCK_SETTING.getKey(), "write").build()); - assertThat(noClusterManagerBlockService.getNoMasterBlock(), sameInstance(NO_MASTER_BLOCK_WRITES)); + assertThat(noClusterManagerBlockService.getNoClusterManagerBlock(), sameInstance(NO_CLUSTER_MANAGER_BLOCK_WRITES)); clusterSettings.applySettings(Settings.builder().put(NO_CLUSTER_MANAGER_BLOCK_SETTING.getKey(), "metadata_write").build()); - assertThat(noClusterManagerBlockService.getNoMasterBlock(), sameInstance(NO_MASTER_BLOCK_METADATA_WRITES)); + assertThat(noClusterManagerBlockService.getNoClusterManagerBlock(), sameInstance(NO_CLUSTER_MANAGER_BLOCK_METADATA_WRITES)); } } diff --git a/server/src/test/java/org/opensearch/cluster/service/ClusterApplierServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/ClusterApplierServiceTests.java index 830a7d2a557f2..62dae0622eb85 100644 --- a/server/src/test/java/org/opensearch/cluster/service/ClusterApplierServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/service/ClusterApplierServiceTests.java @@ -319,7 +319,7 @@ public void offMaster() { nodes = state.nodes(); nodesBuilder = DiscoveryNodes.builder(nodes).clusterManagerNodeId(null); state = ClusterState.builder(state) - .blocks(ClusterBlocks.builder().addGlobalBlock(NoClusterManagerBlockService.NO_MASTER_BLOCK_WRITES)) + .blocks(ClusterBlocks.builder().addGlobalBlock(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_WRITES)) .nodes(nodesBuilder) .build(); setState(timedClusterApplierService, state); diff --git a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java index 8827064974a19..e39520ce72568 100644 --- a/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/service/MasterServiceTests.java @@ -92,6 +92,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; +import static org.hamcrest.Matchers.is; public class MasterServiceTests extends OpenSearchTestCase { @@ -1098,6 +1099,18 @@ public void onAckTimeout() { } } + public void testDeprecatedMasterServiceUpdateTaskThreadName() { + Thread.currentThread().setName(MasterService.MASTER_UPDATE_THREAD_NAME); + assertThat(MasterService.assertClusterManagerUpdateThread(), is(Boolean.TRUE)); + assertThrows(AssertionError.class, () -> MasterService.assertNotClusterManagerUpdateThread("test")); + Thread.currentThread().setName(MasterService.CLUSTER_MANAGER_UPDATE_THREAD_NAME); + assertThat(MasterService.assertClusterManagerUpdateThread(), is(Boolean.TRUE)); + assertThrows(AssertionError.class, () -> MasterService.assertNotClusterManagerUpdateThread("test")); + Thread.currentThread().setName("test not cluster manager update thread"); + assertThat(MasterService.assertNotClusterManagerUpdateThread("test"), is(Boolean.TRUE)); + assertThrows(AssertionError.class, () -> MasterService.assertClusterManagerUpdateThread()); + } + /** * Returns the cluster state that the cluster-manager service uses (and that is provided by the discovery layer) */ diff --git a/server/src/test/java/org/opensearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java b/server/src/test/java/org/opensearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java index 939e8b2d4e782..1f2360abde2ad 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java +++ b/server/src/test/java/org/opensearch/indices/cluster/IndicesClusterStateServiceRandomUpdatesTests.java @@ -370,22 +370,26 @@ public ClusterState randomlyUpdateClusterState( Supplier indicesServiceSupplier ) { // randomly remove no_cluster_manager blocks - if (randomBoolean() && state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID)) { + if (randomBoolean() && state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID)) { state = ClusterState.builder(state) - .blocks(ClusterBlocks.builder().blocks(state.blocks()).removeGlobalBlock(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID)) + .blocks( + ClusterBlocks.builder() + .blocks(state.blocks()) + .removeGlobalBlock(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID) + ) .build(); } // randomly add no_cluster_manager blocks - if (rarely() && state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID) == false) { + if (rarely() && state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID) == false) { ClusterBlock block = randomBoolean() - ? NoClusterManagerBlockService.NO_MASTER_BLOCK_ALL - : NoClusterManagerBlockService.NO_MASTER_BLOCK_WRITES; + ? NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ALL + : NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_WRITES; state = ClusterState.builder(state).blocks(ClusterBlocks.builder().blocks(state.blocks()).addGlobalBlock(block)).build(); } // if no_cluster_manager block is in place, make no other cluster state changes - if (state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_MASTER_BLOCK_ID)) { + if (state.blocks().hasGlobalBlockWithId(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID)) { return state; } diff --git a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java index 36469b2ee1999..d6ec70238f70d 100644 --- a/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java +++ b/test/framework/src/main/java/org/opensearch/cluster/coordination/AbstractCoordinatorTestCase.java @@ -139,7 +139,7 @@ import static org.opensearch.cluster.coordination.LeaderChecker.LEADER_CHECK_INTERVAL_SETTING; import static org.opensearch.cluster.coordination.LeaderChecker.LEADER_CHECK_RETRY_COUNT_SETTING; import static org.opensearch.cluster.coordination.LeaderChecker.LEADER_CHECK_TIMEOUT_SETTING; -import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_MASTER_BLOCK_ID; +import static org.opensearch.cluster.coordination.NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_ID; import static org.opensearch.cluster.coordination.Reconfigurator.CLUSTER_AUTO_SHRINK_VOTING_CONFIGURATION; import static org.opensearch.discovery.PeerFinder.DISCOVERY_FIND_PEERS_INTERVAL_SETTING; import static org.opensearch.gateway.GatewayService.STATE_NOT_RECOVERED_BLOCK; @@ -567,7 +567,7 @@ void stabilise(long stabilisationDurationMillis) { assertTrue(leaderId + " exists in its last-applied state", leader.getLastAppliedClusterState().getNodes().nodeExists(leaderId)); assertThat( leaderId + " has no NO_CLUSTER_MANAGER_BLOCK", - leader.getLastAppliedClusterState().blocks().hasGlobalBlockWithId(NO_MASTER_BLOCK_ID), + leader.getLastAppliedClusterState().blocks().hasGlobalBlockWithId(NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(false) ); assertThat( @@ -622,7 +622,7 @@ void stabilise(long stabilisationDurationMillis) { ); assertThat( nodeId + " has no NO_CLUSTER_MANAGER_BLOCK", - clusterNode.getLastAppliedClusterState().blocks().hasGlobalBlockWithId(NO_MASTER_BLOCK_ID), + clusterNode.getLastAppliedClusterState().blocks().hasGlobalBlockWithId(NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(false) ); assertThat( @@ -639,7 +639,7 @@ void stabilise(long stabilisationDurationMillis) { ); assertThat( nodeId + " has NO_CLUSTER_MANAGER_BLOCK", - clusterNode.getLastAppliedClusterState().blocks().hasGlobalBlockWithId(NO_MASTER_BLOCK_ID), + clusterNode.getLastAppliedClusterState().blocks().hasGlobalBlockWithId(NO_CLUSTER_MANAGER_BLOCK_ID), equalTo(true) ); assertFalse( diff --git a/test/framework/src/main/java/org/opensearch/test/RandomObjects.java b/test/framework/src/main/java/org/opensearch/test/RandomObjects.java index a0227c51c8085..1f3500dccce8f 100644 --- a/test/framework/src/main/java/org/opensearch/test/RandomObjects.java +++ b/test/framework/src/main/java/org/opensearch/test/RandomObjects.java @@ -331,7 +331,7 @@ private static Tuple randomShardInfoFailure(Random random) { int type = randomIntBetween(random, 0, 3); switch (type) { case 0: - actualException = new ClusterBlockException(singleton(NoClusterManagerBlockService.NO_MASTER_BLOCK_WRITES)); + actualException = new ClusterBlockException(singleton(NoClusterManagerBlockService.NO_CLUSTER_MANAGER_BLOCK_WRITES)); expectedException = new OpenSearchException( "OpenSearch exception [type=cluster_block_exception, " + "reason=blocked by: [SERVICE_UNAVAILABLE/2/no cluster-manager];]" From 82ff463fd1bc07c241c8674b72182f2875a60425 Mon Sep 17 00:00:00 2001 From: Gaurav Bafna <85113518+gbbafna@users.noreply.github.com> Date: Fri, 29 Jul 2022 15:21:56 +0530 Subject: [PATCH 5/6] Making shard copy count a multiple of attribute count (#3462) * Making all copies a multiple of attribute count Signed-off-by: Gaurav Bafna --- .../admin/indices/rollover/RolloverIT.java | 43 +++++++ .../allocation/AwarenessAllocationIT.java | 8 ++ .../settings/UpdateNumberOfReplicasIT.java | 46 ++++++- .../template/SimpleIndexTemplateIT.java | 47 ++++++-- .../snapshots/RestoreSnapshotIT.java | 66 ++++++++-- .../metadata/MetadataCreateIndexService.java | 31 ++++- .../MetadataIndexTemplateService.java | 6 +- .../MetadataUpdateSettingsService.java | 19 ++- .../allocation/AwarenessReplicaBalance.java | 114 ++++++++++++++++++ .../common/settings/ClusterSettings.java | 2 + .../main/java/org/opensearch/node/Node.java | 9 +- .../MetadataRolloverServiceTests.java | 10 +- .../MetadataCreateIndexServiceTests.java | 65 +++++++++- .../MetadataIndexTemplateServiceTests.java | 18 ++- .../allocation/AwarenessAllocationTests.java | 1 + .../AwarenessReplicaBalanceTests.java | 66 ++++++++++ .../indices/cluster/ClusterStateChanges.java | 11 +- .../snapshots/SnapshotResiliencyTests.java | 4 +- .../test/OpenSearchIntegTestCase.java | 23 ++++ 19 files changed, 551 insertions(+), 38 deletions(-) create mode 100644 server/src/main/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalance.java create mode 100644 server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalanceTests.java diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/rollover/RolloverIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/rollover/RolloverIT.java index 178dc2c6ffa87..d0ff3ef19a028 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/rollover/RolloverIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/indices/rollover/RolloverIT.java @@ -61,6 +61,7 @@ import java.util.List; import java.util.Set; +import static org.hamcrest.Matchers.containsString; import static org.opensearch.index.mapper.MapperService.SINGLE_MAPPING_NAME; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.hamcrest.Matchers.containsInAnyOrder; @@ -172,6 +173,7 @@ public void testRolloverWithNoWriteIndex() { } public void testRolloverWithIndexSettings() throws Exception { + Alias testAlias = new Alias("test_alias"); boolean explicitWriteIndex = randomBoolean(); if (explicitWriteIndex) { @@ -210,6 +212,47 @@ public void testRolloverWithIndexSettings() throws Exception { } } + public void testRolloverWithIndexSettingsBalancedReplica() throws Exception { + Alias testAlias = new Alias("test_alias"); + boolean explicitWriteIndex = randomBoolean(); + if (explicitWriteIndex) { + testAlias.writeIndex(true); + } + assertAcked(prepareCreate("test_index-2").addAlias(testAlias).get()); + manageReplicaBalanceSetting(true); + index("test_index-2", "type1", "1", "field", "value"); + flush("test_index-2"); + final Settings settings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0) + .build(); + + final IllegalArgumentException restoreError = expectThrows( + IllegalArgumentException.class, + () -> client().admin().indices().prepareRolloverIndex("test_alias").settings(settings).alias(new Alias("extra_alias")).get() + ); + + assertThat( + restoreError.getMessage(), + containsString("expected total copies needs to be a multiple of total awareness attributes [2]") + ); + + final Settings balancedReplicaSettings = Settings.builder() + .put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1) + .put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1) + .build(); + + client().admin() + .indices() + .prepareRolloverIndex("test_alias") + .settings(balancedReplicaSettings) + .alias(new Alias("extra_alias")) + .waitForActiveShards(0) + .get(); + + manageReplicaBalanceSetting(false); + } + public void testRolloverWithIndexSettingsWithoutPrefix() throws Exception { Alias testAlias = new Alias("test_alias"); boolean explicitWriteIndex = randomBoolean(); diff --git a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java index 87d15bf9d3750..a8d4d016ce288 100644 --- a/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/cluster/allocation/AwarenessAllocationIT.java @@ -36,12 +36,14 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.opensearch.action.admin.cluster.health.ClusterHealthResponse; +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.cluster.metadata.IndexMetadata.State; import org.opensearch.cluster.routing.IndexRoutingTable; import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.ShardRouting; +import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; import org.opensearch.common.Priority; import org.opensearch.common.settings.Settings; @@ -77,6 +79,12 @@ public void testSimpleAwareness() throws Exception { logger.info("--> starting 2 nodes on the same rack"); internalCluster().startNodes(2, Settings.builder().put(commonSettings).put("node.attr.rack_id", "rack_1").build()); + Settings settings = Settings.builder() + .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), false) + .build(); + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings(settings); + createIndex("test1"); createIndex("test2"); diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateNumberOfReplicasIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateNumberOfReplicasIT.java index f78ecd82834c2..98001b447e8b2 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateNumberOfReplicasIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/settings/UpdateNumberOfReplicasIT.java @@ -45,11 +45,11 @@ import java.io.IOException; import java.util.EnumSet; +import static org.hamcrest.Matchers.equalTo; import static org.opensearch.common.xcontent.XContentFactory.jsonBuilder; import static org.opensearch.index.query.QueryBuilders.matchAllQuery; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; -import static org.hamcrest.Matchers.equalTo; @OpenSearchIntegTestCase.ClusterScope(minNumDataNodes = 2) public class UpdateNumberOfReplicasIT extends OpenSearchIntegTestCase { @@ -606,4 +606,48 @@ public void testUpdateNumberOfReplicasAllowNoIndices() { assertThat(numberOfReplicas, equalTo(0)); } + public void testAwarenessReplicaBalance() { + createIndex("aware-replica", Settings.builder().put("index.number_of_replicas", 0).build()); + createIndex(".system-index", Settings.builder().put("index.number_of_replicas", 0).build()); + manageReplicaBalanceSetting(true); + int updated = 0; + + try { + // replica count of 1 is ideal + client().admin() + .indices() + .prepareUpdateSettings("aware-replica") + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)) + .execute() + .actionGet(); + updated++; + + // system index - should be able to update + client().admin() + .indices() + .prepareUpdateSettings(".system-index") + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2)) + .execute() + .actionGet(); + updated++; + + client().admin() + .indices() + .prepareUpdateSettings("aware-replica") + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 2)) + .execute() + .actionGet(); + fail("should have thrown an exception about the replica count"); + + } catch (IllegalArgumentException e) { + assertEquals( + "Validation Failed: 1: expected total copies needs to be a multiple of total awareness attributes [2];", + e.getMessage() + ); + assertEquals(2, updated); + } finally { + manageReplicaBalanceSetting(false); + } + } + } diff --git a/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java b/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java index 0e15a0c895432..42c0145676f2d 100644 --- a/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/indices/template/SimpleIndexTemplateIT.java @@ -32,18 +32,19 @@ package org.opensearch.indices.template; +import org.junit.After; import org.opensearch.action.ActionRequestValidationException; import org.opensearch.action.admin.indices.alias.Alias; import org.opensearch.action.admin.indices.alias.get.GetAliasesResponse; import org.opensearch.action.admin.indices.settings.get.GetSettingsResponse; import org.opensearch.action.admin.indices.template.get.GetIndexTemplatesResponse; import org.opensearch.action.admin.indices.template.put.PutIndexTemplateRequestBuilder; - import org.opensearch.action.bulk.BulkResponse; import org.opensearch.action.index.IndexRequest; import org.opensearch.action.search.SearchResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.AliasMetadata; +import org.opensearch.cluster.metadata.IndexMetadata; import org.opensearch.common.ParsingException; import org.opensearch.common.bytes.BytesArray; import org.opensearch.common.settings.Settings; @@ -52,12 +53,11 @@ import org.opensearch.index.mapper.MapperParsingException; import org.opensearch.index.query.QueryBuilders; import org.opensearch.indices.InvalidAliasNameException; +import org.opensearch.indices.InvalidIndexTemplateException; import org.opensearch.plugins.Plugin; import org.opensearch.search.SearchHit; -import org.opensearch.test.OpenSearchIntegTestCase; import org.opensearch.test.InternalSettingsPlugin; - -import org.junit.After; +import org.opensearch.test.OpenSearchIntegTestCase; import java.io.IOException; import java.util.ArrayList; @@ -68,11 +68,6 @@ import java.util.List; import java.util.Set; -import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; -import static org.opensearch.index.query.QueryBuilders.termQuery; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; -import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertRequestBuilderThrows; import static org.hamcrest.Matchers.anyOf; import static org.hamcrest.Matchers.containsInAnyOrder; import static org.hamcrest.Matchers.containsString; @@ -83,6 +78,11 @@ import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; +import static org.opensearch.action.support.WriteRequest.RefreshPolicy.IMMEDIATE; +import static org.opensearch.index.query.QueryBuilders.termQuery; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertAcked; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertHitCount; +import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertRequestBuilderThrows; public class SimpleIndexTemplateIT extends OpenSearchIntegTestCase { @@ -1029,4 +1029,33 @@ public void testPartitionedTemplate() throws Exception { GetSettingsResponse getSettingsResponse = client().admin().indices().prepareGetSettings("test_good").get(); assertEquals("6", getSettingsResponse.getIndexToSettings().get("test_good").get("index.routing_partition_size")); } + + public void testAwarenessReplicaBalance() throws IOException { + manageReplicaBalanceSetting(true); + try { + client().admin() + .indices() + .preparePutTemplate("template_1") + .setPatterns(Arrays.asList("a*", "b*")) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1)) + .get(); + + client().admin() + .indices() + .preparePutTemplate("template_1") + .setPatterns(Arrays.asList("a*", "b*")) + .setSettings(Settings.builder().put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0)) + .get(); + + fail("should have thrown an exception about the replica count"); + } catch (InvalidIndexTemplateException e) { + assertEquals( + "index_template [template_1] invalid, cause [Validation Failed: 1: expected total copies needs to be a multiple of total awareness attributes [2];]", + e.getMessage() + ); + } finally { + manageReplicaBalanceSetting(false); + } + } + } diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java index 3a7fdd4093657..a40378b9c2dfa 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RestoreSnapshotIT.java @@ -59,6 +59,14 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThan; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.not; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_REPLICAS; import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_NUMBER_OF_SHARDS; import static org.opensearch.index.IndexSettings.INDEX_REFRESH_INTERVAL_SETTING; @@ -70,14 +78,6 @@ import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertIndexTemplateExists; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertIndexTemplateMissing; import static org.opensearch.test.hamcrest.OpenSearchAssertions.assertRequestBuilderThrows; -import static org.hamcrest.Matchers.allOf; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThan; -import static org.hamcrest.Matchers.is; -import static org.hamcrest.Matchers.not; -import static org.hamcrest.Matchers.notNullValue; -import static org.hamcrest.Matchers.nullValue; public class RestoreSnapshotIT extends AbstractSnapshotIntegTestCase { @@ -973,4 +973,54 @@ public void testForbidDisableSoftDeletesDuringRestore() throws Exception { ); assertThat(restoreError.getMessage(), containsString("cannot disable setting [index.soft_deletes.enabled] on restore")); } + + public void testRestoreBalancedReplica() { + try { + createRepository("test-repo", "fs"); + createIndex("test-index", Settings.builder().put("index.number_of_replicas", 0).build()); + createIndex(".system-index", Settings.builder().put("index.number_of_replicas", 0).build()); + ensureGreen(); + clusterAdmin().prepareCreateSnapshot("test-repo", "snapshot-0") + .setIndices("test-index", ".system-index") + .setWaitForCompletion(true) + .get(); + manageReplicaBalanceSetting(true); + + final IllegalArgumentException restoreError = expectThrows( + IllegalArgumentException.class, + () -> clusterAdmin().prepareRestoreSnapshot("test-repo", "snapshot-0") + .setRenamePattern("test-index") + .setRenameReplacement("new-index") + .setIndices("test-index") + .get() + ); + assertThat( + restoreError.getMessage(), + containsString("expected total copies needs to be a multiple of total awareness attributes [2]") + ); + + RestoreSnapshotResponse restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "snapshot-0") + .setRenamePattern(".system-index") + .setRenameReplacement(".system-index-restore-1") + .setWaitForCompletion(true) + .setIndices(".system-index") + .execute() + .actionGet(); + + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + + restoreSnapshotResponse = clusterAdmin().prepareRestoreSnapshot("test-repo", "snapshot-0") + .setRenamePattern("test-index") + .setRenameReplacement("new-index") + .setIndexSettings(Settings.builder().put("index.number_of_replicas", 1).build()) + .setWaitForCompletion(true) + .setIndices("test-index") + .execute() + .actionGet(); + + assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0)); + } finally { + manageReplicaBalanceSetting(false); + } + } } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java index 642b0f7b8d36f..d78e5e872fd2b 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataCreateIndexService.java @@ -59,6 +59,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.ShardRoutingState; import org.opensearch.cluster.routing.allocation.AllocationService; +import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; import org.opensearch.common.Priority; @@ -102,6 +103,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicInteger; import java.util.function.BiConsumer; @@ -145,6 +147,7 @@ public class MetadataCreateIndexService { private final ShardLimitValidator shardLimitValidator; private final boolean forbidPrivateIndexSettings; private final Set indexSettingProviders = new HashSet<>(); + private AwarenessReplicaBalance awarenessReplicaBalance; public MetadataCreateIndexService( final Settings settings, @@ -158,7 +161,8 @@ public MetadataCreateIndexService( final ThreadPool threadPool, final NamedXContentRegistry xContentRegistry, final SystemIndices systemIndices, - final boolean forbidPrivateIndexSettings + final boolean forbidPrivateIndexSettings, + final AwarenessReplicaBalance awarenessReplicaBalance ) { this.settings = settings; this.clusterService = clusterService; @@ -172,6 +176,7 @@ public MetadataCreateIndexService( this.systemIndices = systemIndices; this.forbidPrivateIndexSettings = forbidPrivateIndexSettings; this.shardLimitValidator = shardLimitValidator; + this.awarenessReplicaBalance = awarenessReplicaBalance; } /** @@ -1150,7 +1155,7 @@ private void validate(CreateIndexClusterStateUpdateRequest request, ClusterState public void validateIndexSettings(String indexName, final Settings settings, final boolean forbidPrivateIndexSettings) throws IndexCreationException { - List validationErrors = getIndexSettingsValidationErrors(settings, forbidPrivateIndexSettings); + List validationErrors = getIndexSettingsValidationErrors(settings, forbidPrivateIndexSettings, indexName); if (validationErrors.isEmpty() == false) { ValidationException validationException = new ValidationException(); @@ -1159,11 +1164,31 @@ public void validateIndexSettings(String indexName, final Settings settings, fin } } - List getIndexSettingsValidationErrors(final Settings settings, final boolean forbidPrivateIndexSettings) { + List getIndexSettingsValidationErrors(final Settings settings, final boolean forbidPrivateIndexSettings, String indexName) { + List validationErrors = getIndexSettingsValidationErrors(settings, forbidPrivateIndexSettings, Optional.of(indexName)); + return validationErrors; + } + + List getIndexSettingsValidationErrors( + final Settings settings, + final boolean forbidPrivateIndexSettings, + Optional indexName + ) { List validationErrors = validateIndexCustomPath(settings, env.sharedDataDir()); if (forbidPrivateIndexSettings) { validationErrors.addAll(validatePrivateSettingsNotExplicitlySet(settings, indexScopedSettings)); } + if (indexName.isEmpty() || indexName.get().charAt(0) != '.') { + // Apply aware replica balance only to non system indices + int replicaCount = settings.getAsInt( + IndexMetadata.SETTING_NUMBER_OF_REPLICAS, + INDEX_NUMBER_OF_REPLICAS_SETTING.getDefault(Settings.EMPTY) + ); + Optional error = awarenessReplicaBalance.validate(replicaCount); + if (error.isPresent()) { + validationErrors.add(error.get()); + } + } return validationErrors; } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java index 1a0f4f0f83e00..7e91b491a234c 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataIndexTemplateService.java @@ -1471,7 +1471,11 @@ private void validate(String name, @Nullable Settings settings, List ind validationErrors.add(t.getMessage()); } } - List indexSettingsValidation = metadataCreateIndexService.getIndexSettingsValidationErrors(settings, true); + List indexSettingsValidation = metadataCreateIndexService.getIndexSettingsValidationErrors( + settings, + true, + Optional.empty() + ); validationErrors.addAll(indexSettingsValidation); } diff --git a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java index 1390860271577..eb142be815d27 100644 --- a/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java +++ b/server/src/main/java/org/opensearch/cluster/metadata/MetadataUpdateSettingsService.java @@ -46,6 +46,7 @@ import org.opensearch.cluster.block.ClusterBlocks; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.allocation.AllocationService; +import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Priority; import org.opensearch.common.ValidationException; @@ -89,6 +90,8 @@ public class MetadataUpdateSettingsService { private final ShardLimitValidator shardLimitValidator; private final ThreadPool threadPool; + private AwarenessReplicaBalance awarenessReplicaBalance; + @Inject public MetadataUpdateSettingsService( ClusterService clusterService, @@ -96,7 +99,8 @@ public MetadataUpdateSettingsService( IndexScopedSettings indexScopedSettings, IndicesService indicesService, ShardLimitValidator shardLimitValidator, - ThreadPool threadPool + ThreadPool threadPool, + AwarenessReplicaBalance awarenessReplicaBalance ) { this.clusterService = clusterService; this.threadPool = threadPool; @@ -104,6 +108,7 @@ public MetadataUpdateSettingsService( this.indexScopedSettings = indexScopedSettings; this.indicesService = indicesService; this.shardLimitValidator = shardLimitValidator; + this.awarenessReplicaBalance = awarenessReplicaBalance; } public void updateSettings( @@ -193,6 +198,18 @@ public ClusterState execute(ClusterState currentState) { if (IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.exists(openSettings)) { final int updatedNumberOfReplicas = IndexMetadata.INDEX_NUMBER_OF_REPLICAS_SETTING.get(openSettings); if (preserveExisting == false) { + for (Index index : request.indices()) { + if (index.getName().charAt(0) != '.') { + // No replica count validation for system indices + Optional error = awarenessReplicaBalance.validate(updatedNumberOfReplicas); + if (error.isPresent()) { + ValidationException ex = new ValidationException(); + ex.addValidationError(error.get()); + throw ex; + } + } + } + // Verify that this won't take us over the cluster shard limit. int totalNewShards = Arrays.stream(request.indices()) .mapToInt(i -> getTotalNewShards(i, currentState, updatedNumberOfReplicas)) diff --git a/server/src/main/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalance.java b/server/src/main/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalance.java new file mode 100644 index 0000000000000..accf0b69a4f0e --- /dev/null +++ b/server/src/main/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalance.java @@ -0,0 +1,114 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.allocation; + +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Setting; +import org.opensearch.common.settings.Settings; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; + +import static java.lang.Math.max; +import static org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING; +import static org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING; + +/** + * This {@link AwarenessReplicaBalance} gives total unique values of awareness attributes + * It takes in effect only iff cluster.routing.allocation.awareness.attributes and + * cluster.routing.allocation.awareness.force.zone.values both are specified. + * + * This is used in enforcing total copy of shard is a maximum of unique values of awareness attributes + * Helps in balancing shards across all awareness attributes and ensuring high availability of data. + */ +public class AwarenessReplicaBalance { + public static final Setting CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING = Setting.boolSetting( + "cluster.routing.allocation.awareness.balance", + false, + Setting.Property.Dynamic, + Setting.Property.NodeScope + ); + + private volatile List awarenessAttributes; + + private volatile Map> forcedAwarenessAttributes; + + private volatile Boolean awarenessBalance; + + public AwarenessReplicaBalance(Settings settings, ClusterSettings clusterSettings) { + this.awarenessAttributes = CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.get(settings); + clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING, this::setAwarenessAttributes); + setForcedAwarenessAttributes(CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.get(settings)); + clusterSettings.addSettingsUpdateConsumer( + CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING, + this::setForcedAwarenessAttributes + ); + setAwarenessBalance(CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.get(settings)); + clusterSettings.addSettingsUpdateConsumer(CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING, this::setAwarenessBalance); + + } + + private void setAwarenessBalance(Boolean awarenessBalance) { + this.awarenessBalance = awarenessBalance; + } + + private void setForcedAwarenessAttributes(Settings forceSettings) { + Map> forcedAwarenessAttributes = new HashMap<>(); + Map forceGroups = forceSettings.getAsGroups(); + for (Map.Entry entry : forceGroups.entrySet()) { + List aValues = entry.getValue().getAsList("values"); + if (aValues.size() > 0) { + forcedAwarenessAttributes.put(entry.getKey(), aValues); + } + } + this.forcedAwarenessAttributes = forcedAwarenessAttributes; + } + + private void setAwarenessAttributes(List awarenessAttributes) { + this.awarenessAttributes = awarenessAttributes; + } + + /* + For a cluster having zone as awareness attribute , it will return the size of zones if set it forced awareness attributes + + If there are multiple forced awareness attributes, it will return size of the largest list, as all copies of data + is supposed to get distributed amongst those. + + cluster.routing.allocation.awareness.attributes: rack_id , zone + cluster.routing.allocation.awareness.force.zone.values: zone1, zone2 + cluster.routing.allocation.awareness.force.rack_id.values: rack_id1, rack_id2, rack_id3 + + In this case, awareness attributes would be 3. + */ + public int maxAwarenessAttributes() { + int awarenessAttributes = 1; + if (this.awarenessBalance == false) { + return awarenessAttributes; + } + for (String awarenessAttribute : this.awarenessAttributes) { + if (forcedAwarenessAttributes.containsKey(awarenessAttribute)) { + awarenessAttributes = max(awarenessAttributes, forcedAwarenessAttributes.get(awarenessAttribute).size()); + } + } + return awarenessAttributes; + } + + public Optional validate(int replicaCount) { + if ((replicaCount + 1) % maxAwarenessAttributes() != 0) { + String errorMessage = "expected total copies needs to be a multiple of total awareness attributes [" + + maxAwarenessAttributes() + + "]"; + return Optional.of(errorMessage); + } + return Optional.empty(); + } + +} diff --git a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java index 11cb2ca316235..600c42d036fa1 100644 --- a/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java +++ b/server/src/main/java/org/opensearch/common/settings/ClusterSettings.java @@ -32,6 +32,7 @@ package org.opensearch.common.settings; import org.apache.logging.log4j.LogManager; +import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.action.search.CreatePitController; import org.opensearch.cluster.routing.allocation.decider.NodeLoadAwareAllocationDecider; import org.opensearch.index.IndexModule; @@ -219,6 +220,7 @@ public void apply(Settings value, Settings current, Settings previous) { Arrays.asList( AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING, AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING, + AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING, BalancedShardsAllocator.INDEX_BALANCE_FACTOR_SETTING, BalancedShardsAllocator.SHARD_BALANCE_FACTOR_SETTING, BalancedShardsAllocator.SHARD_MOVE_PRIMARY_FIRST_SETTING, diff --git a/server/src/main/java/org/opensearch/node/Node.java b/server/src/main/java/org/opensearch/node/Node.java index 6cc8128cffd52..7e90d82c4b663 100644 --- a/server/src/main/java/org/opensearch/node/Node.java +++ b/server/src/main/java/org/opensearch/node/Node.java @@ -37,6 +37,7 @@ import org.apache.lucene.util.Constants; import org.apache.lucene.util.SetOnce; import org.opensearch.common.util.FeatureFlags; +import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.index.IndexingPressureService; import org.opensearch.indices.replication.SegmentReplicationSourceFactory; import org.opensearch.indices.replication.SegmentReplicationTargetService; @@ -652,6 +653,10 @@ protected Node( final AliasValidator aliasValidator = new AliasValidator(); final ShardLimitValidator shardLimitValidator = new ShardLimitValidator(settings, clusterService, systemIndices); + final AwarenessReplicaBalance awarenessReplicaBalance = new AwarenessReplicaBalance( + settings, + clusterService.getClusterSettings() + ); final MetadataCreateIndexService metadataCreateIndexService = new MetadataCreateIndexService( settings, clusterService, @@ -664,7 +669,8 @@ protected Node( threadPool, xContentRegistry, systemIndices, - forbidPrivateIndexSettings + forbidPrivateIndexSettings, + awarenessReplicaBalance ); pluginsService.filterPlugins(Plugin.class) .forEach( @@ -921,6 +927,7 @@ protected Node( b.bind(IndicesService.class).toInstance(indicesService); b.bind(AliasValidator.class).toInstance(aliasValidator); b.bind(MetadataCreateIndexService.class).toInstance(metadataCreateIndexService); + b.bind(AwarenessReplicaBalance.class).toInstance(awarenessReplicaBalance); b.bind(MetadataCreateDataStreamService.class).toInstance(metadataCreateDataStreamService); b.bind(SearchService.class).toInstance(searchService); b.bind(SearchTransportService.class).toInstance(searchTransportService); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java index afe35538adaf5..d54c17041e96f 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/MetadataRolloverServiceTests.java @@ -57,6 +57,7 @@ import org.opensearch.cluster.metadata.MetadataIndexAliasesService; import org.opensearch.cluster.metadata.Template; import org.opensearch.cluster.routing.allocation.AllocationService; +import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.CheckedFunction; import org.opensearch.common.Strings; @@ -617,7 +618,8 @@ public void testRolloverClusterState() throws Exception { testThreadPool, null, systemIndices, - false + false, + new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()) ); MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService( clusterService, @@ -754,7 +756,8 @@ public void testRolloverClusterStateForDataStream() throws Exception { testThreadPool, null, systemIndices, - false + false, + new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()) ); MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService( clusterService, @@ -931,7 +934,8 @@ public void testRolloverClusterStateForDataStreamNoTemplate() throws Exception { testThreadPool, null, new SystemIndices(emptyMap()), - false + false, + new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()) ); MetadataIndexAliasesService indexAliasesService = new MetadataIndexAliasesService( clusterService, diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java index 89550b491500d..ac03ab49bbbbd 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataCreateIndexServiceTests.java @@ -51,13 +51,16 @@ import org.opensearch.cluster.node.DiscoveryNodes; import org.opensearch.cluster.routing.RoutingTable; import org.opensearch.cluster.routing.allocation.AllocationService; +import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; import org.opensearch.cluster.routing.allocation.decider.AllocationDeciders; +import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.MaxRetryAllocationDecider; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Strings; import org.opensearch.common.collect.ImmutableOpenMap; import org.opensearch.common.compress.CompressedXContent; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Setting; import org.opensearch.common.settings.Settings; @@ -95,6 +98,7 @@ import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; @@ -108,7 +112,6 @@ import static java.util.Collections.emptyMap; import static java.util.Collections.singleton; import static java.util.Collections.singletonList; -import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; import static org.hamcrest.Matchers.endsWith; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.hasKey; @@ -117,6 +120,8 @@ import static org.hamcrest.Matchers.notNullValue; import static org.hamcrest.Matchers.nullValue; import static org.hamcrest.Matchers.startsWith; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_ROUTING_SHARDS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_NUMBER_OF_SHARDS_SETTING; import static org.opensearch.cluster.metadata.IndexMetadata.INDEX_READ_ONLY_BLOCK; @@ -130,6 +135,7 @@ import static org.opensearch.cluster.metadata.MetadataCreateIndexService.getIndexNumberOfRoutingShards; import static org.opensearch.cluster.metadata.MetadataCreateIndexService.parseV1Mappings; import static org.opensearch.cluster.metadata.MetadataCreateIndexService.resolveAndValidateAliases; +import static org.opensearch.index.IndexSettings.INDEX_SOFT_DELETES_SETTING; import static org.opensearch.indices.ShardLimitValidatorTests.createTestShardLimitService; public class MetadataCreateIndexServiceTests extends OpenSearchTestCase { @@ -592,7 +598,8 @@ public void testValidateIndexName() throws Exception { threadPool, null, new SystemIndices(Collections.emptyMap()), - false + false, + new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()) ); validateIndexName(checkerService, "index?name", "must not contain the following characters " + Strings.INVALID_FILENAME_CHARS); @@ -673,7 +680,8 @@ public void testValidateDotIndex() { threadPool, null, new SystemIndices(Collections.singletonMap("foo", systemIndexDescriptors)), - false + false, + new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()) ); // Check deprecations assertFalse(checkerService.validateDotIndex(".test2", false)); @@ -1064,6 +1072,52 @@ public void testParseMappingsWithTypelessTemplate() throws Exception { assertThat(mappings, Matchers.hasKey(MapperService.SINGLE_MAPPING_NAME)); } + public void testvalidateIndexSettings() { + ClusterService clusterService = mock(ClusterService.class); + ThreadPool threadPool = new TestThreadPool(getTestName()); + Settings settings = Settings.builder() + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone, rack") + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a, b") + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "rack.values", "c, d, e") + .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true) + .build(); + ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + when(clusterService.getSettings()).thenReturn(settings); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + MetadataCreateIndexService checkerService = new MetadataCreateIndexService( + settings, + clusterService, + null, + null, + null, + createTestShardLimitService(randomIntBetween(1, 1000), clusterService), + new Environment(Settings.builder().put("path.home", "dummy").build(), null), + IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, + threadPool, + null, + new SystemIndices(Collections.emptyMap()), + true, + new AwarenessReplicaBalance(settings, clusterService.getClusterSettings()) + ); + + List validationErrors = checkerService.getIndexSettingsValidationErrors(settings, false, Optional.empty()); + assertThat(validationErrors.size(), is(1)); + assertThat(validationErrors.get(0), is("expected total copies needs to be a multiple of total awareness attributes [3]")); + + settings = Settings.builder() + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone, rack") + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a, b") + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "rack.values", "c, d, e") + .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true) + .put(SETTING_NUMBER_OF_REPLICAS, 2) + .build(); + + validationErrors = checkerService.getIndexSettingsValidationErrors(settings, false, Optional.empty()); + assertThat(validationErrors.size(), is(0)); + + threadPool.shutdown(); + } + public void testBuildIndexMetadata() { IndexMetadata sourceIndexMetadata = IndexMetadata.builder("parent") .settings(Settings.builder().put("index.version.created", Version.CURRENT).build()) @@ -1186,10 +1240,11 @@ public void testIndexLifecycleNameSetting() { threadPool, null, new SystemIndices(Collections.emptyMap()), - true + true, + new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()) ); - final List validationErrors = checkerService.getIndexSettingsValidationErrors(ilnSetting, true); + final List validationErrors = checkerService.getIndexSettingsValidationErrors(ilnSetting, true, Optional.empty()); assertThat(validationErrors.size(), is(1)); assertThat(validationErrors.get(0), is("expected [index.lifecycle.name] to be private but it was not")); })); diff --git a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java index c3a16a1e25bc8..887d8469bd01c 100644 --- a/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java +++ b/server/src/test/java/org/opensearch/cluster/metadata/MetadataIndexTemplateServiceTests.java @@ -38,9 +38,11 @@ import org.opensearch.action.support.master.AcknowledgedResponse; import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.MetadataIndexTemplateService.PutRequest; +import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Strings; import org.opensearch.common.compress.CompressedXContent; +import org.opensearch.common.settings.ClusterSettings; import org.opensearch.common.settings.IndexScopedSettings; import org.opensearch.common.settings.Settings; import org.opensearch.common.unit.TimeValue; @@ -85,7 +87,10 @@ import static org.hamcrest.Matchers.empty; import static org.hamcrest.Matchers.is; import static org.hamcrest.Matchers.matchesRegex; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import static org.opensearch.common.settings.Settings.builder; +import static org.opensearch.env.Environment.PATH_HOME_SETTING; import static org.opensearch.index.mapper.DataStreamFieldMapper.Defaults.TIMESTAMP_FIELD; import static org.opensearch.indices.ShardLimitValidatorTests.createTestShardLimitService; @@ -2084,9 +2089,14 @@ public void testLegacyNoopUpdate() { } private static List putTemplate(NamedXContentRegistry xContentRegistry, PutRequest request) { + ClusterService clusterService = mock(ClusterService.class); + Settings settings = Settings.builder().put(PATH_HOME_SETTING.getKey(), "dummy").build(); + ClusterSettings clusterSettings = new ClusterSettings(settings, ClusterSettings.BUILT_IN_CLUSTER_SETTINGS); + when(clusterService.getSettings()).thenReturn(settings); + when(clusterService.getClusterSettings()).thenReturn(clusterSettings); MetadataCreateIndexService createIndexService = new MetadataCreateIndexService( Settings.EMPTY, - null, + clusterService, null, null, null, @@ -2096,7 +2106,8 @@ private static List putTemplate(NamedXContentRegistry xContentRegistr null, xContentRegistry, new SystemIndices(Collections.emptyMap()), - true + true, + new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()) ); MetadataIndexTemplateService service = new MetadataIndexTemplateService( null, @@ -2158,7 +2169,8 @@ private MetadataIndexTemplateService getMetadataIndexTemplateService() { null, xContentRegistry(), new SystemIndices(Collections.emptyMap()), - true + true, + new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()) ); return new MetadataIndexTemplateService( clusterService, diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessAllocationTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessAllocationTests.java index b2adcd21cd8c9..c0cec7e3201bb 100644 --- a/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessAllocationTests.java +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessAllocationTests.java @@ -64,6 +64,7 @@ import java.util.Map; import static java.util.Collections.singletonMap; +import static org.hamcrest.MatcherAssert.assertThat; import static org.opensearch.cluster.routing.ShardRoutingState.INITIALIZING; import static org.opensearch.cluster.routing.ShardRoutingState.RELOCATING; import static org.opensearch.cluster.routing.ShardRoutingState.STARTED; diff --git a/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalanceTests.java b/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalanceTests.java new file mode 100644 index 0000000000000..e2431765709e6 --- /dev/null +++ b/server/src/test/java/org/opensearch/cluster/routing/allocation/AwarenessReplicaBalanceTests.java @@ -0,0 +1,66 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +package org.opensearch.cluster.routing.allocation; + +import org.opensearch.cluster.OpenSearchAllocationTestCase; +import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; +import org.opensearch.common.settings.ClusterSettings; +import org.opensearch.common.settings.Settings; + +import java.util.Optional; + +import static org.hamcrest.Matchers.equalTo; + +public class AwarenessReplicaBalanceTests extends OpenSearchAllocationTestCase { + + private static final ClusterSettings EMPTY_CLUSTER_SETTINGS = new ClusterSettings( + Settings.EMPTY, + ClusterSettings.BUILT_IN_CLUSTER_SETTINGS + ); + + public void testNoForcedAwarenessAttribute() { + Settings settings = Settings.builder().put("cluster.routing.allocation.awareness.attributes", "rack_id").build(); + + AwarenessReplicaBalance awarenessReplicaBalance = new AwarenessReplicaBalance(settings, EMPTY_CLUSTER_SETTINGS); + assertThat(awarenessReplicaBalance.maxAwarenessAttributes(), equalTo(1)); + + assertEquals(awarenessReplicaBalance.validate(0), Optional.empty()); + assertEquals(awarenessReplicaBalance.validate(1), Optional.empty()); + } + + public void testForcedAwarenessAttribute() { + Settings settings = Settings.builder() + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone, rack") + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a, b") + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "rack.values", "c, d, e") + .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true) + .build(); + + AwarenessReplicaBalance awarenessReplicaBalance = new AwarenessReplicaBalance(settings, EMPTY_CLUSTER_SETTINGS); + assertThat(awarenessReplicaBalance.maxAwarenessAttributes(), equalTo(3)); + assertEquals(awarenessReplicaBalance.validate(2), Optional.empty()); + assertEquals( + awarenessReplicaBalance.validate(1), + Optional.of("expected total copies needs to be a multiple of total awareness attributes [3]") + ); + } + + public void testForcedAwarenessAttributeDisabled() { + Settings settings = Settings.builder() + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone, rack") + .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true) + .build(); + + AwarenessReplicaBalance awarenessReplicaBalance = new AwarenessReplicaBalance(settings, EMPTY_CLUSTER_SETTINGS); + assertThat(awarenessReplicaBalance.maxAwarenessAttributes(), equalTo(1)); + assertEquals(awarenessReplicaBalance.validate(0), Optional.empty()); + assertEquals(awarenessReplicaBalance.validate(1), Optional.empty()); + } + +} diff --git a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java index 769fdc220bec4..7fe17e570d157 100644 --- a/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java +++ b/server/src/test/java/org/opensearch/indices/cluster/ClusterStateChanges.java @@ -81,6 +81,7 @@ import org.opensearch.cluster.node.DiscoveryNode; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.allocation.AllocationService; +import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.cluster.routing.allocation.FailedShard; import org.opensearch.cluster.routing.allocation.RandomAllocationDeciderTests; import org.opensearch.cluster.routing.allocation.allocator.BalancedShardsAllocator; @@ -190,6 +191,7 @@ public ClusterStateChanges(NamedXContentRegistry xContentRegistry, ThreadPool th // mocks clusterService = mock(ClusterService.class); when(clusterService.getClusterSettings()).thenReturn(clusterSettings); + when(clusterService.getSettings()).thenReturn(SETTINGS); IndicesService indicesService = mock(IndicesService.class); // MetadataCreateIndexService uses withTempIndexService to check mappings -> fake it here try { @@ -272,13 +274,17 @@ public IndexMetadata upgradeIndexMetadata(IndexMetadata indexMetadata, Version m transportVerifyShardIndexBlockAction ); MetadataDeleteIndexService deleteIndexService = new MetadataDeleteIndexService(SETTINGS, clusterService, allocationService); + + final AwarenessReplicaBalance awarenessReplicaBalance = new AwarenessReplicaBalance(SETTINGS, clusterService.getClusterSettings()); + MetadataUpdateSettingsService metadataUpdateSettingsService = new MetadataUpdateSettingsService( clusterService, allocationService, IndexScopedSettings.DEFAULT_SCOPED_SETTINGS, indicesService, shardLimitValidator, - threadPool + threadPool, + awarenessReplicaBalance ); MetadataCreateIndexService createIndexService = new MetadataCreateIndexService( SETTINGS, @@ -292,7 +298,8 @@ public IndexMetadata upgradeIndexMetadata(IndexMetadata indexMetadata, Version m threadPool, xContentRegistry, systemIndices, - true + true, + awarenessReplicaBalance ); transportCloseIndexAction = new TransportCloseIndexAction( diff --git a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java index cca27366f7df0..9a7f0e2005555 100644 --- a/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java +++ b/server/src/test/java/org/opensearch/snapshots/SnapshotResiliencyTests.java @@ -141,6 +141,7 @@ import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.UnassignedInfo; import org.opensearch.cluster.routing.allocation.AllocationService; +import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.cluster.routing.allocation.command.AllocateEmptyPrimaryAllocationCommand; import org.opensearch.cluster.service.ClusterApplierService; import org.opensearch.cluster.service.ClusterService; @@ -1895,7 +1896,8 @@ public void onFailure(final Exception e) { threadPool, namedXContentRegistry, systemIndices, - false + false, + new AwarenessReplicaBalance(Settings.EMPTY, clusterService.getClusterSettings()) ); actions.put( CreateIndexAction.INSTANCE, diff --git a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java index 0f868efdd5c45..8d8df2fec39f9 100644 --- a/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java +++ b/test/framework/src/main/java/org/opensearch/test/OpenSearchIntegTestCase.java @@ -49,6 +49,7 @@ import org.opensearch.action.admin.cluster.node.hotthreads.NodeHotThreads; import org.opensearch.action.admin.cluster.node.info.NodeInfo; import org.opensearch.action.admin.cluster.node.info.NodesInfoResponse; +import org.opensearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest; import org.opensearch.action.admin.cluster.state.ClusterStateResponse; import org.opensearch.action.admin.cluster.tasks.PendingClusterTasksResponse; import org.opensearch.action.admin.indices.create.CreateIndexRequestBuilder; @@ -91,7 +92,9 @@ import org.opensearch.cluster.routing.IndexShardRoutingTable; import org.opensearch.cluster.routing.ShardRouting; import org.opensearch.cluster.routing.UnassignedInfo; +import org.opensearch.cluster.routing.allocation.AwarenessReplicaBalance; import org.opensearch.cluster.routing.allocation.DiskThresholdSettings; +import org.opensearch.cluster.routing.allocation.decider.AwarenessAllocationDecider; import org.opensearch.cluster.routing.allocation.decider.EnableAllocationDecider; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.Nullable; @@ -2408,4 +2411,24 @@ protected boolean willSufferDebian8MemoryProblem() { boolean java15Plus = Runtime.version().compareTo(Version.parse("15")) >= 0; return anyDebian8Nodes && java15Plus == false; } + + public void manageReplicaBalanceSetting(boolean apply) { + Settings settings; + if (apply) { + settings = Settings.builder() + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey(), "zone") + .put(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values", "a, b") + .put(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey(), true) + .build(); + } else { + settings = Settings.builder() + .putNull(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_ATTRIBUTE_SETTING.getKey()) + .putNull(AwarenessAllocationDecider.CLUSTER_ROUTING_ALLOCATION_AWARENESS_FORCE_GROUP_SETTING.getKey() + "zone.values") + .putNull(AwarenessReplicaBalance.CLUSTER_ROUTING_ALLOCATION_AWARENESS_BALANCE_SETTING.getKey()) + .build(); + } + ClusterUpdateSettingsRequest updateSettingsRequest = new ClusterUpdateSettingsRequest(); + updateSettingsRequest.persistentSettings(settings); + assertAcked(client().admin().cluster().updateSettings(updateSettingsRequest).actionGet()); + } } From 2a1b239c2af8d7f333a404c66467a50dc0a52c80 Mon Sep 17 00:00:00 2001 From: Rishikesh Pasham <62345295+Rishikesh1159@users.noreply.github.com> Date: Sat, 30 Jul 2022 00:23:11 +0000 Subject: [PATCH 6/6] Deprecate and Rename abstract methods from 'Master' terminology to 'ClusterManager'. (#4032) This commit deprecates and renames the following abstract methods: ``` In class TransportMasterNodeAction - abstract void masterOperation(Request request, ClusterState state, ActionListener listener) In class TransportClusterInfoAction - abstract void doMasterOperation(Request request, String[] concreteIndices, ClusterState state, ActionListener listener) In class TestCluster: public abstract int numDataAndMasterNodes(); ``` and the following concrete methods: ``` In class TransportMasterNodeAction - void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) In class TransportClusterInfoAction - void masterOperation(final Request request, final ClusterState state, final ActionListener listener) ``` Signed-off-by: Rishikesh1159 --- .../repositories/RepositoryBlocksIT.java | 2 +- .../cluster/snapshots/SnapshotBlocksIT.java | 2 +- .../BlobStoreRepositoryCleanupIT.java | 2 +- .../opensearch/snapshots/RepositoriesIT.java | 2 +- ...ansportClusterAllocationExplainAction.java | 2 +- ...nsportAddVotingConfigExclusionsAction.java | 2 +- ...portClearVotingConfigExclusionsAction.java | 2 +- .../health/TransportClusterHealthAction.java | 9 ++++--- .../TransportCleanupRepositoryAction.java | 2 +- .../TransportDeleteRepositoryAction.java | 2 +- .../get/TransportGetRepositoriesAction.java | 2 +- .../put/TransportPutRepositoryAction.java | 2 +- .../TransportVerifyRepositoryAction.java | 2 +- .../TransportClusterRerouteAction.java | 2 +- .../TransportClusterUpdateSettingsAction.java | 2 +- .../TransportClusterSearchShardsAction.java | 2 +- .../clone/TransportCloneSnapshotAction.java | 6 ++++- .../create/TransportCreateSnapshotAction.java | 2 +- .../delete/TransportDeleteSnapshotAction.java | 2 +- .../get/TransportGetSnapshotsAction.java | 2 +- .../TransportRestoreSnapshotAction.java | 2 +- .../TransportSnapshotsStatusAction.java | 2 +- .../state/TransportClusterStateAction.java | 2 +- .../TransportDeleteStoredScriptAction.java | 7 +++-- .../TransportGetStoredScriptAction.java | 7 +++-- .../TransportPutStoredScriptAction.java | 7 +++-- .../TransportPendingClusterTasksAction.java | 2 +- .../alias/TransportIndicesAliasesAction.java | 2 +- .../alias/get/TransportGetAliasesAction.java | 2 +- .../close/TransportCloseIndexAction.java | 4 +-- .../indices/create/AutoCreateAction.java | 6 ++++- .../create/TransportCreateIndexAction.java | 2 +- .../TransportDeleteDanglingIndexAction.java | 2 +- .../datastream/CreateDataStreamAction.java | 2 +- .../datastream/DeleteDataStreamAction.java | 2 +- .../datastream/GetDataStreamAction.java | 2 +- .../delete/TransportDeleteIndexAction.java | 2 +- .../indices/TransportIndicesExistsAction.java | 2 +- .../indices/get/TransportGetIndexAction.java | 2 +- .../get/TransportGetMappingsAction.java | 2 +- .../put/TransportAutoPutMappingAction.java | 2 +- .../put/TransportPutMappingAction.java | 2 +- .../open/TransportOpenIndexAction.java | 2 +- .../TransportAddIndexBlockAction.java | 4 +-- .../rollover/TransportRolloverAction.java | 4 +-- .../get/TransportGetSettingsAction.java | 2 +- .../put/TransportUpdateSettingsAction.java | 2 +- .../TransportIndicesShardStoresAction.java | 2 +- .../indices/shrink/TransportResizeAction.java | 2 +- ...ransportDeleteComponentTemplateAction.java | 2 +- ...rtDeleteComposableIndexTemplateAction.java | 2 +- .../TransportDeleteIndexTemplateAction.java | 2 +- .../TransportGetComponentTemplateAction.java | 2 +- ...sportGetComposableIndexTemplateAction.java | 2 +- .../get/TransportGetIndexTemplatesAction.java | 2 +- .../TransportSimulateIndexTemplateAction.java | 2 +- .../post/TransportSimulateTemplateAction.java | 2 +- .../TransportPutComponentTemplateAction.java | 2 +- ...sportPutComposableIndexTemplateAction.java | 2 +- .../put/TransportPutIndexTemplateAction.java | 2 +- .../post/TransportUpgradeSettingsAction.java | 2 +- .../ingest/DeletePipelineTransportAction.java | 2 +- .../ingest/GetPipelineTransportAction.java | 2 +- .../ingest/PutPipelineTransportAction.java | 2 +- .../TransportClusterManagerNodeAction.java | 26 ++++++++++++++++--- .../info/TransportClusterInfoAction.java | 25 +++++++++++++++--- .../master/TransportMasterNodeAction.java | 5 ---- .../info/TransportClusterInfoAction.java | 9 ------- .../CompletionPersistentTaskAction.java | 2 +- .../RemovePersistentTaskAction.java | 2 +- .../persistent/StartPersistentTaskAction.java | 2 +- .../UpdatePersistentTaskStatusAction.java | 2 +- .../snapshots/SnapshotsService.java | 2 +- .../indices/get/GetIndexActionTests.java | 4 +-- .../TransportRolloverActionTests.java | 4 +-- .../settings/get/GetSettingsActionTests.java | 8 ++++-- ...ransportClusterManagerNodeActionTests.java | 7 ++--- .../TransportMasterNodeActionUtils.java | 4 +-- .../InternalOrPrivateSettingsPlugin.java | 2 +- .../opensearch/test/ExternalTestCluster.java | 2 +- .../opensearch/test/InternalTestCluster.java | 2 +- .../java/org/opensearch/test/TestCluster.java | 13 +++++++++- 82 files changed, 171 insertions(+), 114 deletions(-) diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/repositories/RepositoryBlocksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/repositories/RepositoryBlocksIT.java index 4d3b60da0987f..aff7c5d9876ac 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/repositories/RepositoryBlocksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/repositories/RepositoryBlocksIT.java @@ -97,7 +97,7 @@ public void testVerifyRepositoryWithBlocks() { .prepareVerifyRepository("test-repo-blocks") .execute() .actionGet(); - assertThat(response.getNodes().size(), equalTo(cluster().numDataAndMasterNodes())); + assertThat(response.getNodes().size(), equalTo(cluster().numDataAndClusterManagerNodes())); } finally { setClusterReadOnly(false); } diff --git a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java index 1731c607a066d..474a5069ad091 100644 --- a/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/action/admin/cluster/snapshots/SnapshotBlocksIT.java @@ -92,7 +92,7 @@ protected void setUpRepository() throws Exception { logger.info("--> verify the repository"); VerifyRepositoryResponse verifyResponse = client().admin().cluster().prepareVerifyRepository(REPOSITORY_NAME).get(); - assertThat(verifyResponse.getNodes().size(), equalTo(cluster().numDataAndMasterNodes())); + assertThat(verifyResponse.getNodes().size(), equalTo(cluster().numDataAndClusterManagerNodes())); logger.info("--> create a snapshot"); CreateSnapshotResponse snapshotResponse = client().admin() diff --git a/server/src/internalClusterTest/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java b/server/src/internalClusterTest/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java index f3954578bffc8..29aab140231c7 100644 --- a/server/src/internalClusterTest/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/repositories/blobstore/BlobStoreRepositoryCleanupIT.java @@ -54,7 +54,7 @@ public class BlobStoreRepositoryCleanupIT extends AbstractSnapshotIntegTestCase public void testClusterManagerFailoverDuringCleanup() throws Exception { startBlockedCleanup("test-repo"); - final int nodeCount = internalCluster().numDataAndMasterNodes(); + final int nodeCount = internalCluster().numDataAndClusterManagerNodes(); logger.info("--> stopping cluster-manager node"); internalCluster().stopCurrentClusterManagerNode(); diff --git a/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java b/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java index 9aaafa64ce60c..47fd106ff5a8f 100644 --- a/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java +++ b/server/src/internalClusterTest/java/org/opensearch/snapshots/RepositoriesIT.java @@ -74,7 +74,7 @@ public void testRepositoryCreation() throws Exception { logger.info("--> verify the repository"); int numberOfFiles = FileSystemUtils.files(location).length; VerifyRepositoryResponse verifyRepositoryResponse = client.admin().cluster().prepareVerifyRepository("test-repo-1").get(); - assertThat(verifyRepositoryResponse.getNodes().size(), equalTo(cluster().numDataAndMasterNodes())); + assertThat(verifyRepositoryResponse.getNodes().size(), equalTo(cluster().numDataAndClusterManagerNodes())); logger.info("--> verify that we didn't leave any files as a result of verification"); assertThat(FileSystemUtils.files(location).length, equalTo(numberOfFiles)); diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java index 16721d5a9ec07..7060ac43af7f9 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/allocation/TransportClusterAllocationExplainAction.java @@ -125,7 +125,7 @@ protected ClusterBlockException checkBlock(ClusterAllocationExplainRequest reque } @Override - protected void masterOperation( + protected void clusterManagerOperation( final ClusterAllocationExplainRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java index ab72ce964668f..d0f5e8f198809 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportAddVotingConfigExclusionsAction.java @@ -126,7 +126,7 @@ protected AddVotingConfigExclusionsResponse read(StreamInput in) throws IOExcept } @Override - protected void masterOperation( + protected void clusterManagerOperation( AddVotingConfigExclusionsRequest request, ClusterState state, ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java index 3a9da6cebef53..1fc02db4309b1 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/configuration/TransportClearVotingConfigExclusionsAction.java @@ -101,7 +101,7 @@ protected ClearVotingConfigExclusionsResponse read(StreamInput in) throws IOExce } @Override - protected void masterOperation( + protected void clusterManagerOperation( ClearVotingConfigExclusionsRequest request, ClusterState initialState, ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthAction.java index a15fc15ef6a2b..38198a1b74fb3 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/health/TransportClusterHealthAction.java @@ -118,14 +118,17 @@ protected ClusterBlockException checkBlock(ClusterHealthRequest request, Cluster } @Override - protected final void masterOperation(ClusterHealthRequest request, ClusterState state, ActionListener listener) - throws Exception { + protected final void clusterManagerOperation( + ClusterHealthRequest request, + ClusterState state, + ActionListener listener + ) throws Exception { logger.warn("attempt to execute a cluster health operation without a task"); throw new UnsupportedOperationException("task parameter is required for this operation"); } @Override - protected void masterOperation( + protected void clusterManagerOperation( final Task task, final ClusterHealthRequest request, final ClusterState unusedState, diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java index f3c5d51e1907a..a3804db687a2d 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/cleanup/TransportCleanupRepositoryAction.java @@ -174,7 +174,7 @@ protected CleanupRepositoryResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation( + protected void clusterManagerOperation( CleanupRepositoryRequest request, ClusterState state, ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java index 4e2072d6a8fa3..08e3bc6df0d83 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/delete/TransportDeleteRepositoryAction.java @@ -95,7 +95,7 @@ protected ClusterBlockException checkBlock(DeleteRepositoryRequest request, Clus } @Override - protected void masterOperation( + protected void clusterManagerOperation( final DeleteRepositoryRequest request, ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java index 6e61752c78656..de942ef284f3b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/get/TransportGetRepositoriesAction.java @@ -99,7 +99,7 @@ protected ClusterBlockException checkBlock(GetRepositoriesRequest request, Clust } @Override - protected void masterOperation( + protected void clusterManagerOperation( final GetRepositoriesRequest request, ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java index 58ccc4e7ff818..6a5be14df93fd 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/put/TransportPutRepositoryAction.java @@ -95,7 +95,7 @@ protected ClusterBlockException checkBlock(PutRepositoryRequest request, Cluster } @Override - protected void masterOperation( + protected void clusterManagerOperation( final PutRepositoryRequest request, ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java index a673f34058a83..5215078f52d3b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/repositories/verify/TransportVerifyRepositoryAction.java @@ -95,7 +95,7 @@ protected ClusterBlockException checkBlock(VerifyRepositoryRequest request, Clus } @Override - protected void masterOperation( + protected void clusterManagerOperation( final VerifyRepositoryRequest request, ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java index 5080ce2c0fd67..3e5ebdd6a17d3 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/reroute/TransportClusterRerouteAction.java @@ -119,7 +119,7 @@ protected ClusterRerouteResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation( + protected void clusterManagerOperation( final ClusterRerouteRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java index 47024e98f82b1..ef404375485a2 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/settings/TransportClusterUpdateSettingsAction.java @@ -124,7 +124,7 @@ protected ClusterUpdateSettingsResponse read(StreamInput in) throws IOException } @Override - protected void masterOperation( + protected void clusterManagerOperation( final ClusterUpdateSettingsRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java index ae2d2aeb827ba..2f7c194e0acd7 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/shards/TransportClusterSearchShardsAction.java @@ -108,7 +108,7 @@ protected ClusterSearchShardsResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation( + protected void clusterManagerOperation( final ClusterSearchShardsRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java index f4e90bbbfb961..e9f5153f78700 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/clone/TransportCloneSnapshotAction.java @@ -90,7 +90,11 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation(CloneSnapshotRequest request, ClusterState state, ActionListener listener) { + protected void clusterManagerOperation( + CloneSnapshotRequest request, + ClusterState state, + ActionListener listener + ) { snapshotsService.cloneSnapshot(request, ActionListener.map(listener, v -> new AcknowledgedResponse(true))); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java index 4b28bafc258cf..ed4af6d915792 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/create/TransportCreateSnapshotAction.java @@ -98,7 +98,7 @@ protected ClusterBlockException checkBlock(CreateSnapshotRequest request, Cluste } @Override - protected void masterOperation( + protected void clusterManagerOperation( final CreateSnapshotRequest request, ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java index d5e18dcd31fc9..c78968c2a0848 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/delete/TransportDeleteSnapshotAction.java @@ -95,7 +95,7 @@ protected ClusterBlockException checkBlock(DeleteSnapshotRequest request, Cluste } @Override - protected void masterOperation( + protected void clusterManagerOperation( final DeleteSnapshotRequest request, ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java index 0be3f8be0bc80..d05e62045a1a2 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/get/TransportGetSnapshotsAction.java @@ -122,7 +122,7 @@ protected ClusterBlockException checkBlock(GetSnapshotsRequest request, ClusterS } @Override - protected void masterOperation( + protected void clusterManagerOperation( final GetSnapshotsRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java index fa7c0c6efa469..e7d95b9e40880 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/restore/TransportRestoreSnapshotAction.java @@ -102,7 +102,7 @@ protected ClusterBlockException checkBlock(RestoreSnapshotRequest request, Clust } @Override - protected void masterOperation( + protected void clusterManagerOperation( final RestoreSnapshotRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java index e73a68e335316..bd7391a7939a1 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/snapshots/status/TransportSnapshotsStatusAction.java @@ -136,7 +136,7 @@ protected SnapshotsStatusResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation( + protected void clusterManagerOperation( final SnapshotsStatusRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java index a30b6fd580f1a..9d65146716496 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/state/TransportClusterStateAction.java @@ -115,7 +115,7 @@ protected ClusterBlockException checkBlock(ClusterStateRequest request, ClusterS } @Override - protected void masterOperation( + protected void clusterManagerOperation( final ClusterStateRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java index 365d7df0f650d..4bc8d836a8200 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportDeleteStoredScriptAction.java @@ -90,8 +90,11 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation(DeleteStoredScriptRequest request, ClusterState state, ActionListener listener) - throws Exception { + protected void clusterManagerOperation( + DeleteStoredScriptRequest request, + ClusterState state, + ActionListener listener + ) throws Exception { scriptService.deleteStoredScript(clusterService, request, listener); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java index d2d5a49fcde23..8dbadf34ab06b 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportGetStoredScriptAction.java @@ -89,8 +89,11 @@ protected GetStoredScriptResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation(GetStoredScriptRequest request, ClusterState state, ActionListener listener) - throws Exception { + protected void clusterManagerOperation( + GetStoredScriptRequest request, + ClusterState state, + ActionListener listener + ) throws Exception { listener.onResponse(new GetStoredScriptResponse(request.id(), scriptService.getStoredScript(state, request))); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java index db76394c45cd5..bb259f173d470 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/storedscripts/TransportPutStoredScriptAction.java @@ -90,8 +90,11 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation(PutStoredScriptRequest request, ClusterState state, ActionListener listener) - throws Exception { + protected void clusterManagerOperation( + PutStoredScriptRequest request, + ClusterState state, + ActionListener listener + ) throws Exception { scriptService.putStoredScript(clusterService, request, listener); } diff --git a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java index 8962f0395cc6f..e08a76c5cfc2a 100644 --- a/server/src/main/java/org/opensearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java +++ b/server/src/main/java/org/opensearch/action/admin/cluster/tasks/TransportPendingClusterTasksAction.java @@ -100,7 +100,7 @@ protected ClusterBlockException checkBlock(PendingClusterTasksRequest request, C } @Override - protected void masterOperation( + protected void clusterManagerOperation( PendingClusterTasksRequest request, ClusterState state, ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java index aacd8e25ff827..3e453b42c3d7c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/TransportIndicesAliasesAction.java @@ -126,7 +126,7 @@ protected ClusterBlockException checkBlock(IndicesAliasesRequest request, Cluste } @Override - protected void masterOperation( + protected void clusterManagerOperation( final IndicesAliasesRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java index a2f975ff9cbbc..fe9c2dbccdf7b 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/alias/get/TransportGetAliasesAction.java @@ -112,7 +112,7 @@ protected GetAliasesResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation(GetAliasesRequest request, ClusterState state, ActionListener listener) { + protected void clusterManagerOperation(GetAliasesRequest request, ClusterState state, ActionListener listener) { String[] concreteIndices; // Switch to a context which will drop any deprecation warnings, because there may be indices resolved here which are not // returned in the final response. We'll add warnings back later if necessary in checkSystemIndexAccess. diff --git a/server/src/main/java/org/opensearch/action/admin/indices/close/TransportCloseIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/close/TransportCloseIndexAction.java index dc6c94b73065d..c4c789a8de90e 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/close/TransportCloseIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/close/TransportCloseIndexAction.java @@ -140,7 +140,7 @@ protected ClusterBlockException checkBlock(CloseIndexRequest request, ClusterSta } @Override - protected void masterOperation( + protected void clusterManagerOperation( final CloseIndexRequest request, final ClusterState state, final ActionListener listener @@ -149,7 +149,7 @@ protected void masterOperation( } @Override - protected void masterOperation( + protected void clusterManagerOperation( final Task task, final CloseIndexRequest request, final ClusterState state, diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/AutoCreateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/create/AutoCreateAction.java index fdfeb37ac64a7..73a2996945aff 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/AutoCreateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/AutoCreateAction.java @@ -112,7 +112,11 @@ protected CreateIndexResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation(CreateIndexRequest request, ClusterState state, ActionListener finalListener) { + protected void clusterManagerOperation( + CreateIndexRequest request, + ClusterState state, + ActionListener finalListener + ) { AtomicReference indexNameRef = new AtomicReference<>(); ActionListener listener = ActionListener.wrap(response -> { String indexName = indexNameRef.get(); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java index 4e145b9905c95..daba4d0f167f8 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/create/TransportCreateIndexAction.java @@ -95,7 +95,7 @@ protected ClusterBlockException checkBlock(CreateIndexRequest request, ClusterSt } @Override - protected void masterOperation( + protected void clusterManagerOperation( final CreateIndexRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java index 9ad2058647592..015a0f6727ab7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/dangling/delete/TransportDeleteDanglingIndexAction.java @@ -115,7 +115,7 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation( + protected void clusterManagerOperation( DeleteDanglingIndexRequest deleteRequest, ClusterState state, ActionListener deleteListener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java index 763216912dc05..ddc93dd1fcf6c 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/CreateDataStreamAction.java @@ -162,7 +162,7 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation(Request request, ClusterState state, ActionListener listener) + protected void clusterManagerOperation(Request request, ClusterState state, ActionListener listener) throws Exception { CreateDataStreamClusterStateUpdateRequest updateRequest = new CreateDataStreamClusterStateUpdateRequest( request.name, diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java index b9b855d364726..74b0a84782283 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/DeleteDataStreamAction.java @@ -192,7 +192,7 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation(Request request, ClusterState state, ActionListener listener) + protected void clusterManagerOperation(Request request, ClusterState state, ActionListener listener) throws Exception { clusterService.submitStateUpdateTask( "remove-data-stream [" + Strings.arrayToCommaDelimitedString(request.names) + "]", diff --git a/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java b/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java index 6140d10bd293c..61fad265e16e6 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/datastream/GetDataStreamAction.java @@ -313,7 +313,7 @@ protected Response read(StreamInput in) throws IOException { } @Override - protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + protected void clusterManagerOperation(Request request, ClusterState state, ActionListener listener) throws Exception { List dataStreams = getDataStreams(state, indexNameExpressionResolver, request); List dataStreamInfos = new ArrayList<>(dataStreams.size()); for (DataStream dataStream : dataStreams) { diff --git a/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java index fc79a8be39527..a3d14846338e7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/delete/TransportDeleteIndexAction.java @@ -115,7 +115,7 @@ protected ClusterBlockException checkBlock(DeleteIndexRequest request, ClusterSt } @Override - protected void masterOperation( + protected void clusterManagerOperation( final DeleteIndexRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java index a7f73a203f4c5..f5f7e0e9ea7b7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/exists/indices/TransportIndicesExistsAction.java @@ -103,7 +103,7 @@ protected ClusterBlockException checkBlock(IndicesExistsRequest request, Cluster } @Override - protected void masterOperation( + protected void clusterManagerOperation( final IndicesExistsRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java index 0142e70d18221..de272bab332a7 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/get/TransportGetIndexAction.java @@ -98,7 +98,7 @@ protected GetIndexResponse read(StreamInput in) throws IOException { } @Override - protected void doMasterOperation( + protected void doClusterManagerOperation( final GetIndexRequest request, String[] concreteIndices, final ClusterState state, diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java index 71438ad300e0c..e724320728b66 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/get/TransportGetMappingsAction.java @@ -88,7 +88,7 @@ protected GetMappingsResponse read(StreamInput in) throws IOException { } @Override - protected void doMasterOperation( + protected void doClusterManagerOperation( final GetMappingsRequest request, String[] concreteIndices, final ClusterState state, diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java index e9e8770478853..c4dad614c53dd 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportAutoPutMappingAction.java @@ -107,7 +107,7 @@ protected ClusterBlockException checkBlock(PutMappingRequest request, ClusterSta } @Override - protected void masterOperation( + protected void clusterManagerOperation( final PutMappingRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java index a6a038fe0708b..de546f428bafa 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/mapping/put/TransportPutMappingAction.java @@ -119,7 +119,7 @@ protected ClusterBlockException checkBlock(PutMappingRequest request, ClusterSta } @Override - protected void masterOperation( + protected void clusterManagerOperation( final PutMappingRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java b/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java index 98816f8677e49..2ccb5f6d22886 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/open/TransportOpenIndexAction.java @@ -114,7 +114,7 @@ protected ClusterBlockException checkBlock(OpenIndexRequest request, ClusterStat } @Override - protected void masterOperation( + protected void clusterManagerOperation( final OpenIndexRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java b/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java index e7fde50a0f46c..560d2e6389c63 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/readonly/TransportAddIndexBlockAction.java @@ -123,13 +123,13 @@ protected ClusterBlockException checkBlock(AddIndexBlockRequest request, Cluster } @Override - protected void masterOperation(AddIndexBlockRequest request, ClusterState state, ActionListener listener) + protected void clusterManagerOperation(AddIndexBlockRequest request, ClusterState state, ActionListener listener) throws Exception { throw new UnsupportedOperationException("The task parameter is required"); } @Override - protected void masterOperation( + protected void clusterManagerOperation( final Task task, final AddIndexBlockRequest request, final ClusterState state, diff --git a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java index 29bd2e0ab5a1e..4e5e7ec9184fe 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/rollover/TransportRolloverAction.java @@ -130,13 +130,13 @@ protected ClusterBlockException checkBlock(RolloverRequest request, ClusterState } @Override - protected void masterOperation(RolloverRequest request, ClusterState state, ActionListener listener) + protected void clusterManagerOperation(RolloverRequest request, ClusterState state, ActionListener listener) throws Exception { throw new UnsupportedOperationException("The task parameter is required"); } @Override - protected void masterOperation( + protected void clusterManagerOperation( Task task, final RolloverRequest rolloverRequest, final ClusterState state, diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/TransportGetSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/TransportGetSettingsAction.java index 000d6d70d7af7..cfa75167afa09 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/get/TransportGetSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/get/TransportGetSettingsAction.java @@ -110,7 +110,7 @@ private static boolean isFilteredRequest(GetSettingsRequest request) { } @Override - protected void masterOperation(GetSettingsRequest request, ClusterState state, ActionListener listener) { + protected void clusterManagerOperation(GetSettingsRequest request, ClusterState state, ActionListener listener) { Index[] concreteIndices = indexNameExpressionResolver.concreteIndices(state, request); ImmutableOpenMap.Builder indexToSettingsBuilder = ImmutableOpenMap.builder(); ImmutableOpenMap.Builder indexToDefaultSettingsBuilder = ImmutableOpenMap.builder(); diff --git a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java index ad893c69a9847..a959fb043e4c6 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/settings/put/TransportUpdateSettingsAction.java @@ -116,7 +116,7 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation( + protected void clusterManagerOperation( final UpdateSettingsRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java index b2f2c9e5d03a3..cfe682e47f688 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shards/TransportIndicesShardStoresAction.java @@ -121,7 +121,7 @@ protected IndicesShardStoresResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation( + protected void clusterManagerOperation( IndicesShardStoresRequest request, ClusterState state, ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java index 88af27fa21ea3..ba079aeb03921 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/shrink/TransportResizeAction.java @@ -127,7 +127,7 @@ protected ClusterBlockException checkBlock(ResizeRequest request, ClusterState s } @Override - protected void masterOperation( + protected void clusterManagerOperation( final ResizeRequest resizeRequest, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java index 841810ce9e664..75cc8ffe05f73 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComponentTemplateAction.java @@ -102,7 +102,7 @@ protected ClusterBlockException checkBlock(DeleteComponentTemplateAction.Request } @Override - protected void masterOperation( + protected void clusterManagerOperation( final DeleteComponentTemplateAction.Request request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java index 0e2043848aca3..52464dbd90e3f 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteComposableIndexTemplateAction.java @@ -102,7 +102,7 @@ protected ClusterBlockException checkBlock(DeleteComposableIndexTemplateAction.R } @Override - protected void masterOperation( + protected void clusterManagerOperation( final DeleteComposableIndexTemplateAction.Request request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java index 80f008072b024..3d3c2bd0613f2 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/delete/TransportDeleteIndexTemplateAction.java @@ -102,7 +102,7 @@ protected ClusterBlockException checkBlock(DeleteIndexTemplateRequest request, C } @Override - protected void masterOperation( + protected void clusterManagerOperation( final DeleteIndexTemplateRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java index c6016ad78a681..36e3a1d0e6264 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComponentTemplateAction.java @@ -96,7 +96,7 @@ protected ClusterBlockException checkBlock(GetComponentTemplateAction.Request re } @Override - protected void masterOperation( + protected void clusterManagerOperation( GetComponentTemplateAction.Request request, ClusterState state, ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java index 405dc7afc769f..327a40be64a2a 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetComposableIndexTemplateAction.java @@ -96,7 +96,7 @@ protected ClusterBlockException checkBlock(GetComposableIndexTemplateAction.Requ } @Override - protected void masterOperation( + protected void clusterManagerOperation( GetComposableIndexTemplateAction.Request request, ClusterState state, ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java index 44969022c9e06..d74ff9e309842 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/get/TransportGetIndexTemplatesAction.java @@ -96,7 +96,7 @@ protected ClusterBlockException checkBlock(GetIndexTemplatesRequest request, Clu } @Override - protected void masterOperation( + protected void clusterManagerOperation( GetIndexTemplatesRequest request, ClusterState state, ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java index 51a634e876886..ee2a049d5f4a5 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateIndexTemplateAction.java @@ -127,7 +127,7 @@ protected SimulateIndexTemplateResponse read(StreamInput in) throws IOException } @Override - protected void masterOperation( + protected void clusterManagerOperation( SimulateIndexTemplateRequest request, ClusterState state, ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java index 5b7395b3bc3a1..71ded5687ac72 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/post/TransportSimulateTemplateAction.java @@ -113,7 +113,7 @@ protected SimulateIndexTemplateResponse read(StreamInput in) throws IOException } @Override - protected void masterOperation( + protected void clusterManagerOperation( SimulateTemplateAction.Request request, ClusterState state, ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java index 30f196142515b..925913c4e8d3e 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComponentTemplateAction.java @@ -106,7 +106,7 @@ protected ClusterBlockException checkBlock(PutComponentTemplateAction.Request re } @Override - protected void masterOperation( + protected void clusterManagerOperation( final PutComponentTemplateAction.Request request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java index d8bda8839aa42..20ba5376f1add 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutComposableIndexTemplateAction.java @@ -99,7 +99,7 @@ protected ClusterBlockException checkBlock(PutComposableIndexTemplateAction.Requ } @Override - protected void masterOperation( + protected void clusterManagerOperation( final PutComposableIndexTemplateAction.Request request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java index 8e8bcbab6f2b7..c93d83cd5e70d 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java @@ -106,7 +106,7 @@ protected ClusterBlockException checkBlock(PutIndexTemplateRequest request, Clus } @Override - protected void masterOperation( + protected void clusterManagerOperation( final PutIndexTemplateRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java index f27edbb8a4b62..df0d5cf57e7de 100644 --- a/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java +++ b/server/src/main/java/org/opensearch/action/admin/indices/upgrade/post/TransportUpgradeSettingsAction.java @@ -102,7 +102,7 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation( + protected void clusterManagerOperation( final UpgradeSettingsRequest request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineTransportAction.java b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineTransportAction.java index 590badeb54756..9085b2347765c 100644 --- a/server/src/main/java/org/opensearch/action/ingest/DeletePipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/ingest/DeletePipelineTransportAction.java @@ -88,7 +88,7 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation(DeletePipelineRequest request, ClusterState state, ActionListener listener) + protected void clusterManagerOperation(DeletePipelineRequest request, ClusterState state, ActionListener listener) throws Exception { ingestService.delete(request, listener); } diff --git a/server/src/main/java/org/opensearch/action/ingest/GetPipelineTransportAction.java b/server/src/main/java/org/opensearch/action/ingest/GetPipelineTransportAction.java index 3a5493bfa4b36..5a59c8255361e 100644 --- a/server/src/main/java/org/opensearch/action/ingest/GetPipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/ingest/GetPipelineTransportAction.java @@ -85,7 +85,7 @@ protected GetPipelineResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation(GetPipelineRequest request, ClusterState state, ActionListener listener) + protected void clusterManagerOperation(GetPipelineRequest request, ClusterState state, ActionListener listener) throws Exception { listener.onResponse(new GetPipelineResponse(IngestService.getPipelines(state, request.getIds()))); } diff --git a/server/src/main/java/org/opensearch/action/ingest/PutPipelineTransportAction.java b/server/src/main/java/org/opensearch/action/ingest/PutPipelineTransportAction.java index 448aa2855c3aa..61a2deedfd511 100644 --- a/server/src/main/java/org/opensearch/action/ingest/PutPipelineTransportAction.java +++ b/server/src/main/java/org/opensearch/action/ingest/PutPipelineTransportAction.java @@ -103,7 +103,7 @@ protected AcknowledgedResponse read(StreamInput in) throws IOException { } @Override - protected void masterOperation(PutPipelineRequest request, ClusterState state, ActionListener listener) + protected void clusterManagerOperation(PutPipelineRequest request, ClusterState state, ActionListener listener) throws Exception { NodesInfoRequest nodesInfoRequest = new NodesInfoRequest(); nodesInfoRequest.clear().addMetric(NodesInfoRequest.Metric.INGEST.metricName()); diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java b/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java index a30189a0899a4..d65ba3eddf776 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeAction.java @@ -117,13 +117,30 @@ protected TransportClusterManagerNodeAction( protected abstract Response read(StreamInput in) throws IOException; - protected abstract void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception; + /** + * @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #clusterManagerOperation(ClusterManagerNodeRequest, ClusterState, ActionListener)} + */ + @Deprecated + protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + throw new UnsupportedOperationException("Must be overridden"); + } + + protected void clusterManagerOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + masterOperation(request, state, listener); + } + + /** @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #clusterManagerOperation(Task, ClusterManagerNodeRequest, ClusterState, ActionListener)} */ + @Deprecated + protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) throws Exception { + clusterManagerOperation(task, request, state, listener); + } /** * Override this operation if access to the task parameter is needed */ - protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) throws Exception { - masterOperation(request, state, listener); + protected void clusterManagerOperation(Task task, Request request, ClusterState state, ActionListener listener) + throws Exception { + clusterManagerOperation(request, state, listener); } protected boolean localExecute(Request request) { @@ -201,7 +218,7 @@ protected void doStart(ClusterState clusterState) { } }); threadPool.executor(executor) - .execute(ActionRunnable.wrap(delegate, l -> masterOperation(task, request, clusterState, l))); + .execute(ActionRunnable.wrap(delegate, l -> clusterManagerOperation(task, request, clusterState, l))); } } else { if (nodes.getClusterManagerNode() == null) { @@ -304,4 +321,5 @@ protected String getClusterManagerActionName(DiscoveryNode node) { protected String getMasterActionName(DiscoveryNode node) { return getClusterManagerActionName(node); } + } diff --git a/server/src/main/java/org/opensearch/action/support/clustermanager/info/TransportClusterInfoAction.java b/server/src/main/java/org/opensearch/action/support/clustermanager/info/TransportClusterInfoAction.java index caf89fc7b6c8e..1411ff7b30695 100644 --- a/server/src/main/java/org/opensearch/action/support/clustermanager/info/TransportClusterInfoAction.java +++ b/server/src/main/java/org/opensearch/action/support/clustermanager/info/TransportClusterInfoAction.java @@ -76,16 +76,33 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) .indicesBlockedException(ClusterBlockLevel.METADATA_READ, indexNameExpressionResolver.concreteIndexNames(state, request)); } - @Override + /** @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #clusterManagerOperation(ClusterInfoRequest, ClusterState, ActionListener)} */ + @Deprecated protected final void masterOperation(final Request request, final ClusterState state, final ActionListener listener) { + clusterManagerOperation(request, state, listener); + } + + @Override + protected final void clusterManagerOperation(final Request request, final ClusterState state, final ActionListener listener) { String[] concreteIndices = indexNameExpressionResolver.concreteIndexNames(state, request); - doMasterOperation(request, concreteIndices, state, listener); + doClusterManagerOperation(request, concreteIndices, state, listener); } - protected abstract void doMasterOperation( + protected void doClusterManagerOperation( Request request, String[] concreteIndices, ClusterState state, ActionListener listener - ); + ) { + doMasterOperation(request, concreteIndices, state, listener); + } + + /** + * @deprecated As of 2.2, because supporting inclusive language, replaced by {@link #doClusterManagerOperation(ClusterInfoRequest, String[], ClusterState, ActionListener)} + */ + @Deprecated + protected void doMasterOperation(Request request, String[] concreteIndices, ClusterState state, ActionListener listener) { + throw new UnsupportedOperationException("Must be overridden"); + } + } diff --git a/server/src/main/java/org/opensearch/action/support/master/TransportMasterNodeAction.java b/server/src/main/java/org/opensearch/action/support/master/TransportMasterNodeAction.java index c26fa5c343b5c..2524e839ba11e 100644 --- a/server/src/main/java/org/opensearch/action/support/master/TransportMasterNodeAction.java +++ b/server/src/main/java/org/opensearch/action/support/master/TransportMasterNodeAction.java @@ -32,11 +32,9 @@ package org.opensearch.action.support.master; -import org.opensearch.action.ActionListener; import org.opensearch.action.ActionResponse; import org.opensearch.action.support.ActionFilters; import org.opensearch.action.support.clustermanager.TransportClusterManagerNodeAction; -import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.io.stream.Writeable; @@ -87,7 +85,4 @@ protected TransportMasterNodeAction( ); } - @Deprecated - protected abstract void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception; - } diff --git a/server/src/main/java/org/opensearch/action/support/master/info/TransportClusterInfoAction.java b/server/src/main/java/org/opensearch/action/support/master/info/TransportClusterInfoAction.java index 974fad445be9e..26d31b874f2c0 100644 --- a/server/src/main/java/org/opensearch/action/support/master/info/TransportClusterInfoAction.java +++ b/server/src/main/java/org/opensearch/action/support/master/info/TransportClusterInfoAction.java @@ -31,10 +31,8 @@ package org.opensearch.action.support.master.info; -import org.opensearch.action.ActionListener; import org.opensearch.action.ActionResponse; import org.opensearch.action.support.ActionFilters; -import org.opensearch.cluster.ClusterState; import org.opensearch.cluster.metadata.IndexNameExpressionResolver; import org.opensearch.cluster.service.ClusterService; import org.opensearch.common.io.stream.Writeable; @@ -61,11 +59,4 @@ public TransportClusterInfoAction( super(actionName, transportService, clusterService, threadPool, actionFilters, request, indexNameExpressionResolver); } - @Deprecated - protected abstract void doMasterOperation( - Request request, - String[] concreteIndices, - ClusterState state, - ActionListener listener - ); } diff --git a/server/src/main/java/org/opensearch/persistent/CompletionPersistentTaskAction.java b/server/src/main/java/org/opensearch/persistent/CompletionPersistentTaskAction.java index 1dda269a68491..d036457ccae89 100644 --- a/server/src/main/java/org/opensearch/persistent/CompletionPersistentTaskAction.java +++ b/server/src/main/java/org/opensearch/persistent/CompletionPersistentTaskAction.java @@ -193,7 +193,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) } @Override - protected final void masterOperation( + protected final void clusterManagerOperation( final Request request, ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/persistent/RemovePersistentTaskAction.java b/server/src/main/java/org/opensearch/persistent/RemovePersistentTaskAction.java index 56436d0e1aa0c..d07dcc23056d8 100644 --- a/server/src/main/java/org/opensearch/persistent/RemovePersistentTaskAction.java +++ b/server/src/main/java/org/opensearch/persistent/RemovePersistentTaskAction.java @@ -181,7 +181,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) } @Override - protected final void masterOperation( + protected final void clusterManagerOperation( final Request request, ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/persistent/StartPersistentTaskAction.java b/server/src/main/java/org/opensearch/persistent/StartPersistentTaskAction.java index 6f2f3a565427b..4864cf3c23b50 100644 --- a/server/src/main/java/org/opensearch/persistent/StartPersistentTaskAction.java +++ b/server/src/main/java/org/opensearch/persistent/StartPersistentTaskAction.java @@ -254,7 +254,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) } @Override - protected final void masterOperation( + protected final void clusterManagerOperation( final Request request, ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/persistent/UpdatePersistentTaskStatusAction.java b/server/src/main/java/org/opensearch/persistent/UpdatePersistentTaskStatusAction.java index aee79ac1ba3ea..acbb62373ab60 100644 --- a/server/src/main/java/org/opensearch/persistent/UpdatePersistentTaskStatusAction.java +++ b/server/src/main/java/org/opensearch/persistent/UpdatePersistentTaskStatusAction.java @@ -213,7 +213,7 @@ protected ClusterBlockException checkBlock(Request request, ClusterState state) } @Override - protected final void masterOperation( + protected final void clusterManagerOperation( final Request request, final ClusterState state, final ActionListener listener diff --git a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java index 024fb6ec53089..0aa967f87be9b 100644 --- a/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java +++ b/server/src/main/java/org/opensearch/snapshots/SnapshotsService.java @@ -3654,7 +3654,7 @@ protected UpdateIndexShardSnapshotStatusResponse read(StreamInput in) throws IOE } @Override - protected void masterOperation( + protected void clusterManagerOperation( UpdateIndexShardSnapshotStatusRequest request, ClusterState state, ActionListener listener diff --git a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java index 5648f14fa69dd..001efb32c2988 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/get/GetIndexActionTests.java @@ -145,14 +145,14 @@ class TestTransportGetIndexAction extends TransportGetIndexAction { } @Override - protected void doMasterOperation( + protected void doClusterManagerOperation( GetIndexRequest request, String[] concreteIndices, ClusterState state, ActionListener listener ) { ClusterState stateWithIndex = ClusterStateCreationUtils.state(indexName, 1, 1); - super.doMasterOperation(request, concreteIndices, stateWithIndex, listener); + super.doClusterManagerOperation(request, concreteIndices, stateWithIndex, listener); } } diff --git a/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java index 8ef8de81041e3..b206c2e19a65b 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/rollover/TransportRolloverActionTests.java @@ -298,7 +298,7 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr RolloverRequest rolloverRequest = new RolloverRequest("logs-alias", "logs-index-000003"); rolloverRequest.addMaxIndexDocsCondition(500L); rolloverRequest.dryRun(true); - transportRolloverAction.masterOperation(mock(Task.class), rolloverRequest, stateBefore, future); + transportRolloverAction.clusterManagerOperation(mock(Task.class), rolloverRequest, stateBefore, future); RolloverResponse response = future.actionGet(); assertThat(response.getOldIndex(), equalTo("logs-index-000002")); @@ -314,7 +314,7 @@ public void testConditionEvaluationWhenAliasToWriteAndReadIndicesConsidersOnlyPr rolloverRequest = new RolloverRequest("logs-alias", "logs-index-000003"); rolloverRequest.addMaxIndexDocsCondition(300L); rolloverRequest.dryRun(true); - transportRolloverAction.masterOperation(mock(Task.class), rolloverRequest, stateBefore, future); + transportRolloverAction.clusterManagerOperation(mock(Task.class), rolloverRequest, stateBefore, future); response = future.actionGet(); assertThat(response.getOldIndex(), equalTo("logs-index-000002")); diff --git a/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsActionTests.java b/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsActionTests.java index cefb59c40949c..dfbc1dccb9d6a 100644 --- a/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsActionTests.java +++ b/server/src/test/java/org/opensearch/action/admin/indices/settings/get/GetSettingsActionTests.java @@ -84,9 +84,13 @@ class TestTransportGetSettingsAction extends TransportGetSettingsAction { } @Override - protected void masterOperation(GetSettingsRequest request, ClusterState state, ActionListener listener) { + protected void clusterManagerOperation( + GetSettingsRequest request, + ClusterState state, + ActionListener listener + ) { ClusterState stateWithIndex = ClusterStateCreationUtils.state(indexName, 1, 1); - super.masterOperation(request, stateWithIndex, listener); + super.clusterManagerOperation(request, stateWithIndex, listener); } } diff --git a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java index f0b62914213c1..9ec9e656257d6 100644 --- a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java +++ b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportClusterManagerNodeActionTests.java @@ -229,7 +229,7 @@ protected Response read(StreamInput in) throws IOException { } @Override - protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + protected void clusterManagerOperation(Request request, ClusterState state, ActionListener listener) throws Exception { listener.onResponse(new Response()); // default implementation, overridden in specific tests } @@ -252,7 +252,7 @@ public void testLocalOperationWithoutBlocks() throws ExecutionException, Interru new Action("internal:testAction", transportService, clusterService, threadPool) { @Override - protected void masterOperation(Task task, Request request, ClusterState state, ActionListener listener) { + protected void clusterManagerOperation(Task task, Request request, ClusterState state, ActionListener listener) { if (clusterManagerOperationFailure) { listener.onFailure(exception); } else { @@ -511,7 +511,8 @@ public void testClusterManagerFailoverAfterStepDown() throws ExecutionException, new Action("internal:testAction", transportService, clusterService, threadPool) { @Override - protected void masterOperation(Request request, ClusterState state, ActionListener listener) throws Exception { + protected void clusterManagerOperation(Request request, ClusterState state, ActionListener listener) + throws Exception { // The other node has become cluster-manager, simulate failures of this node while publishing cluster state through // ZenDiscovery setState(clusterService, ClusterStateCreationUtils.state(localNode, remoteNode, allNodes)); diff --git a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportMasterNodeActionUtils.java b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportMasterNodeActionUtils.java index 3927cd1d13040..b9abddc5622c9 100644 --- a/server/src/test/java/org/opensearch/action/support/clustermanager/TransportMasterNodeActionUtils.java +++ b/server/src/test/java/org/opensearch/action/support/clustermanager/TransportMasterNodeActionUtils.java @@ -39,7 +39,7 @@ public class TransportMasterNodeActionUtils { /** - * Allows to directly call {@link TransportClusterManagerNodeAction#masterOperation(ClusterManagerNodeRequest, ClusterState, ActionListener)} which is + * Allows to directly call {@link TransportClusterManagerNodeAction#clusterManagerOperation(ClusterManagerNodeRequest, ClusterState, ActionListener)} which is * a protected method. */ public static , Response extends ActionResponse> void runClusterManagerOperation( @@ -49,6 +49,6 @@ public static , Response exte ActionListener actionListener ) throws Exception { assert clusterManagerNodeAction.checkBlock(request, clusterState) == null; - clusterManagerNodeAction.masterOperation(request, clusterState, actionListener); + clusterManagerNodeAction.clusterManagerOperation(request, clusterState, actionListener); } } diff --git a/server/src/test/java/org/opensearch/indices/settings/InternalOrPrivateSettingsPlugin.java b/server/src/test/java/org/opensearch/indices/settings/InternalOrPrivateSettingsPlugin.java index 775b4bb185881..2e244908dc4eb 100644 --- a/server/src/test/java/org/opensearch/indices/settings/InternalOrPrivateSettingsPlugin.java +++ b/server/src/test/java/org/opensearch/indices/settings/InternalOrPrivateSettingsPlugin.java @@ -173,7 +173,7 @@ protected UpdateInternalOrPrivateAction.Response read(StreamInput in) throws IOE } @Override - protected void masterOperation( + protected void clusterManagerOperation( final UpdateInternalOrPrivateAction.Request request, final ClusterState state, final ActionListener listener diff --git a/test/framework/src/main/java/org/opensearch/test/ExternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/ExternalTestCluster.java index 33147447bd469..e84bde1a064a8 100644 --- a/test/framework/src/main/java/org/opensearch/test/ExternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/ExternalTestCluster.java @@ -190,7 +190,7 @@ public int numDataNodes() { } @Override - public int numDataAndMasterNodes() { + public int numDataAndClusterManagerNodes() { return numClusterManagerAndDataNodes; } diff --git a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java index c68e1eac3e1d8..d5925c03db480 100644 --- a/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/InternalTestCluster.java @@ -2348,7 +2348,7 @@ public int numDataNodes() { } @Override - public int numDataAndMasterNodes() { + public int numDataAndClusterManagerNodes() { return filterNodes(nodes, DATA_NODE_PREDICATE.or(CLUSTER_MANAGER_NODE_PREDICATE)).size(); } diff --git a/test/framework/src/main/java/org/opensearch/test/TestCluster.java b/test/framework/src/main/java/org/opensearch/test/TestCluster.java index 26081d947431d..c2e90f0369e6c 100644 --- a/test/framework/src/main/java/org/opensearch/test/TestCluster.java +++ b/test/framework/src/main/java/org/opensearch/test/TestCluster.java @@ -127,7 +127,18 @@ public void assertAfterTest() throws Exception { /** * Returns the number of data and cluster-manager eligible nodes in the cluster. */ - public abstract int numDataAndMasterNodes(); + public int numDataAndClusterManagerNodes() { + return numDataAndMasterNodes(); + } + + /** + * Returns the number of data and cluster-manager eligible nodes in the cluster. + * @deprecated As of 2.1, because supporting inclusive language, replaced by {@link #numDataAndClusterManagerNodes()} + */ + @Deprecated + public int numDataAndMasterNodes() { + throw new UnsupportedOperationException("Must be overridden"); + } /** * Returns the http addresses of the nodes within the cluster.