Merge remote-tracking branch 'elastic/master' into index-not-found-ex…

…ception-logging

client/rest-high-level/src/main/java/org/elasticsearch/client/security/user/privileges/Role.java

@@ -311,6 +311,7 @@ public static class ClusterPrivilegeName {
         public static final String TRANSPORT_CLIENT = "transport_client";
         public static final String MANAGE_SECURITY = "manage_security";
         public static final String MANAGE_SAML = "manage_saml";
+        public static final String MANAGE_OIDC = "manage_oidc";
         public static final String MANAGE_TOKEN = "manage_token";
         public static final String MANAGE_PIPELINE = "manage_pipeline";
         public static final String MANAGE_CCR = "manage_ccr";
@@ -319,7 +320,7 @@ public static class ClusterPrivilegeName {
         public static final String READ_ILM = "read_ilm";
         public static final String[] ALL_ARRAY = new String[] { NONE, ALL, MONITOR, MONITOR_ML, MONITOR_WATCHER, MONITOR_ROLLUP, MANAGE,
                 MANAGE_ML, MANAGE_WATCHER, MANAGE_ROLLUP, MANAGE_INDEX_TEMPLATES, MANAGE_INGEST_PIPELINES, TRANSPORT_CLIENT,
-                MANAGE_SECURITY, MANAGE_SAML, MANAGE_TOKEN, MANAGE_PIPELINE, MANAGE_CCR, READ_CCR, MANAGE_ILM, READ_ILM };
+            MANAGE_SECURITY, MANAGE_SAML, MANAGE_OIDC, MANAGE_TOKEN, MANAGE_PIPELINE, MANAGE_CCR, READ_CCR, MANAGE_ILM, READ_ILM};
     }
 
     /**

modules/reindex/src/main/java/org/elasticsearch/index/reindex/TransportUpdateByQueryAction.java

@@ -20,7 +20,6 @@
 package org.elasticsearch.index.reindex;
 
 import org.apache.logging.log4j.Logger;
-import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.support.ActionFilters;
@@ -82,18 +81,13 @@ protected void doExecute(Task task, UpdateByQueryRequest request, ActionListener
      */
     static class AsyncIndexBySearchAction extends AbstractAsyncBulkByScrollAction<UpdateByQueryRequest, TransportUpdateByQueryAction> {
 
-        private final boolean useSeqNoForCAS;
-
         AsyncIndexBySearchAction(BulkByScrollTask task, Logger logger, ParentTaskAssigningClient client,
                 ThreadPool threadPool, TransportUpdateByQueryAction action, UpdateByQueryRequest request, ClusterState clusterState,
                 ActionListener<BulkByScrollResponse> listener) {
             super(task,
-                // not all nodes support sequence number powered optimistic concurrency control, we fall back to version
-                clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0) == false,
-                // all nodes support sequence number powered optimistic concurrency control and we can use it
-                clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0),
+                // use sequence number powered optimistic concurrency control
+                false, true,
                 logger, client, threadPool, action, request, listener);
-            useSeqNoForCAS = clusterState.nodes().getMinNodeVersion().onOrAfter(Version.V_6_7_0);
         }
 
         @Override

server/src/main/java/org/elasticsearch/Build.java

@@ -254,12 +254,7 @@ public static void writeBuild(Build build, StreamOutput out) throws IOException
             out.writeString(build.flavor().displayName());
         }
         if (out.getVersion().onOrAfter(Version.V_6_3_0)) {
-            final Type buildType;
-            if (out.getVersion().before(Version.V_6_7_0) && build.type() == Type.DOCKER) {
-                buildType = Type.TAR;
-            } else {
-                buildType = build.type();
-            }
+            final Type buildType = build.type();
             out.writeString(buildType.displayName());
         }
         out.writeString(build.shortHash());

server/src/main/java/org/elasticsearch/ElasticsearchException.java

@@ -1006,23 +1006,23 @@ private enum ElasticsearchExceptionHandle {
         UNKNOWN_NAMED_OBJECT_EXCEPTION(org.elasticsearch.common.xcontent.UnknownNamedObjectException.class,
                 org.elasticsearch.common.xcontent.UnknownNamedObjectException::new, 148, UNKNOWN_VERSION_ADDED),
         TOO_MANY_BUCKETS_EXCEPTION(MultiBucketConsumerService.TooManyBucketsException.class,
-            MultiBucketConsumerService.TooManyBucketsException::new, 149, Version.V_6_2_0),
+            MultiBucketConsumerService.TooManyBucketsException::new, 149, UNKNOWN_VERSION_ADDED),
         COORDINATION_STATE_REJECTED_EXCEPTION(org.elasticsearch.cluster.coordination.CoordinationStateRejectedException.class,
             org.elasticsearch.cluster.coordination.CoordinationStateRejectedException::new, 150, Version.V_7_0_0),
         SNAPSHOT_IN_PROGRESS_EXCEPTION(org.elasticsearch.snapshots.SnapshotInProgressException.class,
-            org.elasticsearch.snapshots.SnapshotInProgressException::new, 151, Version.V_6_7_0),
+            org.elasticsearch.snapshots.SnapshotInProgressException::new, 151, UNKNOWN_VERSION_ADDED),
         NO_SUCH_REMOTE_CLUSTER_EXCEPTION(org.elasticsearch.transport.NoSuchRemoteClusterException.class,
-            org.elasticsearch.transport.NoSuchRemoteClusterException::new, 152, Version.V_6_7_0),
+            org.elasticsearch.transport.NoSuchRemoteClusterException::new, 152, UNKNOWN_VERSION_ADDED),
         RETENTION_LEASE_ALREADY_EXISTS_EXCEPTION(
                 org.elasticsearch.index.seqno.RetentionLeaseAlreadyExistsException.class,
                 org.elasticsearch.index.seqno.RetentionLeaseAlreadyExistsException::new,
                 153,
-                Version.V_6_7_0),
+                UNKNOWN_VERSION_ADDED),
         RETENTION_LEASE_NOT_FOUND_EXCEPTION(
                 org.elasticsearch.index.seqno.RetentionLeaseNotFoundException.class,
                 org.elasticsearch.index.seqno.RetentionLeaseNotFoundException::new,
                 154,
-                Version.V_6_7_0),
+                UNKNOWN_VERSION_ADDED),
         SHARD_NOT_IN_PRIMARY_MODE_EXCEPTION(
                 org.elasticsearch.index.shard.ShardNotInPrimaryModeException.class,
                 org.elasticsearch.index.shard.ShardNotInPrimaryModeException::new,

server/src/main/java/org/elasticsearch/Version.java

@@ -48,16 +48,6 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_EMPTY = new Version(V_EMPTY_ID, org.apache.lucene.util.Version.LATEST);
     // The below version is missing from the 7.3 JAR
     private static final org.apache.lucene.util.Version LUCENE_7_2_1 = org.apache.lucene.util.Version.fromBits(7, 2, 1);
-    public static final int V_6_2_0_ID = 6020099;
-    public static final Version V_6_2_0 = new Version(V_6_2_0_ID, LUCENE_7_2_1);
-    public static final int V_6_2_1_ID = 6020199;
-    public static final Version V_6_2_1 = new Version(V_6_2_1_ID, LUCENE_7_2_1);
-    public static final int V_6_2_2_ID = 6020299;
-    public static final Version V_6_2_2 = new Version(V_6_2_2_ID, LUCENE_7_2_1);
-    public static final int V_6_2_3_ID = 6020399;
-    public static final Version V_6_2_3 = new Version(V_6_2_3_ID, LUCENE_7_2_1);
-    public static final int V_6_2_4_ID = 6020499;
-    public static final Version V_6_2_4 = new Version(V_6_2_4_ID, LUCENE_7_2_1);
     public static final int V_6_3_0_ID = 6030099;
     public static final Version V_6_3_0 = new Version(V_6_3_0_ID, org.apache.lucene.util.Version.LUCENE_7_3_1);
     public static final int V_6_3_1_ID = 6030199;
@@ -88,12 +78,6 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_6_6_1 = new Version(V_6_6_1_ID, org.apache.lucene.util.Version.LUCENE_7_6_0);
     public static final int V_6_6_2_ID = 6060299;
     public static final Version V_6_6_2 = new Version(V_6_6_2_ID, org.apache.lucene.util.Version.LUCENE_7_6_0);
-    public static final int V_6_7_0_ID = 6070099;
-    public static final Version V_6_7_0 = new Version(V_6_7_0_ID, org.apache.lucene.util.Version.LUCENE_7_7_0);
-    public static final int V_6_7_1_ID = 6070199;
-    public static final Version V_6_7_1 = new Version(V_6_7_1_ID, org.apache.lucene.util.Version.LUCENE_7_7_0);
-    public static final int V_6_7_2_ID = 6070299;
-    public static final Version V_6_7_2 = new Version(V_6_7_2_ID, org.apache.lucene.util.Version.LUCENE_7_7_0);
     public static final int V_6_8_0_ID = 6080099;
     public static final Version V_6_8_0 = new Version(V_6_8_0_ID, org.apache.lucene.util.Version.LUCENE_7_7_0);
     public static final int V_6_8_1_ID = 6080199;
@@ -140,12 +124,6 @@ public static Version fromId(int id) {
                 return V_6_8_1;
             case V_6_8_0_ID:
                 return V_6_8_0;
-            case V_6_7_1_ID:
-                return V_6_7_1;
-            case V_6_7_2_ID:
-                return V_6_7_2;
-            case V_6_7_0_ID:
-                return V_6_7_0;
             case V_6_6_2_ID:
                 return V_6_6_2;
             case V_6_6_1_ID:
@@ -176,16 +154,6 @@ public static Version fromId(int id) {
                 return V_6_3_1;
             case V_6_3_0_ID:
                 return V_6_3_0;
-            case V_6_2_4_ID:
-                return V_6_2_4;
-            case V_6_2_3_ID:
-                return V_6_2_3;
-            case V_6_2_2_ID:
-                return V_6_2_2;
-            case V_6_2_1_ID:
-                return V_6_2_1;
-            case V_6_2_0_ID:
-                return V_6_2_0;
             case V_EMPTY_ID:
                 return V_EMPTY;
             default:

server/src/main/java/org/elasticsearch/cluster/coordination/Reconfigurator.java

@@ -27,15 +27,10 @@
 import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Setting.Property;
 import org.elasticsearch.common.settings.Settings;
-import org.elasticsearch.common.util.set.Sets;
 
-import java.util.Collection;
-import java.util.Collections;
 import java.util.Set;
-import java.util.SortedSet;
 import java.util.TreeSet;
 import java.util.stream.Collectors;
-import java.util.stream.Stream;
 
 /**
  * Computes the optimal configuration of voting nodes in the cluster.
@@ -102,81 +97,86 @@ public VotingConfiguration reconfigure(Set<DiscoveryNode> liveNodes, Set<String>
         logger.trace("{} reconfiguring {} based on liveNodes={}, retiredNodeIds={}, currentMaster={}",
             this, currentConfig, liveNodes, retiredNodeIds, currentMaster);
 
-        /*
-         *  There are three true/false properties of each node in play: live/non-live, retired/non-retired and in-config/not-in-config.
-         *  Firstly we divide the nodes into disjoint sets based on these properties:
-         *
-         *  - nonRetiredMaster
-         *  - nonRetiredNotMasterInConfigNotLiveIds
-         *  - nonRetiredInConfigLiveIds
-         *  - nonRetiredLiveNotInConfigIds
-         *
-         *  The other 5 possibilities are not relevant:
-         *  - retired, in-config, live             -- retired nodes should be removed from the config
-         *  - retired, in-config, non-live         -- retired nodes should be removed from the config
-         *  - retired, not-in-config, live         -- cannot add a retired node back to the config
-         *  - retired, not-in-config, non-live     -- cannot add a retired node back to the config
-         *  - non-retired, non-live, not-in-config -- no evidence this node exists at all
-         */
-
         final Set<String> liveNodeIds = liveNodes.stream()
             .filter(DiscoveryNode::isMasterNode).map(DiscoveryNode::getId).collect(Collectors.toSet());
-        final Set<String> liveInConfigIds = new TreeSet<>(currentConfig.getNodeIds());
-        liveInConfigIds.retainAll(liveNodeIds);
-
-        final SortedSet<String> inConfigNotLiveIds = Sets.unmodifiableSortedDifference(currentConfig.getNodeIds(), liveInConfigIds);
-        final SortedSet<String> nonRetiredInConfigNotLiveIds = new TreeSet<>(inConfigNotLiveIds);
-        nonRetiredInConfigNotLiveIds.removeAll(retiredNodeIds);
-
-        final Set<String> nonRetiredInConfigLiveIds = new TreeSet<>(liveInConfigIds);
-        nonRetiredInConfigLiveIds.removeAll(retiredNodeIds);
-
-        final Set<String> nonRetiredInConfigLiveMasterIds;
-        final Set<String> nonRetiredInConfigLiveNotMasterIds;
-        if (nonRetiredInConfigLiveIds.contains(currentMaster.getId())) {
-            nonRetiredInConfigLiveNotMasterIds = new TreeSet<>(nonRetiredInConfigLiveIds);
-            nonRetiredInConfigLiveNotMasterIds.remove(currentMaster.getId());
-            nonRetiredInConfigLiveMasterIds = Collections.singleton(currentMaster.getId());
-        } else {
-            nonRetiredInConfigLiveNotMasterIds = nonRetiredInConfigLiveIds;
-            nonRetiredInConfigLiveMasterIds = Collections.emptySet();
-        }
-
-        final SortedSet<String> nonRetiredLiveNotInConfigIds = Sets.sortedDifference(liveNodeIds, currentConfig.getNodeIds());
-        nonRetiredLiveNotInConfigIds.removeAll(retiredNodeIds);
+        final Set<String> currentConfigNodeIds = currentConfig.getNodeIds();
+
+        final Set<VotingConfigNode> orderedCandidateNodes = new TreeSet<>();
+        liveNodes.stream()
+            .filter(DiscoveryNode::isMasterNode)
+            .filter(n -> retiredNodeIds.contains(n.getId()) == false)
+            .forEach(n -> orderedCandidateNodes.add(new VotingConfigNode(n.getId(), true,
+                n.getId().equals(currentMaster.getId()), currentConfigNodeIds.contains(n.getId()))));
+        currentConfigNodeIds.stream()
+            .filter(nid -> liveNodeIds.contains(nid) == false)
+            .filter(nid -> retiredNodeIds.contains(nid) == false)
+            .forEach(nid -> orderedCandidateNodes.add(new VotingConfigNode(nid, false, false, true)));
 
         /*
          * Now we work out how many nodes should be in the configuration:
          */
-        final int targetSize;
-
-        final int nonRetiredLiveNodeCount = nonRetiredInConfigLiveIds.size() + nonRetiredLiveNotInConfigIds.size();
-        final int nonRetiredConfigSize = nonRetiredInConfigLiveIds.size() + nonRetiredInConfigNotLiveIds.size();
-        if (autoShrinkVotingConfiguration) {
-            if (nonRetiredLiveNodeCount >= 3) {
-                targetSize = roundDownToOdd(nonRetiredLiveNodeCount);
-            } else {
-                // only have one or two available nodes; may not shrink below 3 nodes automatically, but if
-                // the config (excluding retired nodes) is already smaller than 3 then it's ok.
-                targetSize = nonRetiredConfigSize < 3 ? 1 : 3;
-            }
-        } else {
-            targetSize = Math.max(roundDownToOdd(nonRetiredLiveNodeCount), nonRetiredConfigSize);
-        }
+        final int nonRetiredConfigSize = Math.toIntExact(orderedCandidateNodes.stream().filter(n -> n.inCurrentConfig).count());
+        final int minimumConfigEnforcedSize = autoShrinkVotingConfiguration ? (nonRetiredConfigSize < 3 ? 1 : 3) : nonRetiredConfigSize;
+        final int nonRetiredLiveNodeCount = Math.toIntExact(orderedCandidateNodes.stream().filter(n -> n.live).count());
+        final int targetSize = Math.max(roundDownToOdd(nonRetiredLiveNodeCount), minimumConfigEnforcedSize);
 
-        /*
-         * The new configuration is formed by taking this many nodes in the following preference order:
-         */
         final VotingConfiguration newConfig = new VotingConfiguration(
-            // live master first, then other live nodes, preferring the current config, and if we need more then use non-live nodes
-            Stream.of(nonRetiredInConfigLiveMasterIds, nonRetiredInConfigLiveNotMasterIds, nonRetiredLiveNotInConfigIds,
-                nonRetiredInConfigNotLiveIds).flatMap(Collection::stream).limit(targetSize).collect(Collectors.toSet()));
+            orderedCandidateNodes.stream()
+                .limit(targetSize)
+                .map(n -> n.id)
+                .collect(Collectors.toSet()));
 
+        // new configuration should have a quorum
         if (newConfig.hasQuorum(liveNodeIds)) {
             return newConfig;
         } else {
             // If there are not enough live nodes to form a quorum in the newly-proposed configuration, it's better to do nothing.
             return currentConfig;
         }
     }
+
+    static class VotingConfigNode implements Comparable<VotingConfigNode> {
+        final String id;
+        final boolean live;
+        final boolean currentMaster;
+        final boolean inCurrentConfig;
+
+        VotingConfigNode(String id, boolean live, boolean currentMaster, boolean inCurrentConfig) {
+            this.id = id;
+            this.live = live;
+            this.currentMaster = currentMaster;
+            this.inCurrentConfig = inCurrentConfig;
+        }
+
+        @Override
+        public int compareTo(VotingConfigNode other) {
+            // prefer nodes that are live
+            final int liveComp = Boolean.compare(other.live, live);
+            if (liveComp != 0) {
+                return liveComp;
+            }
+            // prefer nodes that are in current config for stability
+            final int inCurrentConfigComp = Boolean.compare(other.inCurrentConfig, inCurrentConfig);
+            if (inCurrentConfigComp != 0) {
+                return inCurrentConfigComp;
+            }
+            // prefer current master
+            final int currentMasterComp = Boolean.compare(other.currentMaster, currentMaster);
+            if (currentMasterComp != 0) {
+                return currentMasterComp;
+            }
+            // tiebreak by node id to have stable ordering
+            return id.compareTo(other.id);
+        }
+
+        @Override
+        public String toString() {
+            return "VotingConfigNode{" +
+                "id='" + id + '\'' +
+                ", live=" + live +
+                ", currentMaster=" + currentMaster +
+                ", inCurrentConfig=" + inCurrentConfig +
+                '}';
+        }
+    }
 }

server/src/main/java/org/elasticsearch/index/engine/InternalEngine.java

@@ -513,15 +513,8 @@ public Translog.Snapshot readHistoryOperations(String source, MapperService mapp
      * Returns the estimated number of history operations whose seq# at least the provided seq# in this engine.
      */
     @Override
-    public int estimateNumberOfHistoryOperations(String source, MapperService mapperService, long startingSeqNo) throws IOException {
-        if (engineConfig.getIndexSettings().isSoftDeleteEnabled()) {
-            try (Translog.Snapshot snapshot = newChangesSnapshot(source, mapperService, Math.max(0, startingSeqNo),
-                Long.MAX_VALUE, false)) {
-                return snapshot.totalOperations();
-            }
-        } else {
-            return getTranslog().estimateTotalOperationsFromMinSeq(startingSeqNo);
-        }
+    public int estimateNumberOfHistoryOperations(String source, MapperService mapperService, long startingSeqNo) {
+        return getTranslog().estimateTotalOperationsFromMinSeq(startingSeqNo);
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/engine/ReadOnlyEngine.java

@@ -131,7 +131,7 @@ protected void ensureMaxSeqNoEqualsToGlobalCheckpoint(final SeqNoStats seqNoStat
         // that guarantee that all operations have been flushed to Lucene.
         final Version indexVersionCreated = engineConfig.getIndexSettings().getIndexVersionCreated();
         if (indexVersionCreated.onOrAfter(Version.V_7_2_0) ||
-            (seqNoStats.getGlobalCheckpoint() != SequenceNumbers.UNASSIGNED_SEQ_NO && indexVersionCreated.onOrAfter(Version.V_6_7_0))) {
+            (seqNoStats.getGlobalCheckpoint() != SequenceNumbers.UNASSIGNED_SEQ_NO)) {
             if (seqNoStats.getMaxSeqNo() != seqNoStats.getGlobalCheckpoint()) {
                 throw new IllegalStateException("Maximum sequence number [" + seqNoStats.getMaxSeqNo()
                     + "] from last commit does not match global checkpoint [" + seqNoStats.getGlobalCheckpoint() + "]");

server/src/main/java/org/elasticsearch/index/shard/IndexShard.java

@@ -2136,8 +2136,8 @@ public void maybeSyncGlobalCheckpoint(final String reason) {
                     StreamSupport
                             .stream(globalCheckpoints.values().spliterator(), false)
                             .anyMatch(v -> v.value < globalCheckpoint);
-            // only sync if there is a shard lagging the primary
-            if (syncNeeded) {
+            // only sync if index is not closed and there is a shard lagging the primary
+            if (syncNeeded && indexSettings.getIndexMetaData().getState() == IndexMetaData.State.OPEN) {
                 logger.trace("syncing global checkpoint for [{}]", reason);
                 globalCheckpointSyncer.run();
             }