Remove Zen1 (#39466)

Removes all traces of Zen1 from the code base. Some of these commits will also be backported to
7.0/7.x (#39470) as the cluster.coordination package was making use of some things in
discovery.zen and we want to keep 7.x as close as possible to master.

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -63,19 +63,6 @@ class ClusterConfiguration {
     @Input
     boolean debug = false
 
-    /**
-     * Configuration of the setting {@code discovery.zen.minimum_master_nodes} on the nodes.
-     * In case of more than one node, this defaults to the number of nodes
-     */
-    @Input
-    Closure<Integer> minimumMasterNodes = {
-        if (bwcVersion != null && bwcVersion.before("6.5.0")) {
-            return numNodes > 1 ? numNodes : -1
-        } else {
-            return numNodes > 1 ? numNodes.intdiv(2) + 1 : -1
-        }
-    }
-
     /**
      * Whether the initial_master_nodes setting should be automatically derived from the nodes
      * in the cluster. Only takes effect if all nodes in the cluster understand this setting

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -127,42 +127,25 @@ class ClusterFormationTasks {
             nodes.add(node)
             Closure<Map> writeConfigSetup
             Object dependsOn
-            if (node.nodeVersion.onOrAfter("6.5.0")) {
-                writeConfigSetup = { Map esConfig ->
-                    if (config.getAutoSetHostsProvider()) {
-                        // Don't force discovery provider if one is set by the test cluster specs already
-                        final String seedProvidersSettingName =
-                                node.nodeVersion.onOrAfter("7.0.0") ? "discovery.seed_providers" : "discovery.zen.hosts_provider";
-                        if (esConfig.containsKey(seedProvidersSettingName) == false) {
-                            esConfig[seedProvidersSettingName] = 'file'
-                        }
-                        esConfig[node.nodeVersion.onOrAfter("7.0.0") ? "discovery.seed_hosts" : "discovery.zen.ping.unicast.hosts"] = []
-                    }
-                    boolean supportsInitialMasterNodes = hasBwcNodes == false || config.bwcVersion.onOrAfter("7.0.0")
-                    if (esConfig['discovery.type'] == null && config.getAutoSetInitialMasterNodes() && supportsInitialMasterNodes) {
-                        esConfig['cluster.initial_master_nodes'] = nodes.stream().map({ n ->
-                            if (n.config.settings['node.name'] == null) {
-                                return "node-" + n.nodeNum
-                            } else {
-                                return n.config.settings['node.name']
-                            }
-                        }).collect(Collectors.toList())
+            writeConfigSetup = { Map esConfig ->
+                if (config.getAutoSetHostsProvider()) {
+                    if (esConfig.containsKey("discovery.seed_providers") == false) {
+                        esConfig["discovery.seed_providers"] = 'file'
                     }
-                    esConfig
+                    esConfig["discovery.seed_hosts"] = []
                 }
-                dependsOn = startDependencies
-            } else {
-                dependsOn = startTasks.empty ? startDependencies : startTasks.get(0)
-                writeConfigSetup = { Map esConfig ->
-                    String unicastTransportUri = node.config.unicastTransportUri(nodes.get(0), node, project.createAntBuilder())
-                    if (unicastTransportUri == null) {
-                        esConfig['discovery.zen.ping.unicast.hosts'] = []
-                    } else {
-                        esConfig['discovery.zen.ping.unicast.hosts'] = "\"${unicastTransportUri}\""
-                    }
-                    esConfig
+                if (esConfig['discovery.type'] == null && config.getAutoSetInitialMasterNodes()) {
+                    esConfig['cluster.initial_master_nodes'] = nodes.stream().map({ n ->
+                        if (n.config.settings['node.name'] == null) {
+                            return "node-" + n.nodeNum
+                        } else {
+                            return n.config.settings['node.name']
+                        }
+                    }).collect(Collectors.toList())
                 }
+                esConfig
             }
+            dependsOn = startDependencies
             startTasks.add(configureNode(project, prefix, runner, dependsOn, node, config, distro, writeConfigSetup))
         }
 
@@ -381,17 +364,6 @@ class ClusterFormationTasks {
                 // Don't wait for state, just start up quickly. This will also allow new and old nodes in the BWC case to become the master
                 'discovery.initial_state_timeout' : '0s'
         ]
-        int minimumMasterNodes = node.config.minimumMasterNodes.call()
-        if (node.nodeVersion.before("7.0.0") && minimumMasterNodes > 0) {
-            esConfig['discovery.zen.minimum_master_nodes'] = minimumMasterNodes
-        }
-        if (node.nodeVersion.before("7.0.0") && esConfig.containsKey('discovery.zen.master_election.wait_for_joins_timeout') == false) {
-            // If a node decides to become master based on partial information from the pinging, don't let it hang for 30 seconds to correct
-            // its mistake. Instead, only wait 5s to do another round of pinging.
-            // This is necessary since we use 30s as the default timeout in REST requests waiting for cluster formation
-            // so we need to bail quicker than the default 30s for the cluster to form in time.
-            esConfig['discovery.zen.master_election.wait_for_joins_timeout'] = '5s'
-        }
         esConfig['node.max_local_storage_nodes'] = node.config.numNodes
         esConfig['http.port'] = node.config.httpPort
         esConfig['transport.tcp.port'] =  node.config.transportPort

docs/plugins/integrations.asciidoc

@@ -194,7 +194,7 @@ releases 2.0 and later do not support rivers.
   A pluggable elastic JavaScript query DSL builder for Elasticsearch
 
 * https://www.wireshark.org/[Wireshark]:
-  Protocol dissection for Zen discovery, HTTP and the binary protocol
+  Protocol dissection for HTTP and the transport protocol
 
 * https://www.itemsapi.com/[ItemsAPI]:
   Search backend for mobile and web

docs/reference/getting-started.asciidoc

@@ -248,8 +248,8 @@ If everything goes well with installation, you should see a bunch of messages th
 [2018-09-13T12:20:05,202][INFO ][o.e.t.TransportService   ] [localhost.localdomain] publish_address {127.0.0.1:9300}, bound_addresses {[::1]:9300}, {127.0.0.1:9300}
 [2018-09-13T12:20:05,221][WARN ][o.e.b.BootstrapChecks    ] [localhost.localdomain] max file descriptors [4096] for elasticsearch process is too low, increase to at least [65535]
 [2018-09-13T12:20:05,221][WARN ][o.e.b.BootstrapChecks    ] [localhost.localdomain] max virtual memory areas vm.max_map_count [65530] is too low, increase to at least [262144]
-[2018-09-13T12:20:08,355][INFO ][o.e.c.s.MasterService    ] [localhost.localdomain] zen-disco-elected-as-master ([0] nodes joined)[, ], reason: master node changed {previous [], current [{localhost.localdomain}{B0aEHNagTiWx7SYj-l4NTw}{hzsQz6CVQMCTpMCVLM4IHg}{127.0.0.1}{127.0.0.1:9300}{testattr=test}]}
-[2018-09-13T12:20:08,360][INFO ][o.e.c.s.ClusterApplierService] [localhost.localdomain] master node changed {previous [], current [{localhost.localdomain}{B0aEHNagTiWx7SYj-l4NTw}{hzsQz6CVQMCTpMCVLM4IHg}{127.0.0.1}{127.0.0.1:9300}{testattr=test}]}, reason: apply cluster state (from master [master {localhost.localdomain}{B0aEHNagTiWx7SYj-l4NTw}{hzsQz6CVQMCTpMCVLM4IHg}{127.0.0.1}{127.0.0.1:9300}{testattr=test} committed version [1] source [zen-disco-elected-as-master ([0] nodes joined)[, ]]])
+[2018-09-13T12:20:08,355][INFO ][o.e.c.s.MasterService    ] [localhost.localdomain] elected-as-master ([0] nodes joined)[, ], reason: master node changed {previous [], current [{localhost.localdomain}{B0aEHNagTiWx7SYj-l4NTw}{hzsQz6CVQMCTpMCVLM4IHg}{127.0.0.1}{127.0.0.1:9300}{testattr=test}]}
+[2018-09-13T12:20:08,360][INFO ][o.e.c.s.ClusterApplierService] [localhost.localdomain] master node changed {previous [], current [{localhost.localdomain}{B0aEHNagTiWx7SYj-l4NTw}{hzsQz6CVQMCTpMCVLM4IHg}{127.0.0.1}{127.0.0.1:9300}{testattr=test}]}, reason: apply cluster state (from master [master {localhost.localdomain}{B0aEHNagTiWx7SYj-l4NTw}{hzsQz6CVQMCTpMCVLM4IHg}{127.0.0.1}{127.0.0.1:9300}{testattr=test} committed version [1] source [elected-as-master ([0] nodes joined)[, ]]])
 [2018-09-13T12:20:08,384][INFO ][o.e.h.n.Netty4HttpServerTransport] [localhost.localdomain] publish_address {127.0.0.1:9200}, bound_addresses {[::1]:9200}, {127.0.0.1:9200}
 [2018-09-13T12:20:08,384][INFO ][o.e.n.Node               ] [localhost.localdomain] started
 

docs/reference/migration/migrate_8_0.asciidoc

@@ -12,6 +12,7 @@ See also <<release-highlights>> and <<es-release-notes>>.
 coming[8.0.0]
 
 * <<breaking_80_analysis_changes>>
+* <<breaking_80_discovery_changes>>
 * <<breaking_80_mappings_changes>>
 * <<breaking_80_snapshots_changes>>
 
@@ -32,5 +33,6 @@ Elasticsearch 7.x in order to be readable by Elasticsearch 8.x.
 =========================================
 
 include::migrate_8_0/analysis.asciidoc[]
+include::migrate_8_0/discovery.asciidoc[]
 include::migrate_8_0/mappings.asciidoc[]
 include::migrate_8_0/snapshots.asciidoc[]

docs/reference/migration/migrate_8_0/discovery.asciidoc

@@ -0,0 +1,32 @@
+[float]
+[[breaking_80_discovery_changes]]
+=== Discovery changes
+
+[float]
+==== Removal of old discovery settings
+
+All settings under the `discovery.zen` namespace, which existed only for BWC reasons in 7.x,
+will no longer be supported. In particular, this includes:
+
+- `discovery.zen.no_master_block`
+- `discovery.zen.hosts_provider`
+- `discovery.zen.publish_timeout`
+- `discovery.zen.commit_timeout`
+- `discovery.zen.publish_diff.enable`
+- `discovery.zen.ping.unicast.concurrent_connects`
+- `discovery.zen.ping.unicast.hosts.resolve_timeout`
+- `discovery.zen.ping.unicast.hosts`
+- `discovery.zen.unsafe_rolling_upgrades_enabled`
+- `discovery.zen.commit_timeout`
+- `discovery.zen.fd.connect_on_network_disconnect`
+- `discovery.zen.fd.ping_interval`
+- `discovery.zen.fd.ping_timeout`
+- `discovery.zen.fd.ping_retries`
+- `discovery.zen.fd.register_connection_listener`
+- `discovery.zen.join_retry_attempts`
+- `discovery.zen.join_retry_delay`
+- `discovery.zen.max_pings_from_another_master`
+- `discovery.zen.send_leave_request`
+- `discovery.zen.master_election.wait_for_joins_timeout`
+- `discovery.zen.master_election.ignore_non_master_pings`
+- `discovery.zen.publish.max_pending_cluster_states`

docs/reference/modules/discovery/discovery-settings.asciidoc

@@ -8,22 +8,14 @@ Discovery and cluster formation are affected by the following settings:
     Provides a list of master-eligible nodes in the cluster. Each value has the
     format `host:port` or `host`, where `port` defaults to the setting
     `transport.profiles.default.port`. Note that IPv6 hosts must be bracketed.
-    The default value is `["127.0.0.1", "[::1]"]`. See <<unicast.hosts>>. This
-    setting was previously known as `discovery.zen.ping.unicast.hosts`. Its old
-    name is deprecated but continues to work in order to preserve backwards
-    compatibility. Support for the old name will be removed in a future
-    version.
+    The default value is `["127.0.0.1", "[::1]"]`. See <<unicast.hosts>>.
 
 `discovery.seed_providers`::
 
     Specifies which types of <<built-in-hosts-providers,seed hosts provider>>
     to use to obtain the addresses of the seed nodes used to start the
     discovery process. By default, it is the
-    <<settings-based-hosts-provider,settings-based seed hosts provider>>.  This
-    setting was previously known as `discovery.zen.hosts_provider`. Its old
-    name is deprecated but continues to work in order to preserve backwards
-    compatibility. Support for the old name will be removed in a future
-    version.
+    <<settings-based-hosts-provider,settings-based seed hosts provider>>.
 
 `discovery.cluster_formation_warning_timeout`::
 
@@ -55,20 +47,12 @@ Discovery and cluster formation are affected by the following settings:
 `discovery.seed_resolver.max_concurrent_resolvers`::
 
     Specifies how many concurrent DNS lookups to perform when resolving the
-    addresses of seed nodes. Defaults to `10`. This setting was previously
-    known as `discovery.zen.ping.unicast.concurrent_connects`. Its old name is
-    deprecated but continues to work in order to preserve backwards
-    compatibility. Support for the old name will be removed in a future
-    version.
+    addresses of seed nodes. Defaults to `10`.
 
 `discovery.seed_resolver.timeout`::
 
     Specifies how long to wait for each DNS lookup performed when resolving the
-    addresses of seed nodes. Defaults to `5s`. This setting was previously
-    known as `discovery.zen.ping.unicast.hosts.resolve_timeout`. Its old name
-    is deprecated but continues to work in order to preserve backwards
-    compatibility. Support for the old name will be removed in a future
-    version.
+    addresses of seed nodes. Defaults to `5s`.
 
 `cluster.auto_shrink_voting_configuration`::
 
@@ -192,7 +176,4 @@ APIs are not be blocked and can run on any available node.
 * For the cluster to be fully operational, it must have an active master.
 ===============================
 
-WARNING: This setting replaces the `discovery.zen.no_master_block` setting in
-earlier versions. The `discovery.zen.no_master_block` setting is ignored.
-
 --

docs/reference/modules/transport.asciidoc

@@ -156,7 +156,7 @@ PUT _cluster/settings
 {
    "transient" : {
       "transport.tracer.include" : "*",
-      "transport.tracer.exclude" : "internal:discovery/zen/fd*"
+      "transport.tracer.exclude" : "internal:coordination/fault_detection/*"
    }
 }
 --------------------------------------------------

qa/logging-config/build.gradle

@@ -23,7 +23,6 @@ apply plugin: 'elasticsearch.rest-test'
 apply plugin: 'elasticsearch.standalone-test'
 
 integTestCluster {
-    autoSetInitialMasterNodes = false
     autoSetHostsProvider = false
     /**
      * Provide a custom log4j configuration where layout is an old style pattern and confirm that Elasticsearch 

qa/rolling-upgrade/build.gradle

@@ -77,7 +77,6 @@ for (Version version : bwcVersions.wireCompatible) {
       dependsOn lastRunner, "${baseName}#oldClusterTestCluster#node${stopNode}.stop"
       clusterName = 'rolling-upgrade'
       otherUnicastHostAddresses = { getOtherUnicastHostAddresses() }
-      autoSetInitialMasterNodes = false
       /* Override the data directory so the new node always gets the node we
       * just stopped's data directory. */
       dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir }

server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.ClusterState;
 import org.elasticsearch.cluster.ClusterStateObserver;
 import org.elasticsearch.cluster.block.ClusterBlockException;
+import org.elasticsearch.cluster.coordination.PublicationTransportHandler;
 import org.elasticsearch.cluster.metadata.IndexMetaData;
 import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver;
 import org.elasticsearch.cluster.metadata.MetaData;
@@ -42,8 +43,6 @@
 import java.io.IOException;
 import java.util.function.Predicate;
 
-import static org.elasticsearch.discovery.zen.PublishClusterStateAction.serializeFullClusterState;
-
 public class TransportClusterStateAction extends TransportMasterNodeReadAction<ClusterStateRequest, ClusterStateResponse> {
 
 
@@ -127,7 +126,6 @@ private void buildResponse(final ClusterStateRequest request,
         ClusterState.Builder builder = ClusterState.builder(currentState.getClusterName());
         builder.version(currentState.version());
         builder.stateUUID(currentState.stateUUID());
-        builder.minimumMasterNodesOnPublishingMaster(currentState.getMinimumMasterNodesOnPublishingMaster());
 
         if (request.nodes()) {
             builder.nodes(currentState.nodes());
@@ -185,7 +183,7 @@ private void buildResponse(final ClusterStateRequest request,
             }
         }
         listener.onResponse(new ClusterStateResponse(currentState.getClusterName(), builder.build(),
-            serializeFullClusterState(currentState, Version.CURRENT).length(), false));
+            PublicationTransportHandler.serializeFullClusterState(currentState, Version.CURRENT).length(), false));
     }